Files
Mazyar 582f8b5c02
continuous-integration/drone/push Build is passing
Enhance TED scraper and worker initialization with startup catch-up logic
- Introduced a mutex to ensure only one TED scraper run executes at a time, preventing concurrent executions during startup and scheduled runs.
- Implemented a mechanism to check if today's TED scrape has already been completed during startup, logging appropriate messages for both completed and new runs.
- Added startup catch-up logic for tender translations and unprocessed notices, ensuring that any missed tasks are executed without blocking the application startup.

This update improves the reliability and efficiency of the TED scraper and worker processes, ensuring that all necessary tasks are completed after a server restart.
2026-06-28 00:05:10 +03:30

302 lines
8.9 KiB
Bash

#!/bin/bash
# Comprehensive Test Runner for XML Parser Package
# This script runs the complete test suite with coverage analysis and benchmarks
set -e # Exit on any error
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Configuration
PACKAGE_PATH="./..."
COVERAGE_FILE="coverage.out"
COVERAGE_HTML="coverage.html"
BENCHMARK_OUTPUT="benchmark.txt"
TEST_TIMEOUT="10m"
MIN_COVERAGE=80
echo -e "${BLUE}🚀 Starting XML Parser Test Suite${NC}"
echo "=================================="
# Function to print section headers
print_section() {
echo -e "\n${BLUE}📋 $1${NC}"
echo "----------------------------------------"
}
# Function to check if a command exists
command_exists() {
command -v "$1" >/dev/null 2>&1
}
# Clean up previous test artifacts
print_section "Cleanup"
rm -f ${COVERAGE_FILE} ${COVERAGE_HTML} ${BENCHMARK_OUTPUT}
echo "✅ Cleaned up previous test artifacts"
# Verify Go installation
if ! command_exists go; then
echo -e "${RED}❌ Go is not installed or not in PATH${NC}"
exit 1
fi
GO_VERSION=$(go version)
echo "✅ Go version: ${GO_VERSION}"
# Check if we're in the right directory
if [ ! -f "interfaces.go" ]; then
echo -e "${RED}❌ Not in xmlparser package directory${NC}"
echo "Please run this script from pkg/xmlparser directory"
exit 1
fi
echo "✅ Running from correct directory"
# Run go mod tidy to ensure dependencies are clean
print_section "Dependency Check"
if [ -f "../../go.mod" ]; then
cd ../..
go mod tidy
cd pkg/xmlparser
echo "✅ Dependencies are clean"
else
echo "⚠️ No go.mod found, skipping dependency check"
fi
# Run linting if golangci-lint is available
print_section "Code Quality Check"
if command_exists golangci-lint; then
echo "Running golangci-lint..."
if golangci-lint run --timeout=5m 2>/dev/null; then
echo "✅ Linting passed"
else
echo "⚠️ golangci-lint encountered issues, falling back to go vet"
if go vet ${PACKAGE_PATH}; then
echo "✅ go vet passed"
else
echo -e "${RED}❌ go vet failed${NC}"
exit 1
fi
fi
else
echo "⚠️ golangci-lint not found, using go vet"
# Fallback to basic go vet
if go vet ${PACKAGE_PATH}; then
echo "✅ go vet passed"
else
echo -e "${RED}❌ go vet failed${NC}"
exit 1
fi
fi
# Run go fmt check
echo "Checking code formatting..."
UNFORMATTED=$(gofmt -l .)
if [ -n "$UNFORMATTED" ]; then
echo -e "${RED}❌ Code formatting issues found:${NC}"
echo "$UNFORMATTED"
echo "Run 'gofmt -w .' to fix formatting"
exit 1
else
echo "✅ Code formatting is correct"
fi
# Run unit tests with coverage
print_section "Unit Tests"
echo "Running unit tests with coverage analysis..."
if go test -v -race -timeout=${TEST_TIMEOUT} -coverprofile=${COVERAGE_FILE} ${PACKAGE_PATH}; then
echo -e "${GREEN}✅ All unit tests passed${NC}"
else
echo -e "${RED}❌ Unit tests failed${NC}"
exit 1
fi
# Generate and display coverage report
print_section "Coverage Analysis"
if [ -f ${COVERAGE_FILE} ]; then
# Generate HTML coverage report
go tool cover -html=${COVERAGE_FILE} -o ${COVERAGE_HTML}
echo "✅ HTML coverage report generated: ${COVERAGE_HTML}"
# Extract coverage percentage
COVERAGE=$(go tool cover -func=${COVERAGE_FILE} | grep total | awk '{print $3}' | sed 's/%//')
echo "📊 Coverage Summary:"
go tool cover -func=${COVERAGE_FILE} | tail -10
echo ""
echo "📈 Total Coverage: ${COVERAGE}%"
# Check if coverage meets minimum requirement
if command_exists bc; then
if (( $(echo "$COVERAGE >= $MIN_COVERAGE" | bc -l) )); then
echo -e "${GREEN}✅ Coverage meets minimum requirement (${MIN_COVERAGE}%)${NC}"
else
echo -e "${YELLOW}⚠️ Coverage (${COVERAGE}%) is below minimum requirement (${MIN_COVERAGE}%)${NC}"
echo "Consider adding more tests to improve coverage"
fi
else
# Use shell arithmetic as fallback
if [ $(echo "$COVERAGE" | cut -d. -f1) -ge $MIN_COVERAGE ]; then
echo -e "${GREEN}✅ Coverage meets minimum requirement (${MIN_COVERAGE}%)${NC}"
else
echo -e "${YELLOW}⚠️ Coverage (${COVERAGE}%) is below minimum requirement (${MIN_COVERAGE}%)${NC}"
echo "Consider adding more tests to improve coverage"
fi
fi
else
echo -e "${RED}❌ Coverage file not generated${NC}"
fi
# Run benchmarks
print_section "Performance Benchmarks"
echo "Running performance benchmarks..."
if go test -bench=. -benchmem -run=^$ -timeout=${TEST_TIMEOUT} > ${BENCHMARK_OUTPUT} 2>&1; then
echo -e "${GREEN}✅ Benchmarks completed successfully${NC}"
echo ""
echo "📊 Benchmark Results:"
echo "====================="
cat ${BENCHMARK_OUTPUT} | grep -E "(Benchmark|PASS|FAIL)"
echo ""
echo "🏆 Top 5 Fastest Operations:"
cat ${BENCHMARK_OUTPUT} | grep "^Benchmark" | sort -k3 -n | head -5
echo ""
echo "⚠️ Top 5 Memory Intensive Operations:"
cat ${BENCHMARK_OUTPUT} | grep "^Benchmark" | grep "allocs/op" | sort -k5 -nr | head -5
else
echo -e "${YELLOW}⚠️ Some benchmarks may have failed${NC}"
echo "Benchmark output:"
cat ${BENCHMARK_OUTPUT}
fi
# Run specific test categories
print_section "Test Categories"
echo "Running interface tests..."
if go test -v -run=TestDefaultParserOptions,TestXMLMetadata,TestParseResult,TestMockParsable,TestParserInterface -timeout=30s; then
echo "✅ Interface tests passed"
else
echo -e "${RED}❌ Interface tests failed${NC}"
fi
echo ""
echo "Running parser tests..."
if go test -v -run=TestNewParser,TestParseString,TestParseBytes,TestParse,TestValidate,TestParseWithResult -timeout=1m; then
echo "✅ Parser tests passed"
else
echo -e "${RED}❌ Parser tests failed${NC}"
fi
echo ""
echo "Running utility tests..."
if go test -v -run=TestValidationError,TestSanitizeXMLString,TestValidateXMLName,TestIsValidNamespace,TestNormalizeWhitespace,TestTruncateString -timeout=30s; then
echo "✅ Utility tests passed"
else
echo -e "${RED}❌ Utility tests failed${NC}"
fi
# Test with different Go build tags if needed
print_section "Build Verification"
echo "Verifying package builds correctly..."
if go build -v ${PACKAGE_PATH}; then
echo "✅ Package builds successfully"
else
echo -e "${RED}❌ Package build failed${NC}"
exit 1
fi
# Run race condition detection
print_section "Race Condition Detection"
echo "Running tests with race detector..."
if go test -race -run=TestConcurrent -timeout=2m; then
echo "✅ No race conditions detected"
else
echo -e "${YELLOW}⚠️ Race condition tests completed with warnings${NC}"
fi
# Memory leak detection (basic)
print_section "Memory Analysis"
echo "Running memory allocation tests..."
if go test -run=TestParserMemoryUsage -timeout=1m; then
echo "✅ Memory tests passed"
else
echo -e "${YELLOW}⚠️ Memory tests completed with warnings${NC}"
fi
# Run TED model tests if they exist
if [ -d "ted" ]; then
print_section "TED Model Tests"
echo "Running TED XML model tests..."
if go test -v ./ted -timeout=1m; then
echo "✅ TED model tests passed"
else
echo -e "${RED}❌ TED model tests failed${NC}"
fi
fi
# Performance regression check (if baseline exists)
if [ -f "benchmark_baseline.txt" ]; then
print_section "Performance Regression Check"
echo "Comparing with baseline performance..."
if command_exists benchcmp; then
benchcmp benchmark_baseline.txt ${BENCHMARK_OUTPUT}
echo "✅ Performance comparison completed"
else
echo "⚠️ benchcmp not available, skipping performance regression check"
echo "Install with: go install golang.org/x/tools/cmd/benchcmp@latest"
fi
else
echo "💡 Consider creating a performance baseline:"
echo " cp ${BENCHMARK_OUTPUT} benchmark_baseline.txt"
fi
# Test summary
print_section "Test Summary"
echo "📋 Test Suite Completed"
echo "======================="
echo "✅ Unit Tests: PASSED"
echo "📊 Coverage: ${COVERAGE}%"
echo "🏆 Benchmarks: COMPLETED"
echo "🔍 Race Detection: PASSED"
echo "📁 Generated Files:"
echo " - ${COVERAGE_HTML} (HTML coverage report)"
echo " - ${COVERAGE_FILE} (Coverage data)"
echo " - ${BENCHMARK_OUTPUT} (Benchmark results)"
if [ -f ${COVERAGE_HTML} ]; then
echo ""
echo "🌐 To view coverage report in browser:"
echo " open ${COVERAGE_HTML}"
fi
echo ""
echo -e "${GREEN}🎉 All tests completed successfully!${NC}"
# Cleanup intermediate files (optional)
read -p "🗑️ Remove intermediate files? (y/N): " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
rm -f ${COVERAGE_FILE}
echo "✅ Intermediate files cleaned up"
fi
echo "=================================="
echo -e "${BLUE}🏁 XML Parser Test Suite Complete${NC}"