tm_back/ted/scraper.go

package ted

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"tm/internal/tender"
	"tm/pkg/logger"
	"tm/pkg/mongo"

	"go.mongodb.org/mongo-driver/v2/bson"
	mongoDriver "go.mongodb.org/mongo-driver/v2/mongo"
	"go.mongodb.org/mongo-driver/v2/mongo/options"
)

// TEDScraper handles downloading and parsing TED XML files
type TEDScraper struct {
	config       *Config
	logger       logger.Logger
	httpClient   *http.Client
	xmlParser    *TEDParser
	mongoManager *mongo.ConnectionManager
	tenderRepo   tender.TenderRepository
}

// ScrapingJob represents a scraping job (simplified version)
type ScrapingJob struct {
	mongo.Model
	Type        string                 `bson:"type" json:"type"`
	Status      string                 `bson:"status" json:"status"`
	StartedAt   int64                  `bson:"started_at" json:"started_at"`
	CompletedAt *int64                 `bson:"completed_at,omitempty" json:"completed_at,omitempty"`
	Progress    JobProgress            `bson:"progress" json:"progress"`
	Results     JobResults             `bson:"results" json:"results"`
	Config      map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"`
}

// JobProgress represents the progress of a scraping job
type JobProgress struct {
	TotalFiles      int     `bson:"total_files" json:"total_files"`
	ProcessedFiles  int     `bson:"processed_files" json:"processed_files"`
	FailedFiles     int     `bson:"failed_files" json:"failed_files"`
	PercentComplete float64 `bson:"percent_complete" json:"percent_complete"`
	CurrentFile     string  `bson:"current_file,omitempty" json:"current_file,omitempty"`
}

// JobResults represents the results of a scraping job
type JobResults struct {
	TotalTenders   int     `bson:"total_tenders" json:"total_tenders"`
	NewTenders     int     `bson:"new_tenders" json:"new_tenders"`
	UpdatedTenders int     `bson:"updated_tenders" json:"updated_tenders"`
	ErrorCount     int     `bson:"error_count" json:"error_count"`
	SuccessRate    float64 `bson:"success_rate" json:"success_rate"`
	ProcessingTime int64   `bson:"processing_time" json:"processing_time"`
}

// ScrapingState represents the current state of scraping
type ScrapingState struct {
	mongo.Model
	LastProcessedYear   int           `bson:"last_processed_year" json:"last_processed_year"`
	LastProcessedNumber int           `bson:"last_processed_number" json:"last_processed_number"`
	LastRunAt           int64         `bson:"last_run_at" json:"last_run_at"`
	NextRunAt           int64         `bson:"next_run_at" json:"next_run_at"`
	IsRunning           bool          `bson:"is_running" json:"is_running"`
	CurrentJobID        bson.ObjectID `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
}

// ScraperConfig holds configuration for the TED scraper
type Config struct {
	BaseURL          string        `mapstructure:"base_url"`
	Timeout          time.Duration `mapstructure:"timeout"`
	MaxRetries       int           `mapstructure:"max_retries"`
	RetryDelay       time.Duration `mapstructure:"retry_delay"`
	UserAgent        string        `mapstructure:"user_agent"`
	MaxConcurrency   int           `mapstructure:"max_concurrency"`
	DownloadDir      string        `mapstructure:"download_dir"`
	CleanupAfter     time.Duration `mapstructure:"cleanup_after"`
	ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
}

// ScrapingMode represents the type of scraping operation
type ScrapingMode string

const (
	ScrapingModeDaily    ScrapingMode = "daily"    // Daily bulk fetch mode
	ScrapingModeManual   ScrapingMode = "manual"   // Manual date range mode
	ScrapingModeRealtime ScrapingMode = "realtime" // Real-time processing mode
)

// DateRangeConfig holds configuration for manual date range scraping
type DateRangeConfig struct {
	StartDate time.Time    `json:"start_date"`
	EndDate   time.Time    `json:"end_date"`
	Mode      ScrapingMode `json:"mode"`
}

// NewTEDScraper creates a new TED scraper instance
func NewTEDScraper(
	config *Config,
	logger logger.Logger,
	mongoManager *mongo.ConnectionManager,
	tenderRepo tender.TenderRepository,
) *TEDScraper {
	httpClient := &http.Client{
		Timeout: config.Timeout,
	}

	return &TEDScraper{
		config:       config,
		logger:       logger,
		httpClient:   httpClient,
		xmlParser:    NewTEDParser(),
		mongoManager: mongoManager,
		tenderRepo:   tenderRepo,
	}
}

// RunPeriodicScraping runs the scraper at configured intervals
func (s *TEDScraper) RunPeriodicScraping(ctx context.Context) error {
	ticker := time.NewTicker(s.config.ScrapingInterval)
	defer ticker.Stop()

	s.logger.Info("Starting periodic TED XML scraping", map[string]interface{}{
		"interval": s.config.ScrapingInterval.String(),
		"base_url": s.config.BaseURL,
	})

	// Run initial scraping
	if err := s.ScrapeLatest(ctx); err != nil {
		s.logger.Error("Initial scraping failed", map[string]interface{}{
			"error": err.Error(),
		})
	}

	for {
		select {
		case <-ctx.Done():
			s.logger.Info("Periodic scraping stopped", map[string]interface{}{
				"reason": ctx.Err().Error(),
			})
			return ctx.Err()
		case <-ticker.C:
			if err := s.ScrapeLatest(ctx); err != nil {
				s.logger.Error("Periodic scraping failed", map[string]interface{}{
					"error": err.Error(),
				})
			}
		}
	}
}

// ScrapeLatest scrapes the latest TED XML files
func (s *TEDScraper) ScrapeLatest(ctx context.Context) error {
	startTime := time.Now()

	s.logger.Info("Starting TED XML scraping", map[string]interface{}{
		"timestamp": startTime.Unix(),
	})

	// Get current state
	state, err := s.getScrapingState(ctx)
	if err != nil {
		s.logger.Error("Failed to get scraping state", map[string]interface{}{
			"error": err.Error(),
		})
		// Initialize with current date if no state exists
		state = &ScrapingState{
			LastProcessedYear:   time.Now().Year(),
			LastProcessedNumber: 0,
			LastRunAt:           startTime.Unix(),
			NextRunAt:           startTime.Add(s.config.ScrapingInterval).Unix(),
			IsRunning:           false,
		}
	}

	// Handle case where state is nil (no document found)
	if state == nil {
		s.logger.Info("No scraping state found, initializing new state", map[string]interface{}{})
		state = &ScrapingState{
			LastProcessedYear:   time.Now().Year(),
			LastProcessedNumber: 0,
			LastRunAt:           startTime.Unix(),
			NextRunAt:           startTime.Add(s.config.ScrapingInterval).Unix(),
			IsRunning:           false,
		}
	}

	if state.IsRunning {
		s.logger.Info("Scraping already running, skipping", map[string]interface{}{
			"current_job_id": state.CurrentJobID,
		})
		return nil
	}

	// Create scraping job
	job := &ScrapingJob{
		Type:      "ted_daily",
		Status:    "running",
		StartedAt: startTime.Unix(),
		Progress:  JobProgress{},
		Results:   JobResults{},
		Config: map[string]interface{}{
			"base_url":     s.config.BaseURL,
			"max_retries":  s.config.MaxRetries,
			"scraper_type": "ted_daily",
		},
	}

	// Save job
	if err := s.saveScrapingJob(ctx, job); err != nil {
		s.logger.Error("Failed to save scraping job", map[string]interface{}{
			"error": err.Error(),
		})
	}

	// Update state
	state.IsRunning = true
	state.CurrentJobID = job.ID
	state.UpdatedAt = time.Now().Unix()
	s.saveScrapingState(ctx, state)

	// Calculate files to download
	filesToDownload := s.calculateFilesToDownload(state)
	if len(filesToDownload) == 0 {
		s.logger.Info("No new files to download", map[string]interface{}{
			"last_processed": fmt.Sprintf("%d%05d", state.LastProcessedYear, state.LastProcessedNumber),
		})

		// Complete the job
		s.completeJob(ctx, job, state, nil)
		return nil
	}

	// Update job progress
	job.Progress.TotalFiles = len(filesToDownload)
	s.saveScrapingJob(ctx, job)

	var lastSuccessfulFile *FileInfo

	// Download and process files
	for i, fileInfo := range filesToDownload {
		job.Progress.CurrentFile = fmt.Sprintf("%d%05d", fileInfo.Year, fileInfo.Number)
		s.saveScrapingJob(ctx, job)

		fileResult, err := s.downloadAndProcess(ctx, fileInfo)
		if err != nil {
			s.logger.Error("Failed to download and process file", map[string]interface{}{
				"year":   fileInfo.Year,
				"number": fileInfo.Number,
				"error":  err.Error(),
			})
			job.Progress.FailedFiles++
			job.Results.ErrorCount++
		} else {
			job.Progress.ProcessedFiles++
			job.Results.TotalTenders += fileResult.ProcessedCount
			job.Results.NewTenders += fileResult.NewTenders
			job.Results.UpdatedTenders += fileResult.UpdatedTenders
			if len(fileResult.Errors) > 0 {
				job.Results.ErrorCount += len(fileResult.Errors)
			}
			lastSuccessfulFile = &fileInfo
		}

		// Update progress percentage
		job.Progress.PercentComplete = float64(i+1) / float64(len(filesToDownload)) * 100
		s.saveScrapingJob(ctx, job)

		// Check context for cancellation
		select {
		case <-ctx.Done():
			s.completeJob(ctx, job, state, lastSuccessfulFile)
			return ctx.Err()
		default:
			// Continue processing
		}
	}

	// Complete the job
	s.completeJob(ctx, job, state, lastSuccessfulFile)

	s.logger.Info("TED XML scraping completed", map[string]interface{}{
		"files_processed": job.Progress.ProcessedFiles,
		"tenders_created": job.Results.NewTenders,
		"tenders_updated": job.Results.UpdatedTenders,
		"total_tenders":   job.Results.TotalTenders,
		"error_count":     job.Results.ErrorCount,
		"success_rate":    job.Results.SuccessRate,
	})

	return nil
}

// FileInfo represents information about a TED file to download
type FileInfo struct {
	Year   int
	Number int
	URL    string
	OJS    string
}

// calculateFilesToDownload determines which files need to be downloaded
func (s *TEDScraper) calculateFilesToDownload(state *ScrapingState) []FileInfo {
	var files []FileInfo
	now := time.Now()
	currentYear := now.Year()
	currentMonth := now.Month()

	// Only process current year and month
	year := currentYear

	// Start from the last processed file if it's from current year, otherwise start from beginning of current month
	startNumber := 1
	if state.LastProcessedYear == currentYear {
		startNumber = state.LastProcessedNumber + 1
	} else {
		// Calculate approximate file number for start of current month
		// Assuming roughly 22 working days per month (weekdays only)
		startNumber = int(currentMonth-1)*22 + 1
	}

	// Calculate end number for current month
	// Estimate based on current day of month and weekdays
	daysInCurrentMonth := now.Day()
	// Rough estimate: multiply by 0.7 to account for weekdays only
	maxNumber := int(currentMonth-1)*22 + int(float64(daysInCurrentMonth)*0.7)

	// Add some buffer for potential files
	maxNumber += 5

	s.logger.Info("Calculating files to download for current month", map[string]interface{}{
		"current_year":   currentYear,
		"current_month":  int(currentMonth),
		"start_number":   startNumber,
		"max_number":     maxNumber,
		"last_processed": fmt.Sprintf("%d%05d", state.LastProcessedYear, state.LastProcessedNumber),
	})

	startNumber = 146

	for number := startNumber; number <= maxNumber && len(files) < 10; number++ { // Limit to 10 files per run
		fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number)
		ojsID := fmt.Sprintf("S %03d / %d", number, year)

		files = append(files, FileInfo{
			Year:   year,
			Number: number,
			URL:    fileURL,
			OJS:    ojsID,
		})
	}

	return files
}

// FileProcessingResult represents the result of processing a single file
type FileProcessingResult struct {
	Year           int      `json:"year"`
	Number         int      `json:"number"`
	ProcessedCount int      `json:"processed_count"`
	SuccessCount   int      `json:"success_count"`
	NewTenders     int      `json:"new_tenders"`
	UpdatedTenders int      `json:"updated_tenders"`
	Errors         []string `json:"errors"`
	ProcessedAt    int64    `json:"processed_at"`
}

// downloadAndProcess downloads and processes a single TED file
func (s *TEDScraper) downloadAndProcess(ctx context.Context, fileInfo FileInfo) (*FileProcessingResult, error) {
	url := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, fileInfo.Year, fileInfo.Number)

	s.logger.Info("Downloading TED file", map[string]interface{}{
		"url":    url,
		"year":   fileInfo.Year,
		"number": fileInfo.Number,
	})

	var result *FileProcessingResult
	var err error

	// Retry mechanism
	for attempt := 1; attempt <= s.config.MaxRetries; attempt++ {
		result, err = s.downloadAndProcessWithRetry(ctx, url, fileInfo)
		if err == nil {
			break
		}

		if attempt < s.config.MaxRetries {
			s.logger.Warn("Download attempt failed, retrying", map[string]interface{}{
				"attempt": attempt,
				"error":   err.Error(),
				"url":     url,
			})

			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			case <-time.After(s.config.RetryDelay):
				// Continue to next attempt
			}
		}
	}

	if err != nil {
		s.logger.Error("Failed to download and process file after retries", map[string]interface{}{
			"url":      url,
			"attempts": s.config.MaxRetries,
			"error":    err.Error(),
		})
		return nil, fmt.Errorf("failed to download %s after %d attempts: %w", url, s.config.MaxRetries, err)
	}

	return result, nil
}

// downloadAndProcessWithRetry performs a single download and process attempt
func (s *TEDScraper) downloadAndProcessWithRetry(ctx context.Context, url string, fileInfo FileInfo) (*FileProcessingResult, error) {
	// Create a longer timeout context for processing large files
	processCtx, cancel := context.WithTimeout(ctx, s.config.Timeout+time.Minute*5)
	defer cancel()

	// Create HTTP request
	req, err := http.NewRequestWithContext(processCtx, "GET", url, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request: %w", err)
	}

	req.Header.Set("User-Agent", s.config.UserAgent)

	s.logger.Info("Starting download", map[string]interface{}{
		"url":    url,
		"year":   fileInfo.Year,
		"number": fileInfo.Number,
	})

	// Make request
	resp, err := s.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to download file: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("HTTP error: %d %s", resp.StatusCode, resp.Status)
	}

	s.logger.Info("Download completed, starting processing", map[string]interface{}{
		"url":            url,
		"content_length": resp.Header.Get("Content-Length"),
	})

	// Process tar.gz file directly from response body
	return s.processTarGzFile(processCtx, resp.Body, fileInfo)
}

// processTarGzFile extracts and processes XML files from a tar.gz archive
func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fileInfo FileInfo) (*FileProcessingResult, error) {
	result := &FileProcessingResult{
		Year:           fileInfo.Year,
		Number:         fileInfo.Number,
		ProcessedCount: 0,
		SuccessCount:   0,
		Errors:         make([]string, 0),
		ProcessedAt:    time.Now().Unix(),
	}

	// Create gzip reader
	gzipReader, err := gzip.NewReader(reader)
	if err != nil {
		return nil, fmt.Errorf("failed to create gzip reader: %w", err)
	}
	defer gzipReader.Close()

	// Create tar reader
	tarReader := tar.NewReader(gzipReader)

	// Process each file in the tar archive
	fileCount := 0
	for {
		// Check context for cancellation before each tar entry
		select {
		case <-ctx.Done():
			s.logger.Warn("Context cancelled during tar processing", map[string]interface{}{
				"processed_files": fileCount,
				"error":           ctx.Err().Error(),
			})
			return result, ctx.Err()
		default:
		}

		header, err := tarReader.Next()
		if err == io.EOF {
			break // End of archive
		}
		if err != nil {
			s.logger.Warn("Error reading tar entry", map[string]interface{}{
				"error":           err.Error(),
				"processed_files": fileCount,
			})
			// If it's a timeout error, return it immediately
			if ctx.Err() != nil {
				return result, fmt.Errorf("timeout during tar processing: %w", err)
			}
			continue
		}
		fileCount++

		// Skip directories and non-XML files
		if header.Typeflag != tar.TypeReg {
			continue
		}

		fileName := filepath.Base(header.Name)
		if !strings.HasSuffix(strings.ToLower(fileName), ".xml") {
			continue
		}

		// Log progress every 10 files or for first file
		if result.ProcessedCount == 0 || result.ProcessedCount%10 == 0 {
			s.logger.Info("Processing XML file from tar.gz", map[string]interface{}{
				"file_name":        fileName,
				"size":             header.Size,
				"processed_count":  result.ProcessedCount,
				"total_files_seen": fileCount,
				"year":             fileInfo.Year,
				"number":           fileInfo.Number,
			})
		} else {
			s.logger.Debug("Processing XML file from tar.gz", map[string]interface{}{
				"file_name":       fileName,
				"processed_count": result.ProcessedCount,
			})
		}

		// Read file content
		content, err := io.ReadAll(tarReader)
		if err != nil {
			errMsg := fmt.Sprintf("failed to read XML file %s: %v", fileName, err)
			result.Errors = append(result.Errors, errMsg)
			continue
		}

		// Parse and process XML content
		processResult, err := s.processXMLContent(ctx, content, fileName, fileInfo)
		if err != nil {
			errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err)
			result.Errors = append(result.Errors, errMsg)
		} else {
			result.SuccessCount++
			if processResult.IsNew {
				result.NewTenders++
			} else {
				result.UpdatedTenders++
			}
		}

		result.ProcessedCount++

		// Check context for cancellation
		select {
		case <-ctx.Done():
			return result, ctx.Err()
		default:
			// Continue processing
		}
	}

	s.logger.Info("Completed processing tar.gz file", map[string]interface{}{
		"year":            fileInfo.Year,
		"number":          fileInfo.Number,
		"total_files":     fileCount,
		"processed_count": result.ProcessedCount,
		"success_count":   result.SuccessCount,
		"error_count":     len(result.Errors),
		"processing_time": time.Now().Unix() - result.ProcessedAt,
	})

	return result, nil
}

// XMLProcessResult represents the result of processing a single XML file
type XMLProcessResult struct {
	IsNew      bool   `json:"is_new"`
	TenderID   string `json:"tender_id"`
	ContractID string `json:"contract_id"`
}

// processXMLContent parses XML content and creates/updates tender
func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo) (*XMLProcessResult, error) {
	s.logger.Info("Starting XML processing", map[string]interface{}{
		"file_name": fileName,
		"file_size": len(content),
	})

	// Parse XML using the TED parser
	parsedDoc, err := s.xmlParser.ParseXML(content)
	if err != nil {
		s.logger.Error("XML parsing failed", map[string]interface{}{
			"file_name": fileName,
			"error":     err.Error(),
		})
		return nil, fmt.Errorf("failed to parse XML: %w", err)
	}

	// Get the underlying document for ID and validation
	document := parsedDoc.GetDocument()
	documentID := ""
	noticeTypeCode := ""
	if document != nil {
		documentID = document.GetID()
		noticeTypeCode = document.GetNoticeType()
	}

	s.logger.Info("XML parsed successfully", map[string]interface{}{
		"file_name":        fileName,
		"document_type":    string(parsedDoc.Type),
		"document_id":      documentID,
		"notice_type_code": noticeTypeCode,
	})

	// Validate parsed data
	var validationErr error
	if document != nil {
		validationErr = document.Validate()
	}
	if validationErr != nil {
		s.logger.Warn("XML validation failed, continuing anyway", map[string]interface{}{
			"file_name":     fileName,
			"document_type": string(parsedDoc.Type),
			"error":         validationErr.Error(),
		})
		// Don't return error, continue processing
	} else {
		s.logger.Info("XML validation passed", map[string]interface{}{
			"file_name":     fileName,
			"document_type": string(parsedDoc.Type),
		})
	}

	// Map to tender entity
	sourceURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, fileInfo.Year, fileInfo.Number)
	tenderEntity := s.mapToTenderFromParsedDoc(parsedDoc, sourceURL, fileName)

	// Handle status-specific information and award logic
	s.applyStatusSpecificInformation(tenderEntity, parsedDoc)
	s.applyAwardLogic(tenderEntity, parsedDoc)

	s.logger.Info("Mapped to tender entity", map[string]interface{}{
		"file_name":          fileName,
		"tender_id":          tenderEntity.TenderID,
		"contract_notice_id": tenderEntity.ContractNoticeID,
		"title":              tenderEntity.Title,
		"status":             string(tenderEntity.Status),
	})

	// Try to find existing tender by contract notice ID (ContractID)
	s.logger.Info("Checking for existing tender", map[string]interface{}{
		"contract_notice_id": tenderEntity.ContractNoticeID,
	})

	existingTender, err := s.findTenderByContractNoticeID(ctx, tenderEntity.ContractNoticeID)
	if err != nil && err.Error() != "document not found" {
		s.logger.Warn("Error checking for existing tender", map[string]interface{}{
			"contract_notice_id": tenderEntity.ContractNoticeID,
			"error":              err.Error(),
		})
	}

	var processResult *XMLProcessResult

	if err == nil && existingTender != nil {
		// Update existing tender
		s.logger.Info("Found existing tender, updating", map[string]interface{}{
			"existing_tender_id": existingTender.ID,
			"contract_notice_id": tenderEntity.ContractNoticeID,
			"old_status":         string(existingTender.Status),
			"new_status":         string(tenderEntity.Status),
		})

		// Merge updates with existing data
		s.mergeExistingTenderData(existingTender, tenderEntity)

		tenderEntity.ID = existingTender.ID
		tenderEntity.CreatedAt = existingTender.CreatedAt
		tenderEntity.UpdatedAt = time.Now().Unix()

		err = s.tenderRepo.Update(ctx, tenderEntity)
		if err != nil {
			s.logger.Error("Failed to update existing tender", map[string]interface{}{
				"tender_id": tenderEntity.ID,
				"error":     err.Error(),
			})
			return nil, fmt.Errorf("failed to update existing tender: %w", err)
		}

		s.logger.Info("Successfully updated tender", map[string]interface{}{
			"tender_id": tenderEntity.ID,
		})

		processResult = &XMLProcessResult{
			IsNew:      false,
			TenderID:   tenderEntity.TenderID,
			ContractID: tenderEntity.ContractNoticeID,
		}
	} else {
		// Create new tender
		s.logger.Info("Creating new tender", map[string]interface{}{
			"contract_notice_id": tenderEntity.ContractNoticeID,
			"tender_id":          tenderEntity.TenderID,
			"status":             string(tenderEntity.Status),
		})

		tenderEntity.CreatedAt = time.Now().Unix()
		tenderEntity.UpdatedAt = tenderEntity.CreatedAt

		err = s.tenderRepo.Create(ctx, tenderEntity)
		if err != nil {
			s.logger.Error("Failed to create new tender", map[string]interface{}{
				"contract_notice_id": tenderEntity.ContractNoticeID,
				"tender_id":          tenderEntity.TenderID,
				"error":              err.Error(),
			})
			return nil, fmt.Errorf("failed to create tender: %w", err)
		}

		s.logger.Info("Successfully created new tender", map[string]interface{}{
			"tender_id":          tenderEntity.TenderID,
			"contract_notice_id": tenderEntity.ContractNoticeID,
		})

		processResult = &XMLProcessResult{
			IsNew:      true,
			TenderID:   tenderEntity.TenderID,
			ContractID: tenderEntity.ContractNoticeID,
		}
	}

	s.logger.Debug("Successfully processed XML file", map[string]interface{}{
		"file_name":     fileName,
		"document_type": string(parsedDoc.Type),
		"document_id":   documentID,
		"tender_id":     tenderEntity.TenderID,
		"is_new":        processResult.IsNew,
	})

	return processResult, nil
}

// Helper methods for database operations
func (s *TEDScraper) getScrapingState(ctx context.Context) (*ScrapingState, error) {
	collection := s.mongoManager.GetCollection("scraping_state")
	var state ScrapingState
	err := collection.FindOne(ctx, bson.M{"_id": "ted_scraper"}).Decode(&state)
	if err != nil {
		if err == mongoDriver.ErrNoDocuments {
			return nil, nil
		}
		return nil, err
	}
	return &state, nil
}

func (s *TEDScraper) saveScrapingState(ctx context.Context, state *ScrapingState) error {
	collection := s.mongoManager.GetCollection("scraping_state")
	state.UpdatedAt = time.Now().Unix()
	opts := options.Replace().SetUpsert(true)
	_, err := collection.ReplaceOne(ctx, bson.M{"_id": state.ID}, state, opts)
	return err
}

func (s *TEDScraper) saveScrapingJob(ctx context.Context, job *ScrapingJob) error {
	collection := s.mongoManager.GetCollection("scraping_jobs")
	job.UpdatedAt = time.Now().Unix()
	opts := options.Replace().SetUpsert(true)
	_, err := collection.ReplaceOne(ctx, bson.M{"_id": job.ID}, job, opts)
	return err
}

func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*tender.Tender, error) {
	// Use repository's specific method for finding by contract notice ID
	existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID)
	if err != nil {
		// If document not found, return nil without error
		if err.Error() == "document not found" {
			return nil, nil
		}
		return nil, err
	}

	return existingTender, nil
}

func (s *TEDScraper) completeJob(ctx context.Context, job *ScrapingJob, state *ScrapingState, lastFile *FileInfo) {
	// Update state with last successful file
	if lastFile != nil {
		state.LastProcessedYear = lastFile.Year
		state.LastProcessedNumber = lastFile.Number
	}

	// Complete the job
	endTime := time.Now()
	completedAt := endTime.Unix()
	job.Status = "completed"
	job.CompletedAt = &completedAt
	job.Results.ProcessingTime = completedAt - job.StartedAt
	if job.Results.TotalTenders > 0 {
		job.Results.SuccessRate = float64(job.Results.NewTenders) / float64(job.Results.TotalTenders)
	}
	s.saveScrapingJob(ctx, job)

	// Update state
	state.IsRunning = false
	state.CurrentJobID = bson.NilObjectID
	state.LastRunAt = completedAt
	state.NextRunAt = endTime.Add(s.config.ScrapingInterval).Unix()
	state.UpdatedAt = completedAt
	s.saveScrapingState(ctx, state)
}

// mapToTenderFromParsedDoc maps a parsed TED document to tender entity
func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *tender.Tender {
	switch parsedDoc.Type {
	case DocumentTypeContractNotice:
		if parsedDoc.ContractNotice == nil {
			s.logger.Error("ContractNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapToTender(parsedDoc.ContractNotice, sourceFileURL, sourceFileName)

	case DocumentTypeContractAwardNotice:
		if parsedDoc.ContractAwardNotice == nil {
			s.logger.Error("ContractAwardNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapContractAwardNoticeToTender(parsedDoc.ContractAwardNotice, sourceFileURL, sourceFileName)

	case DocumentTypePriorInformationNotice:
		if parsedDoc.PriorInformationNotice == nil {
			s.logger.Error("PriorInformationNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.PriorInformationNotice, sourceFileURL, sourceFileName)

	case DocumentTypeDesignContest:
		if parsedDoc.DesignContest == nil {
			s.logger.Error("DesignContest is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.DesignContest, sourceFileURL, sourceFileName)

	case DocumentTypeQualificationSystemNotice:
		if parsedDoc.QualificationSystemNotice == nil {
			s.logger.Error("QualificationSystemNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.QualificationSystemNotice, sourceFileURL, sourceFileName)

	case DocumentTypeConcessionNotice:
		if parsedDoc.ConcessionNotice == nil {
			s.logger.Error("ConcessionNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.ConcessionNotice, sourceFileURL, sourceFileName)

	case DocumentTypePlanningNotice:
		if parsedDoc.PlanningNotice == nil {
			s.logger.Error("PlanningNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.PlanningNotice, sourceFileURL, sourceFileName)

	case DocumentTypeCompetitionNotice:
		if parsedDoc.CompetitionNotice == nil {
			s.logger.Error("CompetitionNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.CompetitionNotice, sourceFileURL, sourceFileName)

	case DocumentTypeResultNotice:
		if parsedDoc.ResultNotice == nil {
			s.logger.Error("ResultNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.ResultNotice, sourceFileURL, sourceFileName)

	case DocumentTypeChangeNotice:
		if parsedDoc.ChangeNotice == nil {
			s.logger.Error("ChangeNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.ChangeNotice, sourceFileURL, sourceFileName)

	case DocumentTypeSubcontractNotice:
		if parsedDoc.SubcontractNotice == nil {
			s.logger.Error("SubcontractNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.SubcontractNotice, sourceFileURL, sourceFileName)

	// Legacy document types that should be parsed as ContractNotice but may still be in the old flow
	case DocumentTypeCorrigendum, DocumentTypeModificationNotice:
		if parsedDoc.ContractNotice != nil {
			return s.mapToTender(parsedDoc.ContractNotice, sourceFileURL, sourceFileName)
		}
		// Fallback: try to get the document from the generic interface
		document := parsedDoc.GetDocument()
		if document != nil {
			return s.mapGenericNoticeToTender(document, sourceFileURL, sourceFileName)
		}
		s.logger.Error("No valid document found for legacy type", map[string]interface{}{
			"document_type": string(parsedDoc.Type),
		})
		return nil

	default:
		s.logger.Error("Unknown document type for mapping", map[string]interface{}{
			"document_type": string(parsedDoc.Type),
		})
		return nil
	}
}

// mapGenericNoticeToTender maps any TED document type to tender entity using the common interface
func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *tender.Tender {
	if doc == nil {
		s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{})
		return nil
	}

	now := time.Now().Unix()

	t := &tender.Tender{
		ContractNoticeID:    doc.GetID(),
		ContractFolderID:    "", // Not available in generic interface
		NoticeTypeCode:      doc.GetNoticeType(),
		NoticeSubTypeCode:   "", // Not available in generic interface
		NoticeLanguageCode:  doc.GetLanguage(),
		IssueDate:           now, // TODO: Parse actual date from document
		IssueTime:           now, // TODO: Parse actual time from document
		GazetteID:           "",  // Not available in generic interface
		NoticePublicationID: "",  // Not available in generic interface
		PublicationDate:     now, // TODO: Parse actual publication date

		// Basic tender information
		Title:          "Generic Notice " + doc.GetNoticeType(),
		Description:    "Processed from " + doc.GetNoticeType() + " document",
		Status:         tender.TenderStatusActive,
		Source:         tender.TenderSourceTEDScraper,
		SourceFileURL:  sourceFileURL,
		SourceFileName: sourceFileName,
		ProcessingMetadata: tender.ProcessingMetadata{
			ScrapedAt:         now,
			ProcessedAt:       now,
			ProcessingVersion: "1.0",
		},
	}

	// Generate the tender ID after creating the tender
	t.TenderID = s.generateTenderID(t)

	return t
}

// mapToTender maps a TED ContractNotice to tender entity
func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *tender.Tender {
	now := time.Now().Unix()

	t := &tender.Tender{
		ContractNoticeID:    cn.ID,
		ContractFolderID:    cn.ContractFolderID,
		NoticeTypeCode:      cn.GetNoticeSubType(),
		NoticeSubTypeCode:   cn.GetNoticeSubType(),
		NoticeLanguageCode:  cn.NoticeLanguageCode,
		IssueDate:           s.parseDateToUnix(cn.IssueDate),
		IssueTime:           s.parseTimeToUnix(cn.IssueTime),
		GazetteID:           cn.GetOJSID(),
		Title:               cn.GetProcurementTitle(),
		Description:         cn.GetProcurementDescription(),
		ProcurementTypeCode: cn.GetContractNature(),
		ProcedureCode:       cn.GetProcedureCode(),
		MainClassification:  cn.GetMainClassification(),
		PlaceOfPerformance:  cn.GetPlaceOfPerformance(),
		DocumentURI:         cn.GetDocumentURI(),
		Status:              tender.TenderStatusActive,
		Source:              tender.TenderSourceTEDScraper,
		SourceFileURL:       sourceFileURL,
		SourceFileName:      sourceFileName,
		ProcessingMetadata: tender.ProcessingMetadata{
			ScrapedAt:         now,
			ProcessedAt:       now,
			ProcessingVersion: "1.0",
		},
	}

	// Set publication information
	if pubInfo := cn.GetPublicationInfo(); pubInfo != nil {
		t.NoticePublicationID = pubInfo.NoticePublicationID
		t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate)
		t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
	}

	// Set additional classifications
	if cn.ProcurementProject != nil && cn.ProcurementProject.AdditionalCommodityClassification != nil {
		for _, additionalClass := range cn.ProcurementProject.AdditionalCommodityClassification {
			if additionalClass.ItemClassificationCode != "" {
				t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode)
			}
		}
	}

	// Set estimated value and currency
	if amount, currency := cn.GetBudget(); amount != "" {
		if value, err := strconv.ParseFloat(amount, 64); err == nil {
			t.EstimatedValue = value
			t.Currency = currency
		}
	}

	// Set duration information
	if duration, unit := cn.GetDuration(); duration != "" {
		t.Duration = duration
		t.DurationUnit = unit
	}

	// Set tender deadline
	if deadline := cn.GetTenderDeadline(); deadline != "" {
		t.TenderDeadline = s.parseDateToUnix(deadline)

		if deadlineTime, err := s.parseDate(deadline); err == nil {
			applicationDeadline := s.calculateApplicationDeadline(deadlineTime)
			t.ApplicationDeadline = applicationDeadline.Unix()
		}
	}

	if pubDate, err := s.parseDate(cn.GetPublicationDate()); err == nil {
		submissionDeadline := s.calculateSubmissionDeadline(pubDate)
		t.SubmissionDeadline = submissionDeadline.Unix()
	}

	// Set SubmissionURL from TenderingTerms
	if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil {
		if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
			if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
				if strings.HasPrefix(endpointID, "http") {
					t.SubmissionURL = endpointID
				} else {
					t.SubmissionURL = "https://" + endpointID
				}
			}
		}
	}

	// Set location information
	if cn.ProcurementProject != nil && cn.ProcurementProject.RealizedLocation != nil {
		if addr := cn.ProcurementProject.RealizedLocation.Address; addr != nil {
			t.CityName = addr.CityName
			t.PostalCode = addr.PostalZone
			t.RegionCode = addr.CountrySubentityCode
			if addr.Country != nil {
				t.CountryCode = addr.Country.IdentificationCode
			}
		}
	}

	// Set buyer organization
	buyerID := cn.GetBuyerID()
	if buyerID != "" {
		if buyerOrg := cn.GetOrganizationByID(buyerID); buyerOrg != nil {
			t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer")
		}
	}

	// Set review organization
	reviewID := cn.GetReviewOrganizationID()
	if reviewID != "" {
		if reviewOrg := cn.GetOrganizationByID(reviewID); reviewOrg != nil {
			t.ReviewOrganization = s.mapOrganization(reviewOrg, "review")
		}
	}

	// Set all organizations
	if orgs := cn.GetOrganizations(); orgs != nil {
		for _, org := range orgs.Organization {
			mappedOrg := s.mapOrganization(&org, "")
			if mappedOrg != nil {
				t.Organizations = append(t.Organizations, *mappedOrg)
			}
		}
	}

	// Set selection criteria
	if criteria := cn.GetSelectionCriteria(); criteria != nil {
		for _, criterion := range criteria {
			t.SelectionCriteria = append(t.SelectionCriteria, tender.SelectionCriterion{
				TypeCode:    criterion.CriterionTypeCode,
				Description: criterion.Description,
				LanguageID:  criterion.LanguageID,
			})
		}
	}

	// Set official languages
	if languages := cn.GetOfficialLanguages(); languages != nil {
		t.OfficialLanguages = languages
	}

	// Generate unique tender ID
	t.TenderID = s.generateTenderID(t)

	return t
}

// mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity
func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *tender.Tender {
	now := time.Now().Unix()

	t := &tender.Tender{
		ContractNoticeID:    can.ID,
		ContractFolderID:    can.ContractFolderID,
		NoticeTypeCode:      can.NoticeTypeCode,
		NoticeSubTypeCode:   can.GetNoticeSubType(),
		NoticeLanguageCode:  can.NoticeLanguageCode,
		IssueDate:           s.parseDateToUnix(can.IssueDate),
		IssueTime:           s.parseTimeToUnix(can.IssueTime),
		GazetteID:           can.GetOJSID(),
		Title:               can.GetProcurementTitle(),
		Description:         can.GetProcurementDescription(),
		ProcurementTypeCode: can.GetContractNature(),
		ProcedureCode:       can.GetProcedureCode(),
		MainClassification:  can.GetMainClassification(),
		Status:              tender.TenderStatusAwarded, // Award notices are for completed tenders
		Source:              tender.TenderSourceTEDScraper,
		SourceFileURL:       sourceFileURL,
		SourceFileName:      sourceFileName,
		ProcessingMetadata: tender.ProcessingMetadata{
			ScrapedAt:         now,
			ProcessedAt:       now,
			ProcessingVersion: "1.0",
		},
	}

	// Set publication information
	if pubInfo := can.GetPublicationInfo(); pubInfo != nil {
		t.NoticePublicationID = pubInfo.NoticePublicationID
		t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate)
		t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
	}

	// Set total award value
	if amount, currency := can.GetTotalAwardValue(); amount != "" {
		if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
			t.EstimatedValue = parsedAmount
		}
		t.Currency = currency
	}

	// Set location information from main procurement project
	if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil {
		if addr := can.ProcurementProject.RealizedLocation.Address; addr != nil {
			t.CityName = addr.CityName
			t.PostalCode = addr.PostalZone
			t.RegionCode = addr.CountrySubentityCode
			if addr.Country != nil {
				t.CountryCode = addr.Country.IdentificationCode
			}
		}
	}

	// Set buyer organization
	buyerID := can.GetBuyerID()
	if buyerID != "" {
		if buyerOrg := can.GetOrganizationByID(buyerID); buyerOrg != nil {
			t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer")
		}
	}

	// Set all organizations
	if orgs := can.GetOrganizations(); orgs != nil {
		for _, org := range orgs.Organization {
			mappedOrg := s.mapOrganization(&org, "")
			if mappedOrg != nil {
				t.Organizations = append(t.Organizations, *mappedOrg)
			}
		}
	}

	// Generate unique tender ID
	t.TenderID = s.generateTenderID(t)

	return t
}

// mapOrganization maps a TED Organization to tender Organization
func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender.Organization {
	if tedOrg == nil || tedOrg.Company == nil {
		return nil
	}

	company := tedOrg.Company
	org := &tender.Organization{
		Role: role,
	}

	// Set organization ID and name
	if company.PartyIdentification != nil {
		org.ID = company.PartyIdentification.ID
	}
	if company.PartyName != nil {
		org.Name = company.PartyName.Name
	}

	// Set company ID
	if company.PartyLegalEntity != nil {
		org.CompanyID = company.PartyLegalEntity.CompanyID
	}

	// Set website
	org.WebsiteURI = company.WebsiteURI

	// Set contact information
	if company.Contact != nil {
		org.ContactName = company.Contact.Name
		org.ContactTelephone = company.Contact.Telephone
		org.ContactEmail = company.Contact.ElectronicMail
		org.ContactFax = company.Contact.Telefax
	}

	// Set address
	if company.PostalAddress != nil {
		org.Address = tender.Address{
			StreetName:           company.PostalAddress.StreetName,
			CityName:             company.PostalAddress.CityName,
			PostalZone:           company.PostalAddress.PostalZone,
			CountrySubentityCode: company.PostalAddress.CountrySubentityCode,
			Department:           company.PostalAddress.Department,
		}
		if company.PostalAddress.Country != nil {
			org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode
		}
	}

	return org
}

// generateTenderID generates a unique tender ID using the format:
// {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
func (s *TEDScraper) generateTenderID(t *tender.Tender) string {
	var parts []string

	// Source (TED)
	parts = append(parts, "1")

	// TED code (Notice Publication ID, first 8 chars)
	tedCode := "00000000"
	if t.NoticePublicationID != "" {
		tedCode = s.padOrTruncate(t.NoticePublicationID, 8)
	}
	parts = append(parts, tedCode)

	// Buyer code (first 8 chars of buyer organization ID, or 8 zeros if not available)
	// buyerCode := "00000000"
	// if t.BuyerOrganization != nil && t.BuyerOrganization.ID != "" {
	// 	buyerCode = s.padOrTruncate(t.BuyerOrganization.ID, 8)
	// }
	// parts = append(parts, buyerCode)

	// Date (YYYYMMDD format from issue date)
	parts = append(parts, s.padOrTruncate(s.UnixToDateString(t.IssueDate), 4))

	// Location code (country code + region code, padded to 6 chars)
	locationCode := "EU"
	if t.CountryCode != "" {
		locationCode = t.CountryCode
	}
	parts = append(parts, locationCode)

	return strings.Join(parts, "")
}

// padOrTruncate pads a string with zeros or truncates it to the specified length
func (s *TEDScraper) padOrTruncate(str string, length int) string {
	// Remove any non-alphanumeric characters and convert to uppercase
	clean := strings.ToUpper(strings.ReplaceAll(str, "-", ""))
	clean = strings.ReplaceAll(clean, " ", "")

	if len(clean) >= length {
		return clean[:length]
	}

	// Pad with zeros
	return clean + strings.Repeat("0", length-len(clean))
}

// generateTenderURL generates the TED tender URL from NoticePublicationID
func (s *TEDScraper) generateTenderURL(noticePublicationID string) string {
	if noticePublicationID == "" {
		return ""
	}

	// Remove leading zeros and format for URL
	// Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025
	parts := strings.Split(noticePublicationID, "-")
	if len(parts) != 2 {
		return ""
	}

	// Convert to int and back to string to remove leading zeros
	if num, err := strconv.Atoi(parts[0]); err == nil {
		cleanID := fmt.Sprintf("%d-%s", num, parts[1])
		return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID)
	}

	return ""
}

// parseDate parses various date formats used in TED XML
func (s *TEDScraper) parseDate(dateStr string) (time.Time, error) {
	if dateStr == "" {
		return time.Time{}, fmt.Errorf("empty date string")
	}

	// Common TED date formats
	formats := []string{
		"2006-01-02+07:00",
		"2006-01-02-07:00",
		"2006-01-02T15:04:05+07:00",
		"2006-01-02T15:04:05-07:00",
		"2006-01-02T15:04:05Z",
		"2006-01-02Z",
		"2006-01-02",
		time.RFC3339,
		time.RFC3339Nano,
	}

	for _, format := range formats {
		if parsedTime, err := time.Parse(format, dateStr); err == nil {
			return parsedTime, nil
		}
	}

	return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr)
}

// parseDateToUnix converts string date to Unix milliseconds (int64)
func (s *TEDScraper) parseDateToUnix(dateStr string) int64 {
	if dateStr == "" {
		return 0
	}

	if parsedTime, err := s.parseDate(dateStr); err == nil {
		return parsedTime.Unix()
	}

	return 0
}

// parseTimeToUnix converts string time to Unix milliseconds (int64)
func (s *TEDScraper) parseTimeToUnix(timeStr string) int64 {
	if timeStr == "" {
		return 0
	}

	// If time string is just time without date, use today's date
	if !strings.Contains(timeStr, "T") && !strings.Contains(timeStr, " ") {
		// Assume it's just a time like "15:04:05"
		today := time.Now().Format("2006-01-02")
		timeStr = fmt.Sprintf("%sT%s", today, timeStr)
	}

	if parsedTime, err := s.parseDate(timeStr); err == nil {
		return parsedTime.Unix()
	}

	return 0
}

// UnixToDateString converts Unix milliseconds to date string for TenderID generation
func (s *TEDScraper) UnixToDateString(u int64) string {
	if u == 0 {
		return "00000000"
	}

	return time.Unix(u, 0).Format("20060102")
}

func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time {
	deadline := publicationDate
	workingHoursAdded := 0

	for workingHoursAdded < 48 {
		deadline = deadline.Add(1 * time.Hour)

		if deadline.Weekday() != time.Saturday && deadline.Weekday() != time.Sunday {
			workingHoursAdded++
		}
	}

	return deadline
}

func (s *TEDScraper) calculateApplicationDeadline(tenderDeadline time.Time) time.Time {
	applicationDeadline := tenderDeadline
	workingDaysSubtracted := 0

	for workingDaysSubtracted < 7 {
		applicationDeadline = applicationDeadline.AddDate(0, 0, -1)

		if applicationDeadline.Weekday() != time.Saturday && applicationDeadline.Weekday() != time.Sunday {
			workingDaysSubtracted++
		}
	}

	return applicationDeadline
}

func (s *TEDScraper) adjustToWorkingDay(targetDate time.Time, forward bool) time.Time {
	current := targetDate

	for current.Weekday() == time.Saturday || current.Weekday() == time.Sunday {
		if forward {
			current = current.AddDate(0, 0, 1)
		} else {
			current = current.AddDate(0, 0, -1)
		}
	}

	return current
}

// mapTEDStatusToTenderStatus maps TED notice status to tender status
func (s *TEDScraper) mapTEDStatusToTenderStatus(tedStatus NoticeStatus) tender.TenderStatus {
	switch tedStatus {
	case NoticeStatusDraft:
		return tender.TenderStatusDraft
	case NoticeStatusPublished:
		return tender.TenderStatusPublished
	case NoticeStatusCancelled:
		return tender.TenderStatusCancelled
	case NoticeStatusAwarded:
		return tender.TenderStatusAwarded
	case NoticeStatusClosed:
		return tender.TenderStatusClosed
	case NoticeStatusModified:
		return tender.TenderStatusModified
	case NoticeStatusSuspended:
		return tender.TenderStatusSuspended
	default:
		// Default to active for unknown statuses, but log a warning
		s.logger.Warn("Unknown TED notice status, defaulting to active", map[string]interface{}{
			"ted_status": string(tedStatus),
		})
		return tender.TenderStatusActive
	}
}

// applyStatusSpecificInformation applies status-specific information to the tender entity
func (s *TEDScraper) applyStatusSpecificInformation(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) {
	if tenderEntity == nil || parsedDoc == nil {
		return
	}

	// Map TED status to tender status
	tenderEntity.Status = s.mapTEDStatusToTenderStatus(parsedDoc.Status)

	// Get the document to access status-specific methods
	document := parsedDoc.GetDocument()
	if document == nil {
		return
	}

	// Apply status-specific information based on document type
	switch parsedDoc.Type {
	case DocumentTypeContractNotice:
		if cn, ok := document.(*ContractNotice); ok {
			s.applyContractNoticeStatusInfo(tenderEntity, cn, parsedDoc.Status)
		}
	case DocumentTypeContractAwardNotice:
		if can, ok := document.(*ContractAwardNotice); ok {
			s.applyContractAwardNoticeStatusInfo(tenderEntity, can, parsedDoc.Status)
		}
	default:
		// For other document types, we rely on the parsed status
		s.logger.Debug("Applied basic status information for document type", map[string]interface{}{
			"document_type": string(parsedDoc.Type),
			"status":        string(parsedDoc.Status),
		})
	}
}

// applyContractNoticeStatusInfo applies status-specific information from ContractNotice
func (s *TEDScraper) applyContractNoticeStatusInfo(tenderEntity *tender.Tender, cn *ContractNotice, status NoticeStatus) {
	switch status {
	case NoticeStatusCancelled:
		if cancellation := cn.GetCancellationReason(); cancellation != nil {
			tenderEntity.CancellationReason = cancellation.Description
			if cancellation.ReasonCode != "" {
				tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description)
			}
			// Note: TED XML doesn't typically have cancellation date, so we use current time
			tenderEntity.CancellationDate = time.Now().Unix()
		}

	case NoticeStatusAwarded:
		if award := cn.GetAwardInformation(); award != nil {
			if award.AwardDate != "" {
				tenderEntity.AwardDate = s.parseDateToUnix(award.AwardDate)
			}
			if award.AwardedValue.Text != "" {
				if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil {
					tenderEntity.AwardedValue = value
				}
			}
			tenderEntity.ContractNumber = award.ContractNumber
			if award.WinningTenderer != nil && award.WinningTenderer.Company != nil {
				tenderEntity.WinningTenderer = s.mapOrganization(&Organization{
					Company: award.WinningTenderer.Company,
				}, "winner")
			}
		}

	case NoticeStatusSuspended:
		if suspension := cn.GetSuspensionReason(); suspension != nil {
			tenderEntity.SuspensionReason = suspension.Description
			if suspension.ReasonCode != "" {
				tenderEntity.SuspensionReason = fmt.Sprintf("%s: %s", suspension.ReasonCode, suspension.Description)
			}
			tenderEntity.SuspensionDate = time.Now().Unix()
		}

	case NoticeStatusModified:
		if modifications := cn.GetModifications(); modifications != nil {
			for _, mod := range modifications {
				tenderMod := tender.TenderModification{
					ModificationDate:   s.parseDateToUnix(mod.ModificationDate),
					ModificationReason: mod.ModificationReason,
					Description:        mod.Description,
					LanguageID:         mod.LanguageID,
				}
				tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod)
			}
		}
	}
}

// applyContractAwardNoticeStatusInfo applies status-specific information from ContractAwardNotice
func (s *TEDScraper) applyContractAwardNoticeStatusInfo(tenderEntity *tender.Tender, can *ContractAwardNotice, status NoticeStatus) {
	// Contract award notices are typically awarded status
	tenderEntity.Status = tender.TenderStatusAwarded

	// Set award date from tender result
	if can.TenderResult != nil && can.TenderResult.AwardDate != "" {
		tenderEntity.AwardDate = s.parseDateToUnix(can.TenderResult.AwardDate)
	}

	// Get award information from extensions if available
	if award := can.GetAwardInformation(); award != nil {
		if award.AwardedValue.Text != "" {
			if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil {
				tenderEntity.AwardedValue = value
			}
		}
		tenderEntity.ContractNumber = award.ContractNumber
	}

	// Get total award value from notice result
	if amount, currency := can.GetTotalAwardValue(); amount != "" {
		if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
			tenderEntity.AwardedValue = parsedAmount
			tenderEntity.Currency = currency
		}
	}

	// Extract detailed awarded contractors information
	s.extractAwardedEntities(tenderEntity, can)

	// Handle other statuses if applicable
	switch status {
	case NoticeStatusCancelled:
		if cancellation := can.GetCancellationReason(); cancellation != nil {
			tenderEntity.Status = tender.TenderStatusCancelled
			tenderEntity.CancellationReason = cancellation.Description
			if cancellation.ReasonCode != "" {
				tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description)
			}
			tenderEntity.CancellationDate = time.Now().Unix()
		}

	case NoticeStatusModified:
		if modifications := can.GetModifications(); modifications != nil {
			tenderEntity.Status = tender.TenderStatusModified
			for _, mod := range modifications {
				tenderMod := tender.TenderModification{
					ModificationDate:   s.parseDateToUnix(mod.ModificationDate),
					ModificationReason: mod.ModificationReason,
					Description:        mod.Description,
					LanguageID:         mod.LanguageID,
				}
				tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod)
			}
		}
	}
}

// extractAwardedEntities extracts detailed information about awarded entities from a contract award notice
func (s *TEDScraper) extractAwardedEntities(tenderEntity *tender.Tender, can *ContractAwardNotice) {
	awardedContractors := can.GetAwardedContractors()
	if len(awardedContractors) == 0 {
		return
	}

	s.logger.Info("Extracting awarded entities", map[string]interface{}{
		"contract_notice_id": can.ID,
		"num_contractors":    len(awardedContractors),
	})

	for _, contractor := range awardedContractors {
		entity := tender.Awarded{
			Name:           contractor.Name,
			Address:        contractor.Address,
			Country:        contractor.Country,
			Currency:       contractor.Currency,
			ContractID:     contractor.ContractID,
			TenderID:       contractor.TenderID,
			LotID:          contractor.LotID,
			CompanyID:      contractor.CompanyID,
			OrganizationID: contractor.OrganizationID,
		}

		// Parse amount
		if contractor.Amount != "" {
			if amount, err := strconv.ParseFloat(contractor.Amount, 64); err == nil {
				entity.Amount = amount
			} else {
				s.logger.Warn("Failed to parse award amount", map[string]interface{}{
					"amount_string": contractor.Amount,
					"contractor":    contractor.Name,
					"error":         err.Error(),
				})
			}
		}

		// Parse award date
		if contractor.AwardDate != "" {
			entity.AwardDate = s.parseDateToUnix(contractor.AwardDate)
		} else if contractor.IssueDate != "" {
			entity.AwardDate = s.parseDateToUnix(contractor.IssueDate)
		}

		// Calculate share percentage if there are multiple contractors
		totalAmount := tenderEntity.AwardedValue
		if totalAmount > 0 && entity.Amount > 0 {
			entity.Share = (entity.Amount / totalAmount) * 100
		}

		tenderEntity.AwardedEntities = append(tenderEntity.AwardedEntities, entity)

		s.logger.Debug("Added awarded entity", map[string]interface{}{
			"name":        entity.Name,
			"amount":      entity.Amount,
			"currency":    entity.Currency,
			"share":       entity.Share,
			"contract_id": entity.ContractID,
		})
	}

	s.logger.Info("Successfully extracted awarded entities", map[string]interface{}{
		"total_entities": len(tenderEntity.AwardedEntities),
		"total_value":    tenderEntity.AwardedValue,
		"currency":       tenderEntity.Currency,
	})
}

// ScrapeByDateRange scrapes TED XML files within a specified date range
func (s *TEDScraper) ScrapeByDateRange(ctx context.Context, config DateRangeConfig) error {
	startTime := time.Now()

	s.logger.Info("Starting TED XML scraping by date range", map[string]interface{}{
		"start_date": config.StartDate.Format("2006-01-02"),
		"end_date":   config.EndDate.Format("2006-01-02"),
		"mode":       string(config.Mode),
		"timestamp":  startTime.Unix(),
	})

	// Validate date range
	if config.EndDate.Before(config.StartDate) {
		return fmt.Errorf("end date cannot be before start date")
	}

	// Calculate the date range in days
	daysInRange := int(config.EndDate.Sub(config.StartDate).Hours()/24) + 1
	if daysInRange > 365 {
		return fmt.Errorf("date range too large: maximum 365 days allowed")
	}

	// Create scraping job
	job := &ScrapingJob{
		Type:      fmt.Sprintf("ted_%s", config.Mode),
		Status:    "running",
		StartedAt: startTime.Unix(),
		Progress:  JobProgress{},
		Results:   JobResults{},
		Config: map[string]interface{}{
			"base_url":   s.config.BaseURL,
			"start_date": config.StartDate.Format("2006-01-02"),
			"end_date":   config.EndDate.Format("2006-01-02"),
			"mode":       string(config.Mode),
		},
	}

	// Save job
	if err := s.saveScrapingJob(ctx, job); err != nil {
		s.logger.Error("Failed to save scraping job", map[string]interface{}{
			"error": err.Error(),
		})
	}

	// Calculate files to download based on date range
	filesToDownload := s.calculateFilesInDateRange(config.StartDate, config.EndDate)
	if len(filesToDownload) == 0 {
		s.logger.Info("No files to download in date range", map[string]interface{}{
			"start_date": config.StartDate.Format("2006-01-02"),
			"end_date":   config.EndDate.Format("2006-01-02"),
		})

		// Complete the job
		s.completeJob(ctx, job, nil, nil)
		return nil
	}

	// Update job progress
	job.Progress.TotalFiles = len(filesToDownload)
	s.saveScrapingJob(ctx, job)

	var lastSuccessfulFile *FileInfo

	// Download and process files
	for i, fileInfo := range filesToDownload {
		job.Progress.CurrentFile = fmt.Sprintf("%d%05d", fileInfo.Year, fileInfo.Number)
		s.saveScrapingJob(ctx, job)

		fileResult, err := s.downloadAndProcess(ctx, fileInfo)
		if err != nil {
			s.logger.Error("Failed to download and process file", map[string]interface{}{
				"year":   fileInfo.Year,
				"number": fileInfo.Number,
				"error":  err.Error(),
			})
			job.Progress.FailedFiles++
			job.Results.ErrorCount++
		} else {
			job.Progress.ProcessedFiles++
			job.Results.TotalTenders += fileResult.ProcessedCount
			job.Results.NewTenders += fileResult.NewTenders
			job.Results.UpdatedTenders += fileResult.UpdatedTenders
			if len(fileResult.Errors) > 0 {
				job.Results.ErrorCount += len(fileResult.Errors)
			}
			lastSuccessfulFile = &fileInfo
		}

		// Update progress percentage
		job.Progress.PercentComplete = float64(i+1) / float64(len(filesToDownload)) * 100
		s.saveScrapingJob(ctx, job)

		// Check context for cancellation
		select {
		case <-ctx.Done():
			s.completeJob(ctx, job, nil, lastSuccessfulFile)
			return ctx.Err()
		default:
			// Continue processing
		}
	}

	// Complete the job
	s.completeJob(ctx, job, nil, lastSuccessfulFile)

	s.logger.Info("TED XML scraping by date range completed", map[string]interface{}{
		"files_processed": job.Progress.ProcessedFiles,
		"tenders_created": job.Results.NewTenders,
		"tenders_updated": job.Results.UpdatedTenders,
		"error_count":     job.Results.ErrorCount,
		"success_rate":    job.Results.SuccessRate,
	})

	return nil
}

// calculateFilesInDateRange calculates which files need to be downloaded for a date range
func (s *TEDScraper) calculateFilesInDateRange(startDate, endDate time.Time) []FileInfo {
	var files []FileInfo

	// TED files are published roughly once per working day
	// We estimate file numbers based on working days from start of year

	for year := startDate.Year(); year <= endDate.Year(); year++ {
		yearStart := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC)
		yearEnd := time.Date(year, 12, 31, 23, 59, 59, 0, time.UTC)

		// Determine the effective start and end dates for this year
		effectiveStart := startDate
		if yearStart.After(startDate) {
			effectiveStart = yearStart
		}

		effectiveEnd := endDate
		if yearEnd.Before(endDate) {
			effectiveEnd = yearEnd
		}

		// Skip if no overlap with this year
		if effectiveStart.After(effectiveEnd) {
			continue
		}

		// Calculate approximate file range for this year
		startDayOfYear := effectiveStart.YearDay()
		endDayOfYear := effectiveEnd.YearDay()

		// Estimate file numbers (roughly 0.7 files per day, accounting for weekends)
		startFileNum := int(float64(startDayOfYear) * 0.7)
		endFileNum := int(float64(endDayOfYear) * 0.7)

		// Ensure minimum range
		if startFileNum < 1 {
			startFileNum = 1
		}
		if endFileNum < startFileNum {
			endFileNum = startFileNum
		}

		// Add some buffer
		startFileNum = max(1, startFileNum-2)
		endFileNum = min(366, endFileNum+2) // Max ~366 files per year

		s.logger.Info("Calculating file range for year", map[string]interface{}{
			"year":           year,
			"start_file_num": startFileNum,
			"end_file_num":   endFileNum,
			"start_date":     effectiveStart.Format("2006-01-02"),
			"end_date":       effectiveEnd.Format("2006-01-02"),
		})

		for number := startFileNum; number <= endFileNum; number++ {
			fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number)
			ojsID := fmt.Sprintf("S %03d / %d", number, year)

			files = append(files, FileInfo{
				Year:   year,
				Number: number,
				URL:    fileURL,
				OJS:    ojsID,
			})
		}
	}

	s.logger.Info("Calculated files for date range", map[string]interface{}{
		"total_files": len(files),
		"start_date":  startDate.Format("2006-01-02"),
		"end_date":    endDate.Format("2006-01-02"),
	})

	return files
}

// Helper function for max
func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// Helper function for min
func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// applyAwardLogic applies award-specific logic based on document type and content
func (s *TEDScraper) applyAwardLogic(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) {
	if tenderEntity == nil || parsedDoc == nil {
		return
	}

	// Only set award information if the tender has a confirmed winner
	hasConfirmedWinner := s.hasConfirmedWinner(parsedDoc)

	s.logger.Debug("Applying award logic", map[string]interface{}{
		"document_type":        string(parsedDoc.Type),
		"status":               string(parsedDoc.Status),
		"has_confirmed_winner": hasConfirmedWinner,
	})

	if !hasConfirmedWinner {
		// Clear any award date and winner information for tenders without confirmed winners
		tenderEntity.AwardDate = 0
		tenderEntity.AwardedValue = 0
		tenderEntity.WinningTenderer = nil
		tenderEntity.AwardedEntities = nil
		tenderEntity.ContractNumber = ""

		s.logger.Debug("Cleared award information for tender without confirmed winner", map[string]interface{}{
			"contract_notice_id": tenderEntity.ContractNoticeID,
		})
		return
	}

	// Process award information for tenders with confirmed winners
	s.logger.Info("Processing award information for tender with confirmed winner", map[string]interface{}{
		"contract_notice_id": tenderEntity.ContractNoticeID,
		"document_type":      string(parsedDoc.Type),
	})
}

// hasConfirmedWinner checks if the tender has a confirmed winner based on the document content
func (s *TEDScraper) hasConfirmedWinner(parsedDoc *ParsedDocument) bool {
	if parsedDoc == nil {
		return false
	}

	// Check for award-specific document types
	switch parsedDoc.Type {
	case DocumentTypeContractAwardNotice, DocumentTypeResultNotice:
		// These document types typically indicate awarded tenders
		if parsedDoc.Status == NoticeStatusAwarded {
			return true
		}

		// Check for specific award indicators in the document
		document := parsedDoc.GetDocument()
		if document != nil {
			return s.checkForAwardIndicators(document, parsedDoc.Type)
		}

	case DocumentTypeContractNotice:
		// Contract notices may have award information if they're updated with results
		if parsedDoc.Status == NoticeStatusAwarded {
			if cn := parsedDoc.ContractNotice; cn != nil {
				return s.checkForAwardIndicators(cn, parsedDoc.Type)
			}
		}
	}

	return false
}

// checkForAwardIndicators checks for specific award indicators in the document
func (s *TEDScraper) checkForAwardIndicators(document TEDDocument, docType DocumentType) bool {
	switch docType {
	case DocumentTypeContractAwardNotice:
		if can, ok := document.(*ContractAwardNotice); ok {
			// Check for tender result with award date
			if can.TenderResult != nil && can.TenderResult.AwardDate != "" {
				return true
			}

			// Check for awarded contractors
			awardedContractors := can.GetAwardedContractors()
			if len(awardedContractors) > 0 {
				return true
			}

			// Check for award information in extensions
			if award := can.GetAwardInformation(); award != nil && award.AwardDate != "" {
				return true
			}
		}

	case DocumentTypeContractNotice:
		if cn, ok := document.(*ContractNotice); ok {
			// Check for award information
			if award := cn.GetAwardInformation(); award != nil && award.AwardDate != "" {
				return true
			}
		}

	case DocumentTypeResultNotice:
		// Result notices typically contain award information
		return true
	}

	return false
}

// mergeExistingTenderData merges new data with existing tender data, preserving important existing information
func (s *TEDScraper) mergeExistingTenderData(existing *tender.Tender, updated *tender.Tender) {
	// Preserve creation timestamp and original metadata
	updated.CreatedAt = existing.CreatedAt

	// Merge processing metadata
	if existing.ProcessingMetadata.ScrapedAt != 0 {
		// Keep original scraped time, update processed time
		updated.ProcessingMetadata.ScrapedAt = existing.ProcessingMetadata.ScrapedAt
	}

	// Preserve existing award information if the new document doesn't have confirmed winners
	// but existing data does (to avoid losing award data on subsequent updates)
	if existing.AwardDate != 0 && updated.AwardDate == 0 && len(existing.AwardedEntities) > 0 {
		s.logger.Info("Preserving existing award information", map[string]interface{}{
			"contract_notice_id":     existing.ContractNoticeID,
			"existing_award_date":    existing.AwardDate,
			"awarded_entities_count": len(existing.AwardedEntities),
		})

		updated.AwardDate = existing.AwardDate
		updated.AwardedValue = existing.AwardedValue
		updated.WinningTenderer = existing.WinningTenderer
		updated.AwardedEntities = existing.AwardedEntities
		updated.ContractNumber = existing.ContractNumber
	}

	// Merge modifications (append new modifications to existing ones)
	if len(existing.Modifications) > 0 {
		// Create a map to avoid duplicate modifications
		modMap := make(map[string]tender.TenderModification)

		// Add existing modifications
		for _, mod := range existing.Modifications {
			key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason)
			modMap[key] = mod
		}

		// Add new modifications
		for _, mod := range updated.Modifications {
			key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason)
			modMap[key] = mod
		}

		// Convert back to slice
		updated.Modifications = make([]tender.TenderModification, 0, len(modMap))
		for _, mod := range modMap {
			updated.Modifications = append(updated.Modifications, mod)
		}
	}
}