tm_back/ted/scraper.go

package ted

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"tm/internal/tender"
	"tm/pkg/logger"
	"tm/pkg/mongo"

	"go.mongodb.org/mongo-driver/bson"
	"go.mongodb.org/mongo-driver/bson/primitive"
	mongoDriver "go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/options"
)

// TEDScraper handles downloading and parsing TED XML files
type TEDScraper struct {
	config       *Config
	logger       logger.Logger
	httpClient   *http.Client
	xmlParser    *TEDParser
	mongoManager *mongo.ConnectionManager
	tenderRepo   tender.TenderRepository
}

// ScrapingJob represents a scraping job (simplified version)
type ScrapingJob struct {
	mongo.Model
	Type        string                 `bson:"type" json:"type"`
	Status      string                 `bson:"status" json:"status"`
	StartedAt   int64                  `bson:"started_at" json:"started_at"`
	CompletedAt *int64                 `bson:"completed_at,omitempty" json:"completed_at,omitempty"`
	Progress    JobProgress            `bson:"progress" json:"progress"`
	Results     JobResults             `bson:"results" json:"results"`
	Config      map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"`
}

// JobProgress represents the progress of a scraping job
type JobProgress struct {
	TotalFiles      int     `bson:"total_files" json:"total_files"`
	ProcessedFiles  int     `bson:"processed_files" json:"processed_files"`
	FailedFiles     int     `bson:"failed_files" json:"failed_files"`
	PercentComplete float64 `bson:"percent_complete" json:"percent_complete"`
	CurrentFile     string  `bson:"current_file,omitempty" json:"current_file,omitempty"`
}

// JobResults represents the results of a scraping job
type JobResults struct {
	TotalTenders   int     `bson:"total_tenders" json:"total_tenders"`
	NewTenders     int     `bson:"new_tenders" json:"new_tenders"`
	UpdatedTenders int     `bson:"updated_tenders" json:"updated_tenders"`
	ErrorCount     int     `bson:"error_count" json:"error_count"`
	SuccessRate    float64 `bson:"success_rate" json:"success_rate"`
	ProcessingTime int64   `bson:"processing_time" json:"processing_time"`
}

// ScrapingState represents the current state of scraping
type ScrapingState struct {
	mongo.Model
	LastProcessedYear   int                `bson:"last_processed_year" json:"last_processed_year"`
	LastProcessedNumber int                `bson:"last_processed_number" json:"last_processed_number"`
	LastRunAt           int64              `bson:"last_run_at" json:"last_run_at"`
	NextRunAt           int64              `bson:"next_run_at" json:"next_run_at"`
	IsRunning           bool               `bson:"is_running" json:"is_running"`
	CurrentJobID        primitive.ObjectID `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
}

// ScraperConfig holds configuration for the TED scraper
type Config struct {
	BaseURL          string        `mapstructure:"base_url"`
	Timeout          time.Duration `mapstructure:"timeout"`
	MaxRetries       int           `mapstructure:"max_retries"`
	RetryDelay       time.Duration `mapstructure:"retry_delay"`
	UserAgent        string        `mapstructure:"user_agent"`
	MaxConcurrency   int           `mapstructure:"max_concurrency"`
	DownloadDir      string        `mapstructure:"download_dir"`
	CleanupAfter     time.Duration `mapstructure:"cleanup_after"`
	ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
}

// NewTEDScraper creates a new TED scraper instance
func NewTEDScraper(
	config *Config,
	logger logger.Logger,
	mongoManager *mongo.ConnectionManager,
	tenderRepo tender.TenderRepository,
) *TEDScraper {
	httpClient := &http.Client{
		Timeout: config.Timeout,
	}

	return &TEDScraper{
		config:       config,
		logger:       logger,
		httpClient:   httpClient,
		xmlParser:    NewTEDParser(),
		mongoManager: mongoManager,
		tenderRepo:   tenderRepo,
	}
}

// RunPeriodicScraping runs the scraper at configured intervals
func (s *TEDScraper) RunPeriodicScraping(ctx context.Context) error {
	ticker := time.NewTicker(s.config.ScrapingInterval)
	defer ticker.Stop()

	s.logger.Info("Starting periodic TED XML scraping", map[string]interface{}{
		"interval": s.config.ScrapingInterval.String(),
		"base_url": s.config.BaseURL,
	})

	// Run initial scraping
	if err := s.ScrapeLatest(ctx); err != nil {
		s.logger.Error("Initial scraping failed", map[string]interface{}{
			"error": err.Error(),
		})
	}

	for {
		select {
		case <-ctx.Done():
			s.logger.Info("Periodic scraping stopped", map[string]interface{}{
				"reason": ctx.Err().Error(),
			})
			return ctx.Err()
		case <-ticker.C:
			if err := s.ScrapeLatest(ctx); err != nil {
				s.logger.Error("Periodic scraping failed", map[string]interface{}{
					"error": err.Error(),
				})
			}
		}
	}
}

// ScrapeLatest scrapes the latest TED XML files
func (s *TEDScraper) ScrapeLatest(ctx context.Context) error {
	startTime := time.Now()

	s.logger.Info("Starting TED XML scraping", map[string]interface{}{
		"timestamp": startTime.Unix(),
	})

	// Get current state
	state, err := s.getScrapingState(ctx)
	if err != nil {
		s.logger.Error("Failed to get scraping state", map[string]interface{}{
			"error": err.Error(),
		})
		// Initialize with current date if no state exists
		state = &ScrapingState{
			LastProcessedYear:   time.Now().Year(),
			LastProcessedNumber: 0,
			LastRunAt:           startTime.Unix(),
			NextRunAt:           startTime.Add(s.config.ScrapingInterval).Unix(),
			IsRunning:           false,
		}
	}

	// Handle case where state is nil (no document found)
	if state == nil {
		s.logger.Info("No scraping state found, initializing new state", map[string]interface{}{})
		state = &ScrapingState{
			LastProcessedYear:   time.Now().Year(),
			LastProcessedNumber: 0,
			LastRunAt:           startTime.Unix(),
			NextRunAt:           startTime.Add(s.config.ScrapingInterval).Unix(),
			IsRunning:           false,
		}
	}

	if state.IsRunning {
		s.logger.Info("Scraping already running, skipping", map[string]interface{}{
			"current_job_id": state.CurrentJobID,
		})
		return nil
	}

	// Create scraping job
	job := &ScrapingJob{
		Type:      "ted_daily",
		Status:    "running",
		StartedAt: startTime.Unix(),
		Progress:  JobProgress{},
		Results:   JobResults{},
		Config: map[string]interface{}{
			"base_url":     s.config.BaseURL,
			"max_retries":  s.config.MaxRetries,
			"scraper_type": "ted_daily",
		},
	}

	// Save job
	if err := s.saveScrapingJob(ctx, job); err != nil {
		s.logger.Error("Failed to save scraping job", map[string]interface{}{
			"error": err.Error(),
		})
	}

	// Update state
	state.IsRunning = true
	state.CurrentJobID = job.ID
	state.UpdatedAt = time.Now().Unix()
	s.saveScrapingState(ctx, state)

	// Calculate files to download
	filesToDownload := s.calculateFilesToDownload(state)
	if len(filesToDownload) == 0 {
		s.logger.Info("No new files to download", map[string]interface{}{
			"last_processed": fmt.Sprintf("%d%05d", state.LastProcessedYear, state.LastProcessedNumber),
		})

		// Complete the job
		s.completeJob(ctx, job, state, nil)
		return nil
	}

	// Update job progress
	job.Progress.TotalFiles = len(filesToDownload)
	s.saveScrapingJob(ctx, job)

	var lastSuccessfulFile *FileInfo

	// Download and process files
	for i, fileInfo := range filesToDownload {
		job.Progress.CurrentFile = fmt.Sprintf("%d%05d", fileInfo.Year, fileInfo.Number)
		s.saveScrapingJob(ctx, job)

		fileResult, err := s.downloadAndProcess(ctx, fileInfo)
		if err != nil {
			s.logger.Error("Failed to download and process file", map[string]interface{}{
				"year":   fileInfo.Year,
				"number": fileInfo.Number,
				"error":  err.Error(),
			})
			job.Progress.FailedFiles++
			job.Results.ErrorCount++
		} else {
			job.Progress.ProcessedFiles++
			job.Results.TotalTenders += fileResult.ProcessedCount
			job.Results.NewTenders += fileResult.SuccessCount
			if len(fileResult.Errors) > 0 {
				job.Results.ErrorCount += len(fileResult.Errors)
			}
			lastSuccessfulFile = &fileInfo
		}

		// Update progress percentage
		job.Progress.PercentComplete = float64(i+1) / float64(len(filesToDownload)) * 100
		s.saveScrapingJob(ctx, job)

		// Check context for cancellation
		select {
		case <-ctx.Done():
			s.completeJob(ctx, job, state, lastSuccessfulFile)
			return ctx.Err()
		default:
			// Continue processing
		}
	}

	// Complete the job
	s.completeJob(ctx, job, state, lastSuccessfulFile)

	s.logger.Info("TED XML scraping completed", map[string]interface{}{
		"files_processed": job.Progress.ProcessedFiles,
		"tenders_created": job.Results.NewTenders,
		"error_count":     job.Results.ErrorCount,
		"success_rate":    job.Results.SuccessRate,
	})

	return nil
}

// FileInfo represents information about a TED file to download
type FileInfo struct {
	Year   int
	Number int
	URL    string
	OJS    string
}

// calculateFilesToDownload determines which files need to be downloaded
func (s *TEDScraper) calculateFilesToDownload(state *ScrapingState) []FileInfo {
	var files []FileInfo
	now := time.Now()
	currentYear := now.Year()
	currentMonth := now.Month()

	// Only process current year and month
	year := currentYear

	// Start from the last processed file if it's from current year, otherwise start from beginning of current month
	startNumber := 1
	if state.LastProcessedYear == currentYear {
		startNumber = state.LastProcessedNumber + 1
	} else {
		// Calculate approximate file number for start of current month
		// Assuming roughly 22 working days per month (weekdays only)
		startNumber = int(currentMonth-1)*22 + 1
	}

	// Calculate end number for current month
	// Estimate based on current day of month and weekdays
	daysInCurrentMonth := now.Day()
	// Rough estimate: multiply by 0.7 to account for weekdays only
	maxNumber := int(currentMonth-1)*22 + int(float64(daysInCurrentMonth)*0.7)

	// Add some buffer for potential files
	maxNumber += 5

	s.logger.Info("Calculating files to download for current month", map[string]interface{}{
		"current_year":   currentYear,
		"current_month":  int(currentMonth),
		"start_number":   startNumber,
		"max_number":     maxNumber,
		"last_processed": fmt.Sprintf("%d%05d", state.LastProcessedYear, state.LastProcessedNumber),
	})

	for number := startNumber; number <= maxNumber && len(files) < 10; number++ { // Limit to 10 files per run
		fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number)
		ojsID := fmt.Sprintf("S %03d / %d", number, year)

		files = append(files, FileInfo{
			Year:   year,
			Number: number,
			URL:    fileURL,
			OJS:    ojsID,
		})
	}

	return files
}

// FileProcessingResult represents the result of processing a single file
type FileProcessingResult struct {
	Year           int      `json:"year"`
	Number         int      `json:"number"`
	ProcessedCount int      `json:"processed_count"`
	SuccessCount   int      `json:"success_count"`
	Errors         []string `json:"errors"`
	ProcessedAt    int64    `json:"processed_at"`
}

// downloadAndProcess downloads and processes a single TED file
func (s *TEDScraper) downloadAndProcess(ctx context.Context, fileInfo FileInfo) (*FileProcessingResult, error) {
	url := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, fileInfo.Year, fileInfo.Number)

	s.logger.Info("Downloading TED file", map[string]interface{}{
		"url":    url,
		"year":   fileInfo.Year,
		"number": fileInfo.Number,
	})

	var result *FileProcessingResult
	var err error

	// Retry mechanism
	for attempt := 1; attempt <= s.config.MaxRetries; attempt++ {
		result, err = s.downloadAndProcessWithRetry(ctx, url, fileInfo)
		if err == nil {
			break
		}

		if attempt < s.config.MaxRetries {
			s.logger.Warn("Download attempt failed, retrying", map[string]interface{}{
				"attempt": attempt,
				"error":   err.Error(),
				"url":     url,
			})

			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			case <-time.After(s.config.RetryDelay):
				// Continue to next attempt
			}
		}
	}

	if err != nil {
		s.logger.Error("Failed to download and process file after retries", map[string]interface{}{
			"url":      url,
			"attempts": s.config.MaxRetries,
			"error":    err.Error(),
		})
		return nil, fmt.Errorf("failed to download %s after %d attempts: %w", url, s.config.MaxRetries, err)
	}

	return result, nil
}

// downloadAndProcessWithRetry performs a single download and process attempt
func (s *TEDScraper) downloadAndProcessWithRetry(ctx context.Context, url string, fileInfo FileInfo) (*FileProcessingResult, error) {
	// Create a longer timeout context for processing large files
	processCtx, cancel := context.WithTimeout(ctx, s.config.Timeout+time.Minute*5)
	defer cancel()

	// Create HTTP request
	req, err := http.NewRequestWithContext(processCtx, "GET", url, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request: %w", err)
	}

	req.Header.Set("User-Agent", s.config.UserAgent)

	s.logger.Info("Starting download", map[string]interface{}{
		"url":    url,
		"year":   fileInfo.Year,
		"number": fileInfo.Number,
	})

	// Make request
	resp, err := s.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to download file: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("HTTP error: %d %s", resp.StatusCode, resp.Status)
	}

	s.logger.Info("Download completed, starting processing", map[string]interface{}{
		"url":            url,
		"content_length": resp.Header.Get("Content-Length"),
	})

	// Process tar.gz file directly from response body
	return s.processTarGzFile(processCtx, resp.Body, fileInfo)
}

// processTarGzFile extracts and processes XML files from a tar.gz archive
func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fileInfo FileInfo) (*FileProcessingResult, error) {
	result := &FileProcessingResult{
		Year:           fileInfo.Year,
		Number:         fileInfo.Number,
		ProcessedCount: 0,
		SuccessCount:   0,
		Errors:         make([]string, 0),
		ProcessedAt:    time.Now().Unix(),
	}

	// Create gzip reader
	gzipReader, err := gzip.NewReader(reader)
	if err != nil {
		return nil, fmt.Errorf("failed to create gzip reader: %w", err)
	}
	defer gzipReader.Close()

	// Create tar reader
	tarReader := tar.NewReader(gzipReader)

	// Process each file in the tar archive
	fileCount := 0
	for {
		// Check context for cancellation before each tar entry
		select {
		case <-ctx.Done():
			s.logger.Warn("Context cancelled during tar processing", map[string]interface{}{
				"processed_files": fileCount,
				"error":           ctx.Err().Error(),
			})
			return result, ctx.Err()
		default:
		}

		header, err := tarReader.Next()
		if err == io.EOF {
			break // End of archive
		}
		if err != nil {
			s.logger.Warn("Error reading tar entry", map[string]interface{}{
				"error":           err.Error(),
				"processed_files": fileCount,
			})
			// If it's a timeout error, return it immediately
			if ctx.Err() != nil {
				return result, fmt.Errorf("timeout during tar processing: %w", err)
			}
			continue
		}
		fileCount++

		// Skip directories and non-XML files
		if header.Typeflag != tar.TypeReg {
			continue
		}

		fileName := filepath.Base(header.Name)
		if !strings.HasSuffix(strings.ToLower(fileName), ".xml") {
			continue
		}

		// Log progress every 10 files or for first file
		if result.ProcessedCount == 0 || result.ProcessedCount%10 == 0 {
			s.logger.Info("Processing XML file from tar.gz", map[string]interface{}{
				"file_name":        fileName,
				"size":             header.Size,
				"processed_count":  result.ProcessedCount,
				"total_files_seen": fileCount,
				"year":             fileInfo.Year,
				"number":           fileInfo.Number,
			})
		} else {
			s.logger.Debug("Processing XML file from tar.gz", map[string]interface{}{
				"file_name":       fileName,
				"processed_count": result.ProcessedCount,
			})
		}

		// Read file content
		content, err := io.ReadAll(tarReader)
		if err != nil {
			errMsg := fmt.Sprintf("failed to read XML file %s: %v", fileName, err)
			result.Errors = append(result.Errors, errMsg)
			continue
		}

		// Parse and process XML content
		if err := s.processXMLContent(ctx, content, fileName, fileInfo, result); err != nil {
			errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err)
			result.Errors = append(result.Errors, errMsg)
		}

		result.ProcessedCount++

		// Check context for cancellation
		select {
		case <-ctx.Done():
			return result, ctx.Err()
		default:
			// Continue processing
		}
	}

	s.logger.Info("Completed processing tar.gz file", map[string]interface{}{
		"year":            fileInfo.Year,
		"number":          fileInfo.Number,
		"total_files":     fileCount,
		"processed_count": result.ProcessedCount,
		"success_count":   result.SuccessCount,
		"error_count":     len(result.Errors),
		"processing_time": time.Now().Unix() - result.ProcessedAt,
	})

	return result, nil
}

// processXMLContent parses XML content and creates/updates tender
func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo, result *FileProcessingResult) error {
	s.logger.Info("Starting XML processing", map[string]interface{}{
		"file_name": fileName,
		"file_size": len(content),
	})

	// Parse XML using the TED parser
	parsedDoc, err := s.xmlParser.ParseXML(content)
	if err != nil {
		s.logger.Error("XML parsing failed", map[string]interface{}{
			"file_name": fileName,
			"error":     err.Error(),
		})
		return fmt.Errorf("failed to parse XML: %w", err)
	}

	// Get the underlying document for ID and validation
	document := parsedDoc.GetDocument()
	documentID := ""
	noticeTypeCode := ""
	if document != nil {
		documentID = document.GetID()
		noticeTypeCode = document.GetNoticeType()
	}

	s.logger.Info("XML parsed successfully", map[string]interface{}{
		"file_name":        fileName,
		"document_type":    string(parsedDoc.Type),
		"document_id":      documentID,
		"notice_type_code": noticeTypeCode,
	})

	// Validate parsed data
	var validationErr error
	if document != nil {
		validationErr = document.Validate()
	}
	if validationErr != nil {
		s.logger.Warn("XML validation failed, continuing anyway", map[string]interface{}{
			"file_name":     fileName,
			"document_type": string(parsedDoc.Type),
			"error":         validationErr.Error(),
		})
		// Don't return error, continue processing
	} else {
		s.logger.Info("XML validation passed", map[string]interface{}{
			"file_name":     fileName,
			"document_type": string(parsedDoc.Type),
		})
	}

	// Map to tender entity
	sourceURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, fileInfo.Year, fileInfo.Number)
	tenderEntity := s.mapToTenderFromParsedDoc(parsedDoc, sourceURL, fileName)

	s.logger.Info("Mapped to tender entity", map[string]interface{}{
		"file_name":          fileName,
		"tender_id":          tenderEntity.TenderID,
		"contract_notice_id": tenderEntity.ContractNoticeID,
		"title":              tenderEntity.Title,
	})

	// Try to find existing tender by contract notice ID
	s.logger.Info("Checking for existing tender", map[string]interface{}{
		"contract_notice_id": tenderEntity.ContractNoticeID,
	})

	existingTender, err := s.findTenderByContractNoticeID(ctx, tenderEntity.ContractNoticeID)
	if err != nil {
		s.logger.Warn("Error checking for existing tender", map[string]interface{}{
			"contract_notice_id": tenderEntity.ContractNoticeID,
			"error":              err.Error(),
		})
	}

	if err == nil && existingTender != nil {
		// Update existing tender
		s.logger.Info("Found existing tender, updating", map[string]interface{}{
			"existing_tender_id": existingTender.ID,
			"contract_notice_id": tenderEntity.ContractNoticeID,
		})

		tenderEntity.ID = existingTender.ID
		tenderEntity.CreatedAt = existingTender.CreatedAt
		tenderEntity.UpdatedAt = time.Now().Unix()

		err = s.tenderRepo.Update(ctx, tenderEntity)
		if err != nil {
			s.logger.Error("Failed to update existing tender", map[string]interface{}{
				"tender_id": tenderEntity.ID,
				"error":     err.Error(),
			})
			return fmt.Errorf("failed to update existing tender: %w", err)
		}

		s.logger.Info("Successfully updated tender", map[string]interface{}{
			"tender_id": tenderEntity.ID,
		})
	} else {
		// Create new tender
		s.logger.Info("Creating new tender", map[string]interface{}{
			"contract_notice_id": tenderEntity.ContractNoticeID,
			"tender_id":          tenderEntity.TenderID,
		})

		tenderEntity.CreatedAt = time.Now().Unix()
		tenderEntity.UpdatedAt = tenderEntity.CreatedAt

		err = s.tenderRepo.Create(ctx, tenderEntity)
		if err != nil {
			s.logger.Error("Failed to create new tender", map[string]interface{}{
				"contract_notice_id": tenderEntity.ContractNoticeID,
				"tender_id":          tenderEntity.TenderID,
				"error":              err.Error(),
			})
			return fmt.Errorf("failed to create tender: %w", err)
		}

		s.logger.Info("Successfully created new tender", map[string]interface{}{
			"tender_id":          tenderEntity.TenderID,
			"contract_notice_id": tenderEntity.ContractNoticeID,
		})
	}

	result.SuccessCount++

	s.logger.Debug("Successfully processed XML file", map[string]interface{}{
		"file_name":     fileName,
		"document_type": string(parsedDoc.Type),
		"document_id":   documentID,
		"tender_id":     tenderEntity.TenderID,
	})

	return nil
}

// Helper methods for database operations
func (s *TEDScraper) getScrapingState(ctx context.Context) (*ScrapingState, error) {
	collection := s.mongoManager.GetCollection("scraping_state")
	var state ScrapingState
	err := collection.FindOne(ctx, bson.M{"_id": "ted_scraper"}).Decode(&state)
	if err != nil {
		if err == mongoDriver.ErrNoDocuments {
			return nil, nil
		}
		return nil, err
	}
	return &state, nil
}

func (s *TEDScraper) saveScrapingState(ctx context.Context, state *ScrapingState) error {
	collection := s.mongoManager.GetCollection("scraping_state")
	state.UpdatedAt = time.Now().Unix()
	opts := options.Replace().SetUpsert(true)
	_, err := collection.ReplaceOne(ctx, bson.M{"_id": state.ID}, state, opts)
	return err
}

func (s *TEDScraper) saveScrapingJob(ctx context.Context, job *ScrapingJob) error {
	collection := s.mongoManager.GetCollection("scraping_jobs")
	job.UpdatedAt = time.Now().Unix()
	opts := options.Replace().SetUpsert(true)
	_, err := collection.ReplaceOne(ctx, bson.M{"_id": job.ID}, job, opts)
	return err
}

func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*tender.Tender, error) {
	// Use repository's specific method for finding by contract notice ID
	existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID)
	if err != nil {
		// If document not found, return nil without error
		if err.Error() == "document not found" {
			return nil, nil
		}
		return nil, err
	}

	return existingTender, nil
}

func (s *TEDScraper) completeJob(ctx context.Context, job *ScrapingJob, state *ScrapingState, lastFile *FileInfo) {
	// Update state with last successful file
	if lastFile != nil {
		state.LastProcessedYear = lastFile.Year
		state.LastProcessedNumber = lastFile.Number
	}

	// Complete the job
	endTime := time.Now()
	completedAt := endTime.Unix()
	job.Status = "completed"
	job.CompletedAt = &completedAt
	job.Results.ProcessingTime = completedAt - job.StartedAt
	if job.Results.TotalTenders > 0 {
		job.Results.SuccessRate = float64(job.Results.NewTenders) / float64(job.Results.TotalTenders)
	}
	s.saveScrapingJob(ctx, job)

	// Update state
	state.IsRunning = false
	state.CurrentJobID = primitive.NilObjectID
	state.LastRunAt = completedAt
	state.NextRunAt = endTime.Add(s.config.ScrapingInterval).Unix()
	state.UpdatedAt = completedAt
	s.saveScrapingState(ctx, state)
}

// mapToTenderFromParsedDoc maps a parsed TED document to tender entity
func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *tender.Tender {
	switch parsedDoc.Type {
	case DocumentTypeContractNotice:
		if parsedDoc.ContractNotice == nil {
			s.logger.Error("ContractNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapToTender(parsedDoc.ContractNotice, sourceFileURL, sourceFileName)

	case DocumentTypeContractAwardNotice:
		if parsedDoc.ContractAwardNotice == nil {
			s.logger.Error("ContractAwardNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapContractAwardNoticeToTender(parsedDoc.ContractAwardNotice, sourceFileURL, sourceFileName)

	case DocumentTypePriorInformationNotice:
		if parsedDoc.PriorInformationNotice == nil {
			s.logger.Error("PriorInformationNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.PriorInformationNotice, sourceFileURL, sourceFileName)

	case DocumentTypeDesignContest:
		if parsedDoc.DesignContest == nil {
			s.logger.Error("DesignContest is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.DesignContest, sourceFileURL, sourceFileName)

	case DocumentTypeQualificationSystemNotice:
		if parsedDoc.QualificationSystemNotice == nil {
			s.logger.Error("QualificationSystemNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.QualificationSystemNotice, sourceFileURL, sourceFileName)

	case DocumentTypeConcessionNotice:
		if parsedDoc.ConcessionNotice == nil {
			s.logger.Error("ConcessionNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.ConcessionNotice, sourceFileURL, sourceFileName)

	case DocumentTypePlanningNotice:
		if parsedDoc.PlanningNotice == nil {
			s.logger.Error("PlanningNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.PlanningNotice, sourceFileURL, sourceFileName)

	case DocumentTypeCompetitionNotice:
		if parsedDoc.CompetitionNotice == nil {
			s.logger.Error("CompetitionNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.CompetitionNotice, sourceFileURL, sourceFileName)

	case DocumentTypeResultNotice:
		if parsedDoc.ResultNotice == nil {
			s.logger.Error("ResultNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.ResultNotice, sourceFileURL, sourceFileName)

	case DocumentTypeChangeNotice:
		if parsedDoc.ChangeNotice == nil {
			s.logger.Error("ChangeNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.ChangeNotice, sourceFileURL, sourceFileName)

	case DocumentTypeSubcontractNotice:
		if parsedDoc.SubcontractNotice == nil {
			s.logger.Error("SubcontractNotice is nil", map[string]interface{}{
				"document_type": string(parsedDoc.Type),
			})
			return nil
		}
		return s.mapGenericNoticeToTender(parsedDoc.SubcontractNotice, sourceFileURL, sourceFileName)

	// Legacy document types that should be parsed as ContractNotice but may still be in the old flow
	case DocumentTypeCorrigendum, DocumentTypeModificationNotice:
		if parsedDoc.ContractNotice != nil {
			return s.mapToTender(parsedDoc.ContractNotice, sourceFileURL, sourceFileName)
		}
		// Fallback: try to get the document from the generic interface
		document := parsedDoc.GetDocument()
		if document != nil {
			return s.mapGenericNoticeToTender(document, sourceFileURL, sourceFileName)
		}
		s.logger.Error("No valid document found for legacy type", map[string]interface{}{
			"document_type": string(parsedDoc.Type),
		})
		return nil

	default:
		s.logger.Error("Unknown document type for mapping", map[string]interface{}{
			"document_type": string(parsedDoc.Type),
		})
		return nil
	}
}

// mapGenericNoticeToTender maps any TED document type to tender entity using the common interface
func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *tender.Tender {
	if doc == nil {
		s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{})
		return nil
	}

	now := time.Now().Unix()

	t := &tender.Tender{
		ContractNoticeID:    doc.GetID(),
		ContractFolderID:    "", // Not available in generic interface
		NoticeTypeCode:      doc.GetNoticeType(),
		NoticeSubTypeCode:   "", // Not available in generic interface
		NoticeLanguageCode:  doc.GetLanguage(),
		IssueDate:           now, // TODO: Parse actual date from document
		IssueTime:           now, // TODO: Parse actual time from document
		GazetteID:           "",  // Not available in generic interface
		NoticePublicationID: "",  // Not available in generic interface
		PublicationDate:     now, // TODO: Parse actual publication date

		// Basic tender information
		Title:          "Generic Notice " + doc.GetNoticeType(),
		Description:    "Processed from " + doc.GetNoticeType() + " document",
		Status:         tender.TenderStatusActive,
		Source:         tender.TenderSourceTEDScraper,
		SourceFileURL:  sourceFileURL,
		SourceFileName: sourceFileName,
		ProcessingMetadata: tender.ProcessingMetadata{
			ScrapedAt:         now,
			ProcessedAt:       now,
			ProcessingVersion: "1.0",
		},
	}

	// Generate the tender ID after creating the tender
	t.TenderID = s.generateTenderID(t)

	return t
}

// mapToTender maps a TED ContractNotice to tender entity
func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *tender.Tender {
	now := time.Now().Unix()

	t := &tender.Tender{
		ContractNoticeID:    cn.ID,
		ContractFolderID:    cn.ContractFolderID,
		NoticeTypeCode:      cn.GetNoticeSubType(),
		NoticeSubTypeCode:   cn.GetNoticeSubType(),
		NoticeLanguageCode:  cn.NoticeLanguageCode,
		IssueDate:           s.parseDateToUnixMilli(cn.IssueDate),
		IssueTime:           s.parseTimeToUnixMilli(cn.IssueTime),
		GazetteID:           cn.GetOJSID(),
		Title:               cn.GetProcurementTitle(),
		Description:         cn.GetProcurementDescription(),
		ProcurementTypeCode: cn.GetContractNature(),
		ProcedureCode:       cn.GetProcedureCode(),
		MainClassification:  cn.GetMainClassification(),
		PlaceOfPerformance:  cn.GetPlaceOfPerformance(),
		DocumentURI:         cn.GetDocumentURI(),
		Status:              tender.TenderStatusActive,
		Source:              tender.TenderSourceTEDScraper,
		SourceFileURL:       sourceFileURL,
		SourceFileName:      sourceFileName,
		ProcessingMetadata: tender.ProcessingMetadata{
			ScrapedAt:         now,
			ProcessedAt:       now,
			ProcessingVersion: "1.0",
		},
	}

	// Set publication information
	if pubInfo := cn.GetPublicationInfo(); pubInfo != nil {
		t.NoticePublicationID = pubInfo.NoticePublicationID
		t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate)
		t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
	}

	// Set additional classifications
	if cn.ProcurementProject != nil && cn.ProcurementProject.AdditionalCommodityClassification != nil {
		for _, additionalClass := range cn.ProcurementProject.AdditionalCommodityClassification {
			if additionalClass.ItemClassificationCode != "" {
				t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode)
			}
		}
	}

	// Set estimated value and currency
	if amount, currency := cn.GetBudget(); amount != "" {
		if value, err := strconv.ParseFloat(amount, 64); err == nil {
			t.EstimatedValue = value
			t.Currency = currency
		}
	}

	// Set duration information
	if duration, unit := cn.GetDuration(); duration != "" {
		t.Duration = duration
		t.DurationUnit = unit
	}

	// Set tender deadline
	if deadline := cn.GetTenderDeadline(); deadline != "" {
		t.TenderDeadline = s.parseDateToUnixMilli(deadline)
		if deadlineTime, err := s.parseDate(deadline); err == nil {
			t.SubmissionDeadline = deadlineTime.UnixMilli()
		}
	}

	// Set location information
	if cn.ProcurementProject != nil && cn.ProcurementProject.RealizedLocation != nil {
		if addr := cn.ProcurementProject.RealizedLocation.Address; addr != nil {
			t.CityName = addr.CityName
			t.PostalCode = addr.PostalZone
			t.RegionCode = addr.CountrySubentityCode
			if addr.Country != nil {
				t.CountryCode = addr.Country.IdentificationCode
			}
		}
	}

	// Set buyer organization
	buyerID := cn.GetBuyerID()
	if buyerID != "" {
		if buyerOrg := cn.GetOrganizationByID(buyerID); buyerOrg != nil {
			t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer")
		}
	}

	// Set review organization
	reviewID := cn.GetReviewOrganizationID()
	if reviewID != "" {
		if reviewOrg := cn.GetOrganizationByID(reviewID); reviewOrg != nil {
			t.ReviewOrganization = s.mapOrganization(reviewOrg, "review")
		}
	}

	// Set all organizations
	if orgs := cn.GetOrganizations(); orgs != nil {
		for _, org := range orgs.Organization {
			mappedOrg := s.mapOrganization(&org, "")
			if mappedOrg != nil {
				t.Organizations = append(t.Organizations, *mappedOrg)
			}
		}
	}

	// Set selection criteria
	if criteria := cn.GetSelectionCriteria(); criteria != nil {
		for _, criterion := range criteria {
			t.SelectionCriteria = append(t.SelectionCriteria, tender.SelectionCriterion{
				TypeCode:    criterion.CriterionTypeCode,
				Description: criterion.Description,
				LanguageID:  criterion.LanguageID,
			})
		}
	}

	// Set official languages
	if languages := cn.GetOfficialLanguages(); languages != nil {
		t.OfficialLanguages = languages
	}

	// Generate unique tender ID
	t.TenderID = s.generateTenderID(t)

	return t
}

// mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity
func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *tender.Tender {
	now := time.Now().Unix()

	t := &tender.Tender{
		ContractNoticeID:    can.ID,
		ContractFolderID:    can.ContractFolderID,
		NoticeTypeCode:      can.NoticeTypeCode,
		NoticeSubTypeCode:   can.GetNoticeSubType(),
		NoticeLanguageCode:  can.NoticeLanguageCode,
		IssueDate:           s.parseDateToUnixMilli(can.IssueDate),
		IssueTime:           s.parseTimeToUnixMilli(can.IssueTime),
		GazetteID:           can.GetOJSID(),
		Title:               can.GetProcurementTitle(),
		Description:         can.GetProcurementDescription(),
		ProcurementTypeCode: can.GetContractNature(),
		ProcedureCode:       can.GetProcedureCode(),
		MainClassification:  can.GetMainClassification(),
		Status:              tender.TenderStatusAwarded, // Award notices are for completed tenders
		Source:              tender.TenderSourceTEDScraper,
		SourceFileURL:       sourceFileURL,
		SourceFileName:      sourceFileName,
		ProcessingMetadata: tender.ProcessingMetadata{
			ScrapedAt:         now,
			ProcessedAt:       now,
			ProcessingVersion: "1.0",
		},
	}

	// Set publication information
	if pubInfo := can.GetPublicationInfo(); pubInfo != nil {
		t.NoticePublicationID = pubInfo.NoticePublicationID
		t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate)
		t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
	}

	// Set total award value
	if amount, currency := can.GetTotalAwardValue(); amount != "" {
		if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
			t.EstimatedValue = parsedAmount
		}
		t.Currency = currency
	}

	// Set location information from main procurement project
	if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil {
		if addr := can.ProcurementProject.RealizedLocation.Address; addr != nil {
			t.CityName = addr.CityName
			t.PostalCode = addr.PostalZone
			t.RegionCode = addr.CountrySubentityCode
			if addr.Country != nil {
				t.CountryCode = addr.Country.IdentificationCode
			}
		}
	}

	// Set buyer organization
	buyerID := can.GetBuyerID()
	if buyerID != "" {
		if buyerOrg := can.GetOrganizationByID(buyerID); buyerOrg != nil {
			t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer")
		}
	}

	// Set all organizations
	if orgs := can.GetOrganizations(); orgs != nil {
		for _, org := range orgs.Organization {
			mappedOrg := s.mapOrganization(&org, "")
			if mappedOrg != nil {
				t.Organizations = append(t.Organizations, *mappedOrg)
			}
		}
	}

	// Generate unique tender ID
	t.TenderID = s.generateTenderID(t)

	return t
}

// mapOrganization maps a TED Organization to tender Organization
func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender.Organization {
	if tedOrg == nil || tedOrg.Company == nil {
		return nil
	}

	company := tedOrg.Company
	org := &tender.Organization{
		Role: role,
	}

	// Set organization ID and name
	if company.PartyIdentification != nil {
		org.ID = company.PartyIdentification.ID
	}
	if company.PartyName != nil {
		org.Name = company.PartyName.Name
	}

	// Set company ID
	if company.PartyLegalEntity != nil {
		org.CompanyID = company.PartyLegalEntity.CompanyID
	}

	// Set website
	org.WebsiteURI = company.WebsiteURI

	// Set contact information
	if company.Contact != nil {
		org.ContactName = company.Contact.Name
		org.ContactTelephone = company.Contact.Telephone
		org.ContactEmail = company.Contact.ElectronicMail
		org.ContactFax = company.Contact.Telefax
	}

	// Set address
	if company.PostalAddress != nil {
		org.Address = tender.Address{
			StreetName:           company.PostalAddress.StreetName,
			CityName:             company.PostalAddress.CityName,
			PostalZone:           company.PostalAddress.PostalZone,
			CountrySubentityCode: company.PostalAddress.CountrySubentityCode,
			Department:           company.PostalAddress.Department,
		}
		if company.PostalAddress.Country != nil {
			org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode
		}
	}

	return org
}

// generateTenderID generates a unique tender ID using the format:
// {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
func (s *TEDScraper) generateTenderID(t *tender.Tender) string {
	var parts []string

	// Source (TED)
	parts = append(parts, "TED")

	// Buyer code (first 8 chars of buyer organization ID, or 8 zeros if not available)
	buyerCode := "00000000"
	if t.BuyerOrganization != nil && t.BuyerOrganization.ID != "" {
		buyerCode = s.padOrTruncate(t.BuyerOrganization.ID, 8)
	}
	parts = append(parts, buyerCode)

	// TED code (Contract Notice ID, first 8 chars)
	tedCode := "00000000"
	if t.ContractNoticeID != "" {
		tedCode = s.padOrTruncate(t.ContractNoticeID, 8)
	}
	parts = append(parts, tedCode)

	// Date (YYYYMMDD format from issue date)
	dateCode := s.unixMilliToDateString(t.IssueDate)
	parts = append(parts, dateCode)

	// Location code (country code + region code, padded to 6 chars)
	locationCode := "000000"
	if t.CountryCode != "" {
		location := t.CountryCode
		if t.RegionCode != "" {
			location += t.RegionCode
		}
		locationCode = s.padOrTruncate(location, 6)
	}
	parts = append(parts, locationCode)

	return strings.Join(parts, "")
}

// padOrTruncate pads a string with zeros or truncates it to the specified length
func (s *TEDScraper) padOrTruncate(str string, length int) string {
	// Remove any non-alphanumeric characters and convert to uppercase
	clean := strings.ToUpper(strings.ReplaceAll(str, "-", ""))
	clean = strings.ReplaceAll(clean, " ", "")

	if len(clean) >= length {
		return clean[:length]
	}

	// Pad with zeros
	return clean + strings.Repeat("0", length-len(clean))
}

// generateTenderURL generates the TED tender URL from NoticePublicationID
func (s *TEDScraper) generateTenderURL(noticePublicationID string) string {
	if noticePublicationID == "" {
		return ""
	}

	// Remove leading zeros and format for URL
	// Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025
	parts := strings.Split(noticePublicationID, "-")
	if len(parts) != 2 {
		return ""
	}

	// Convert to int and back to string to remove leading zeros
	if num, err := strconv.Atoi(parts[0]); err == nil {
		cleanID := fmt.Sprintf("%d-%s", num, parts[1])
		return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID)
	}

	return ""
}

// parseDate parses various date formats used in TED XML
func (s *TEDScraper) parseDate(dateStr string) (time.Time, error) {
	if dateStr == "" {
		return time.Time{}, fmt.Errorf("empty date string")
	}

	// Common TED date formats
	formats := []string{
		"2006-01-02+07:00",
		"2006-01-02-07:00",
		"2006-01-02T15:04:05+07:00",
		"2006-01-02T15:04:05-07:00",
		"2006-01-02T15:04:05Z",
		"2006-01-02Z",
		"2006-01-02",
		time.RFC3339,
		time.RFC3339Nano,
	}

	for _, format := range formats {
		if parsedTime, err := time.Parse(format, dateStr); err == nil {
			return parsedTime, nil
		}
	}

	return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr)
}

// parseDateToUnixMilli converts string date to Unix milliseconds (int64)
func (s *TEDScraper) parseDateToUnixMilli(dateStr string) int64 {
	if dateStr == "" {
		return 0
	}

	if parsedTime, err := s.parseDate(dateStr); err == nil {
		return parsedTime.UnixMilli()
	}

	return 0
}

// parseTimeToUnixMilli converts string time to Unix milliseconds (int64)
func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 {
	if timeStr == "" {
		return 0
	}

	// If time string is just time without date, use today's date
	if !strings.Contains(timeStr, "T") && !strings.Contains(timeStr, " ") {
		// Assume it's just a time like "15:04:05"
		today := time.Now().Format("2006-01-02")
		timeStr = fmt.Sprintf("%sT%s", today, timeStr)
	}

	if parsedTime, err := s.parseDate(timeStr); err == nil {
		return parsedTime.UnixMilli()
	}

	return 0
}

// unixMilliToDateString converts Unix milliseconds to date string for TenderID generation
func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string {
	if unixMilli == 0 {
		return "00000000"
	}

	return time.UnixMilli(unixMilli).Format("20060102")
}