package ted import ( "archive/tar" "compress/gzip" "context" "fmt" "io" "net/http" "path/filepath" "strconv" "strings" "time" "tm/internal/tender" "tm/pkg/logger" "tm/pkg/mongo" "go.mongodb.org/mongo-driver/v2/bson" mongoDriver "go.mongodb.org/mongo-driver/v2/mongo" "go.mongodb.org/mongo-driver/v2/mongo/options" ) // TEDScraper handles downloading and parsing TED XML files type TEDScraper struct { config *Config logger logger.Logger httpClient *http.Client xmlParser *TEDParser mongoManager *mongo.ConnectionManager tenderRepo tender.TenderRepository } // ScrapingJob represents a scraping job (simplified version) type ScrapingJob struct { mongo.Model Type string `bson:"type" json:"type"` Status string `bson:"status" json:"status"` StartedAt int64 `bson:"started_at" json:"started_at"` CompletedAt *int64 `bson:"completed_at,omitempty" json:"completed_at,omitempty"` Progress JobProgress `bson:"progress" json:"progress"` Results JobResults `bson:"results" json:"results"` Config map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"` } // JobProgress represents the progress of a scraping job type JobProgress struct { TotalFiles int `bson:"total_files" json:"total_files"` ProcessedFiles int `bson:"processed_files" json:"processed_files"` FailedFiles int `bson:"failed_files" json:"failed_files"` PercentComplete float64 `bson:"percent_complete" json:"percent_complete"` CurrentFile string `bson:"current_file,omitempty" json:"current_file,omitempty"` } // JobResults represents the results of a scraping job type JobResults struct { TotalTenders int `bson:"total_tenders" json:"total_tenders"` NewTenders int `bson:"new_tenders" json:"new_tenders"` UpdatedTenders int `bson:"updated_tenders" json:"updated_tenders"` ErrorCount int `bson:"error_count" json:"error_count"` SuccessRate float64 `bson:"success_rate" json:"success_rate"` ProcessingTime int64 `bson:"processing_time" json:"processing_time"` } // ScrapingState represents the current state of scraping type ScrapingState struct { mongo.Model LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"` LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"` LastRunAt int64 `bson:"last_run_at" json:"last_run_at"` NextRunAt int64 `bson:"next_run_at" json:"next_run_at"` IsRunning bool `bson:"is_running" json:"is_running"` CurrentJobID bson.ObjectID `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"` } // ScraperConfig holds configuration for the TED scraper type Config struct { BaseURL string `mapstructure:"base_url"` Timeout time.Duration `mapstructure:"timeout"` MaxRetries int `mapstructure:"max_retries"` RetryDelay time.Duration `mapstructure:"retry_delay"` UserAgent string `mapstructure:"user_agent"` MaxConcurrency int `mapstructure:"max_concurrency"` DownloadDir string `mapstructure:"download_dir"` CleanupAfter time.Duration `mapstructure:"cleanup_after"` ScrapingInterval time.Duration `mapstructure:"scraping_interval"` } // ScrapingMode represents the type of scraping operation type ScrapingMode string const ( ScrapingModeDaily ScrapingMode = "daily" // Daily bulk fetch mode ScrapingModeManual ScrapingMode = "manual" // Manual date range mode ScrapingModeRealtime ScrapingMode = "realtime" // Real-time processing mode ) // DateRangeConfig holds configuration for manual date range scraping type DateRangeConfig struct { StartDate time.Time `json:"start_date"` EndDate time.Time `json:"end_date"` Mode ScrapingMode `json:"mode"` } // NewTEDScraper creates a new TED scraper instance func NewTEDScraper( config *Config, logger logger.Logger, mongoManager *mongo.ConnectionManager, tenderRepo tender.TenderRepository, ) *TEDScraper { httpClient := &http.Client{ Timeout: config.Timeout, } return &TEDScraper{ config: config, logger: logger, httpClient: httpClient, xmlParser: NewTEDParser(), mongoManager: mongoManager, tenderRepo: tenderRepo, } } // RunPeriodicScraping runs the scraper at configured intervals func (s *TEDScraper) RunPeriodicScraping(ctx context.Context) error { ticker := time.NewTicker(s.config.ScrapingInterval) defer ticker.Stop() s.logger.Info("Starting periodic TED XML scraping", map[string]interface{}{ "interval": s.config.ScrapingInterval.String(), "base_url": s.config.BaseURL, }) // Run initial scraping if err := s.ScrapeLatest(ctx); err != nil { s.logger.Error("Initial scraping failed", map[string]interface{}{ "error": err.Error(), }) } for { select { case <-ctx.Done(): s.logger.Info("Periodic scraping stopped", map[string]interface{}{ "reason": ctx.Err().Error(), }) return ctx.Err() case <-ticker.C: if err := s.ScrapeLatest(ctx); err != nil { s.logger.Error("Periodic scraping failed", map[string]interface{}{ "error": err.Error(), }) } } } } // ScrapeLatest scrapes the latest TED XML files func (s *TEDScraper) ScrapeLatest(ctx context.Context) error { startTime := time.Now() s.logger.Info("Starting TED XML scraping", map[string]interface{}{ "timestamp": startTime.Unix(), }) // Get current state state, err := s.getScrapingState(ctx) if err != nil { s.logger.Error("Failed to get scraping state", map[string]interface{}{ "error": err.Error(), }) // Initialize with current date if no state exists state = &ScrapingState{ LastProcessedYear: time.Now().Year(), LastProcessedNumber: 0, LastRunAt: startTime.Unix(), NextRunAt: startTime.Add(s.config.ScrapingInterval).Unix(), IsRunning: false, } } // Handle case where state is nil (no document found) if state == nil { s.logger.Info("No scraping state found, initializing new state", map[string]interface{}{}) state = &ScrapingState{ LastProcessedYear: time.Now().Year(), LastProcessedNumber: 0, LastRunAt: startTime.Unix(), NextRunAt: startTime.Add(s.config.ScrapingInterval).Unix(), IsRunning: false, } } if state.IsRunning { s.logger.Info("Scraping already running, skipping", map[string]interface{}{ "current_job_id": state.CurrentJobID, }) return nil } // Create scraping job job := &ScrapingJob{ Type: "ted_daily", Status: "running", StartedAt: startTime.Unix(), Progress: JobProgress{}, Results: JobResults{}, Config: map[string]interface{}{ "base_url": s.config.BaseURL, "max_retries": s.config.MaxRetries, "scraper_type": "ted_daily", }, } // Save job if err := s.saveScrapingJob(ctx, job); err != nil { s.logger.Error("Failed to save scraping job", map[string]interface{}{ "error": err.Error(), }) } // Update state state.IsRunning = true state.CurrentJobID = job.ID state.UpdatedAt = time.Now().Unix() s.saveScrapingState(ctx, state) // Calculate files to download filesToDownload := s.calculateFilesToDownload(state) if len(filesToDownload) == 0 { s.logger.Info("No new files to download", map[string]interface{}{ "last_processed": fmt.Sprintf("%d%05d", state.LastProcessedYear, state.LastProcessedNumber), }) // Complete the job s.completeJob(ctx, job, state, nil) return nil } // Update job progress job.Progress.TotalFiles = len(filesToDownload) s.saveScrapingJob(ctx, job) var lastSuccessfulFile *FileInfo // Download and process files for i, fileInfo := range filesToDownload { job.Progress.CurrentFile = fmt.Sprintf("%d%05d", fileInfo.Year, fileInfo.Number) s.saveScrapingJob(ctx, job) fileResult, err := s.downloadAndProcess(ctx, fileInfo) if err != nil { s.logger.Error("Failed to download and process file", map[string]interface{}{ "year": fileInfo.Year, "number": fileInfo.Number, "error": err.Error(), }) job.Progress.FailedFiles++ job.Results.ErrorCount++ } else { job.Progress.ProcessedFiles++ job.Results.TotalTenders += fileResult.ProcessedCount job.Results.NewTenders += fileResult.NewTenders job.Results.UpdatedTenders += fileResult.UpdatedTenders if len(fileResult.Errors) > 0 { job.Results.ErrorCount += len(fileResult.Errors) } lastSuccessfulFile = &fileInfo } // Update progress percentage job.Progress.PercentComplete = float64(i+1) / float64(len(filesToDownload)) * 100 s.saveScrapingJob(ctx, job) // Check context for cancellation select { case <-ctx.Done(): s.completeJob(ctx, job, state, lastSuccessfulFile) return ctx.Err() default: // Continue processing } } // Complete the job s.completeJob(ctx, job, state, lastSuccessfulFile) s.logger.Info("TED XML scraping completed", map[string]interface{}{ "files_processed": job.Progress.ProcessedFiles, "tenders_created": job.Results.NewTenders, "tenders_updated": job.Results.UpdatedTenders, "total_tenders": job.Results.TotalTenders, "error_count": job.Results.ErrorCount, "success_rate": job.Results.SuccessRate, }) return nil } // FileInfo represents information about a TED file to download type FileInfo struct { Year int Number int URL string OJS string } // calculateFilesToDownload determines which files need to be downloaded func (s *TEDScraper) calculateFilesToDownload(state *ScrapingState) []FileInfo { var files []FileInfo now := time.Now() currentYear := now.Year() currentMonth := now.Month() // Only process current year and month year := currentYear // Start from the last processed file if it's from current year, otherwise start from beginning of current month startNumber := 1 if state.LastProcessedYear == currentYear { startNumber = state.LastProcessedNumber + 1 } else { // Calculate approximate file number for start of current month // Assuming roughly 22 working days per month (weekdays only) startNumber = int(currentMonth-1)*22 + 1 } // Calculate end number for current month // Estimate based on current day of month and weekdays daysInCurrentMonth := now.Day() // Rough estimate: multiply by 0.7 to account for weekdays only maxNumber := int(currentMonth-1)*22 + int(float64(daysInCurrentMonth)*0.7) // Add some buffer for potential files maxNumber += 5 s.logger.Info("Calculating files to download for current month", map[string]interface{}{ "current_year": currentYear, "current_month": int(currentMonth), "start_number": startNumber, "max_number": maxNumber, "last_processed": fmt.Sprintf("%d%05d", state.LastProcessedYear, state.LastProcessedNumber), }) startNumber = 146 for number := startNumber; number <= maxNumber && len(files) < 10; number++ { // Limit to 10 files per run fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number) ojsID := fmt.Sprintf("S %03d / %d", number, year) files = append(files, FileInfo{ Year: year, Number: number, URL: fileURL, OJS: ojsID, }) } return files } // FileProcessingResult represents the result of processing a single file type FileProcessingResult struct { Year int `json:"year"` Number int `json:"number"` ProcessedCount int `json:"processed_count"` SuccessCount int `json:"success_count"` NewTenders int `json:"new_tenders"` UpdatedTenders int `json:"updated_tenders"` Errors []string `json:"errors"` ProcessedAt int64 `json:"processed_at"` } // downloadAndProcess downloads and processes a single TED file func (s *TEDScraper) downloadAndProcess(ctx context.Context, fileInfo FileInfo) (*FileProcessingResult, error) { url := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, fileInfo.Year, fileInfo.Number) s.logger.Info("Downloading TED file", map[string]interface{}{ "url": url, "year": fileInfo.Year, "number": fileInfo.Number, }) var result *FileProcessingResult var err error // Retry mechanism for attempt := 1; attempt <= s.config.MaxRetries; attempt++ { result, err = s.downloadAndProcessWithRetry(ctx, url, fileInfo) if err == nil { break } if attempt < s.config.MaxRetries { s.logger.Warn("Download attempt failed, retrying", map[string]interface{}{ "attempt": attempt, "error": err.Error(), "url": url, }) select { case <-ctx.Done(): return nil, ctx.Err() case <-time.After(s.config.RetryDelay): // Continue to next attempt } } } if err != nil { s.logger.Error("Failed to download and process file after retries", map[string]interface{}{ "url": url, "attempts": s.config.MaxRetries, "error": err.Error(), }) return nil, fmt.Errorf("failed to download %s after %d attempts: %w", url, s.config.MaxRetries, err) } return result, nil } // downloadAndProcessWithRetry performs a single download and process attempt func (s *TEDScraper) downloadAndProcessWithRetry(ctx context.Context, url string, fileInfo FileInfo) (*FileProcessingResult, error) { // Create a longer timeout context for processing large files processCtx, cancel := context.WithTimeout(ctx, s.config.Timeout+time.Minute*5) defer cancel() // Create HTTP request req, err := http.NewRequestWithContext(processCtx, "GET", url, nil) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", s.config.UserAgent) s.logger.Info("Starting download", map[string]interface{}{ "url": url, "year": fileInfo.Year, "number": fileInfo.Number, }) // Make request resp, err := s.httpClient.Do(req) if err != nil { return nil, fmt.Errorf("failed to download file: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP error: %d %s", resp.StatusCode, resp.Status) } s.logger.Info("Download completed, starting processing", map[string]interface{}{ "url": url, "content_length": resp.Header.Get("Content-Length"), }) // Process tar.gz file directly from response body return s.processTarGzFile(processCtx, resp.Body, fileInfo) } // processTarGzFile extracts and processes XML files from a tar.gz archive func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fileInfo FileInfo) (*FileProcessingResult, error) { result := &FileProcessingResult{ Year: fileInfo.Year, Number: fileInfo.Number, ProcessedCount: 0, SuccessCount: 0, Errors: make([]string, 0), ProcessedAt: time.Now().Unix(), } // Create gzip reader gzipReader, err := gzip.NewReader(reader) if err != nil { return nil, fmt.Errorf("failed to create gzip reader: %w", err) } defer gzipReader.Close() // Create tar reader tarReader := tar.NewReader(gzipReader) // Process each file in the tar archive fileCount := 0 for { // Check context for cancellation before each tar entry select { case <-ctx.Done(): s.logger.Warn("Context cancelled during tar processing", map[string]interface{}{ "processed_files": fileCount, "error": ctx.Err().Error(), }) return result, ctx.Err() default: } header, err := tarReader.Next() if err == io.EOF { break // End of archive } if err != nil { s.logger.Warn("Error reading tar entry", map[string]interface{}{ "error": err.Error(), "processed_files": fileCount, }) // If it's a timeout error, return it immediately if ctx.Err() != nil { return result, fmt.Errorf("timeout during tar processing: %w", err) } continue } fileCount++ // Skip directories and non-XML files if header.Typeflag != tar.TypeReg { continue } fileName := filepath.Base(header.Name) if !strings.HasSuffix(strings.ToLower(fileName), ".xml") { continue } // Log progress every 10 files or for first file if result.ProcessedCount == 0 || result.ProcessedCount%10 == 0 { s.logger.Info("Processing XML file from tar.gz", map[string]interface{}{ "file_name": fileName, "size": header.Size, "processed_count": result.ProcessedCount, "total_files_seen": fileCount, "year": fileInfo.Year, "number": fileInfo.Number, }) } else { s.logger.Debug("Processing XML file from tar.gz", map[string]interface{}{ "file_name": fileName, "processed_count": result.ProcessedCount, }) } // Read file content content, err := io.ReadAll(tarReader) if err != nil { errMsg := fmt.Sprintf("failed to read XML file %s: %v", fileName, err) result.Errors = append(result.Errors, errMsg) continue } // Parse and process XML content processResult, err := s.processXMLContent(ctx, content, fileName, fileInfo) if err != nil { errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err) result.Errors = append(result.Errors, errMsg) } else { result.SuccessCount++ if processResult.IsNew { result.NewTenders++ } else { result.UpdatedTenders++ } } result.ProcessedCount++ // Check context for cancellation select { case <-ctx.Done(): return result, ctx.Err() default: // Continue processing } } s.logger.Info("Completed processing tar.gz file", map[string]interface{}{ "year": fileInfo.Year, "number": fileInfo.Number, "total_files": fileCount, "processed_count": result.ProcessedCount, "success_count": result.SuccessCount, "error_count": len(result.Errors), "processing_time": time.Now().Unix() - result.ProcessedAt, }) return result, nil } // XMLProcessResult represents the result of processing a single XML file type XMLProcessResult struct { IsNew bool `json:"is_new"` TenderID string `json:"tender_id"` ContractID string `json:"contract_id"` } // processXMLContent parses XML content and creates/updates tender func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo) (*XMLProcessResult, error) { s.logger.Info("Starting XML processing", map[string]interface{}{ "file_name": fileName, "file_size": len(content), }) // Parse XML using the TED parser parsedDoc, err := s.xmlParser.ParseXML(content) if err != nil { s.logger.Error("XML parsing failed", map[string]interface{}{ "file_name": fileName, "error": err.Error(), }) return nil, fmt.Errorf("failed to parse XML: %w", err) } // Get the underlying document for ID and validation document := parsedDoc.GetDocument() documentID := "" noticeTypeCode := "" if document != nil { documentID = document.GetID() noticeTypeCode = document.GetNoticeType() } s.logger.Info("XML parsed successfully", map[string]interface{}{ "file_name": fileName, "document_type": string(parsedDoc.Type), "document_id": documentID, "notice_type_code": noticeTypeCode, }) // Validate parsed data var validationErr error if document != nil { validationErr = document.Validate() } if validationErr != nil { s.logger.Warn("XML validation failed, continuing anyway", map[string]interface{}{ "file_name": fileName, "document_type": string(parsedDoc.Type), "error": validationErr.Error(), }) // Don't return error, continue processing } else { s.logger.Info("XML validation passed", map[string]interface{}{ "file_name": fileName, "document_type": string(parsedDoc.Type), }) } // Map to tender entity sourceURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, fileInfo.Year, fileInfo.Number) tenderEntity := s.mapToTenderFromParsedDoc(parsedDoc, sourceURL, fileName) // Handle status-specific information and award logic s.applyStatusSpecificInformation(tenderEntity, parsedDoc) s.applyAwardLogic(tenderEntity, parsedDoc) s.logger.Info("Mapped to tender entity", map[string]interface{}{ "file_name": fileName, "tender_id": tenderEntity.TenderID, "contract_notice_id": tenderEntity.ContractNoticeID, "title": tenderEntity.Title, "status": string(tenderEntity.Status), }) // Try to find existing tender by contract notice ID (ContractID) s.logger.Info("Checking for existing tender", map[string]interface{}{ "contract_notice_id": tenderEntity.ContractNoticeID, }) existingTender, err := s.findTenderByContractNoticeID(ctx, tenderEntity.ContractNoticeID) if err != nil && err.Error() != "document not found" { s.logger.Warn("Error checking for existing tender", map[string]interface{}{ "contract_notice_id": tenderEntity.ContractNoticeID, "error": err.Error(), }) } var processResult *XMLProcessResult if err == nil && existingTender != nil { // Update existing tender s.logger.Info("Found existing tender, updating", map[string]interface{}{ "existing_tender_id": existingTender.ID, "contract_notice_id": tenderEntity.ContractNoticeID, "old_status": string(existingTender.Status), "new_status": string(tenderEntity.Status), }) // Merge updates with existing data s.mergeExistingTenderData(existingTender, tenderEntity) tenderEntity.ID = existingTender.ID tenderEntity.CreatedAt = existingTender.CreatedAt tenderEntity.UpdatedAt = time.Now().Unix() err = s.tenderRepo.Update(ctx, tenderEntity) if err != nil { s.logger.Error("Failed to update existing tender", map[string]interface{}{ "tender_id": tenderEntity.ID, "error": err.Error(), }) return nil, fmt.Errorf("failed to update existing tender: %w", err) } s.logger.Info("Successfully updated tender", map[string]interface{}{ "tender_id": tenderEntity.ID, }) processResult = &XMLProcessResult{ IsNew: false, TenderID: tenderEntity.TenderID, ContractID: tenderEntity.ContractNoticeID, } } else { // Create new tender s.logger.Info("Creating new tender", map[string]interface{}{ "contract_notice_id": tenderEntity.ContractNoticeID, "tender_id": tenderEntity.TenderID, "status": string(tenderEntity.Status), }) tenderEntity.CreatedAt = time.Now().Unix() tenderEntity.UpdatedAt = tenderEntity.CreatedAt err = s.tenderRepo.Create(ctx, tenderEntity) if err != nil { s.logger.Error("Failed to create new tender", map[string]interface{}{ "contract_notice_id": tenderEntity.ContractNoticeID, "tender_id": tenderEntity.TenderID, "error": err.Error(), }) return nil, fmt.Errorf("failed to create tender: %w", err) } s.logger.Info("Successfully created new tender", map[string]interface{}{ "tender_id": tenderEntity.TenderID, "contract_notice_id": tenderEntity.ContractNoticeID, }) processResult = &XMLProcessResult{ IsNew: true, TenderID: tenderEntity.TenderID, ContractID: tenderEntity.ContractNoticeID, } } s.logger.Debug("Successfully processed XML file", map[string]interface{}{ "file_name": fileName, "document_type": string(parsedDoc.Type), "document_id": documentID, "tender_id": tenderEntity.TenderID, "is_new": processResult.IsNew, }) return processResult, nil } // Helper methods for database operations func (s *TEDScraper) getScrapingState(ctx context.Context) (*ScrapingState, error) { collection := s.mongoManager.GetCollection("scraping_state") var state ScrapingState err := collection.FindOne(ctx, bson.M{"_id": "ted_scraper"}).Decode(&state) if err != nil { if err == mongoDriver.ErrNoDocuments { return nil, nil } return nil, err } return &state, nil } func (s *TEDScraper) saveScrapingState(ctx context.Context, state *ScrapingState) error { collection := s.mongoManager.GetCollection("scraping_state") state.UpdatedAt = time.Now().Unix() opts := options.Replace().SetUpsert(true) _, err := collection.ReplaceOne(ctx, bson.M{"_id": state.ID}, state, opts) return err } func (s *TEDScraper) saveScrapingJob(ctx context.Context, job *ScrapingJob) error { collection := s.mongoManager.GetCollection("scraping_jobs") job.UpdatedAt = time.Now().Unix() opts := options.Replace().SetUpsert(true) _, err := collection.ReplaceOne(ctx, bson.M{"_id": job.ID}, job, opts) return err } func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*tender.Tender, error) { // Use repository's specific method for finding by contract notice ID existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID) if err != nil { // If document not found, return nil without error if err.Error() == "document not found" { return nil, nil } return nil, err } return existingTender, nil } func (s *TEDScraper) completeJob(ctx context.Context, job *ScrapingJob, state *ScrapingState, lastFile *FileInfo) { // Update state with last successful file if lastFile != nil { state.LastProcessedYear = lastFile.Year state.LastProcessedNumber = lastFile.Number } // Complete the job endTime := time.Now() completedAt := endTime.Unix() job.Status = "completed" job.CompletedAt = &completedAt job.Results.ProcessingTime = completedAt - job.StartedAt if job.Results.TotalTenders > 0 { job.Results.SuccessRate = float64(job.Results.NewTenders) / float64(job.Results.TotalTenders) } s.saveScrapingJob(ctx, job) // Update state state.IsRunning = false state.CurrentJobID = bson.NilObjectID state.LastRunAt = completedAt state.NextRunAt = endTime.Add(s.config.ScrapingInterval).Unix() state.UpdatedAt = completedAt s.saveScrapingState(ctx, state) } // mapToTenderFromParsedDoc maps a parsed TED document to tender entity func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *tender.Tender { switch parsedDoc.Type { case DocumentTypeContractNotice: if parsedDoc.ContractNotice == nil { s.logger.Error("ContractNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapToTender(parsedDoc.ContractNotice, sourceFileURL, sourceFileName) case DocumentTypeContractAwardNotice: if parsedDoc.ContractAwardNotice == nil { s.logger.Error("ContractAwardNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapContractAwardNoticeToTender(parsedDoc.ContractAwardNotice, sourceFileURL, sourceFileName) case DocumentTypePriorInformationNotice: if parsedDoc.PriorInformationNotice == nil { s.logger.Error("PriorInformationNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapGenericNoticeToTender(parsedDoc.PriorInformationNotice, sourceFileURL, sourceFileName) case DocumentTypeDesignContest: if parsedDoc.DesignContest == nil { s.logger.Error("DesignContest is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapGenericNoticeToTender(parsedDoc.DesignContest, sourceFileURL, sourceFileName) case DocumentTypeQualificationSystemNotice: if parsedDoc.QualificationSystemNotice == nil { s.logger.Error("QualificationSystemNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapGenericNoticeToTender(parsedDoc.QualificationSystemNotice, sourceFileURL, sourceFileName) case DocumentTypeConcessionNotice: if parsedDoc.ConcessionNotice == nil { s.logger.Error("ConcessionNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapGenericNoticeToTender(parsedDoc.ConcessionNotice, sourceFileURL, sourceFileName) case DocumentTypePlanningNotice: if parsedDoc.PlanningNotice == nil { s.logger.Error("PlanningNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapGenericNoticeToTender(parsedDoc.PlanningNotice, sourceFileURL, sourceFileName) case DocumentTypeCompetitionNotice: if parsedDoc.CompetitionNotice == nil { s.logger.Error("CompetitionNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapGenericNoticeToTender(parsedDoc.CompetitionNotice, sourceFileURL, sourceFileName) case DocumentTypeResultNotice: if parsedDoc.ResultNotice == nil { s.logger.Error("ResultNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapGenericNoticeToTender(parsedDoc.ResultNotice, sourceFileURL, sourceFileName) case DocumentTypeChangeNotice: if parsedDoc.ChangeNotice == nil { s.logger.Error("ChangeNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapGenericNoticeToTender(parsedDoc.ChangeNotice, sourceFileURL, sourceFileName) case DocumentTypeSubcontractNotice: if parsedDoc.SubcontractNotice == nil { s.logger.Error("SubcontractNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapGenericNoticeToTender(parsedDoc.SubcontractNotice, sourceFileURL, sourceFileName) // Legacy document types that should be parsed as ContractNotice but may still be in the old flow case DocumentTypeCorrigendum, DocumentTypeModificationNotice: if parsedDoc.ContractNotice != nil { return s.mapToTender(parsedDoc.ContractNotice, sourceFileURL, sourceFileName) } // Fallback: try to get the document from the generic interface document := parsedDoc.GetDocument() if document != nil { return s.mapGenericNoticeToTender(document, sourceFileURL, sourceFileName) } s.logger.Error("No valid document found for legacy type", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil default: s.logger.Error("Unknown document type for mapping", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } } // mapGenericNoticeToTender maps any TED document type to tender entity using the common interface func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *tender.Tender { if doc == nil { s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{}) return nil } now := time.Now().Unix() t := &tender.Tender{ ContractNoticeID: doc.GetID(), ContractFolderID: "", // Not available in generic interface NoticeTypeCode: doc.GetNoticeType(), NoticeSubTypeCode: "", // Not available in generic interface NoticeLanguageCode: doc.GetLanguage(), IssueDate: now, // TODO: Parse actual date from document IssueTime: now, // TODO: Parse actual time from document GazetteID: "", // Not available in generic interface NoticePublicationID: "", // Not available in generic interface PublicationDate: now, // TODO: Parse actual publication date // Basic tender information Title: "Generic Notice " + doc.GetNoticeType(), Description: "Processed from " + doc.GetNoticeType() + " document", Status: tender.TenderStatusActive, Source: tender.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, ProcessingMetadata: tender.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", }, } // Generate the tender ID after creating the tender t.TenderID = s.generateTenderID(t) return t } // mapToTender maps a TED ContractNotice to tender entity func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *tender.Tender { now := time.Now().Unix() t := &tender.Tender{ ContractNoticeID: cn.ID, ContractFolderID: cn.ContractFolderID, NoticeTypeCode: cn.GetNoticeSubType(), NoticeSubTypeCode: cn.GetNoticeSubType(), NoticeLanguageCode: cn.NoticeLanguageCode, IssueDate: s.parseDateToUnix(cn.IssueDate), IssueTime: s.parseTimeToUnix(cn.IssueTime), GazetteID: cn.GetOJSID(), Title: cn.GetProcurementTitle(), Description: cn.GetProcurementDescription(), ProcurementTypeCode: cn.GetContractNature(), ProcedureCode: cn.GetProcedureCode(), MainClassification: cn.GetMainClassification(), PlaceOfPerformance: cn.GetPlaceOfPerformance(), DocumentURI: cn.GetDocumentURI(), Status: tender.TenderStatusActive, Source: tender.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, ProcessingMetadata: tender.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", }, } // Set publication information if pubInfo := cn.GetPublicationInfo(); pubInfo != nil { t.NoticePublicationID = pubInfo.NoticePublicationID t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate) t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID) } // Set additional classifications if cn.ProcurementProject != nil && cn.ProcurementProject.AdditionalCommodityClassification != nil { for _, additionalClass := range cn.ProcurementProject.AdditionalCommodityClassification { if additionalClass.ItemClassificationCode != "" { t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode) } } } // Set estimated value and currency if amount, currency := cn.GetBudget(); amount != "" { if value, err := strconv.ParseFloat(amount, 64); err == nil { t.EstimatedValue = value t.Currency = currency } } // Set duration information if duration, unit := cn.GetDuration(); duration != "" { t.Duration = duration t.DurationUnit = unit } // Set tender deadline if deadline := cn.GetTenderDeadline(); deadline != "" { t.TenderDeadline = s.parseDateToUnix(deadline) if deadlineTime, err := s.parseDate(deadline); err == nil { applicationDeadline := s.calculateApplicationDeadline(deadlineTime) t.ApplicationDeadline = applicationDeadline.Unix() } } if pubDate, err := s.parseDate(cn.GetPublicationDate()); err == nil { submissionDeadline := s.calculateSubmissionDeadline(pubDate) t.SubmissionDeadline = submissionDeadline.Unix() } // Set SubmissionURL from TenderingTerms if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil { if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil { if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" { if strings.HasPrefix(endpointID, "http") { t.SubmissionURL = endpointID } else { t.SubmissionURL = "https://" + endpointID } } } } // Set location information if cn.ProcurementProject != nil && cn.ProcurementProject.RealizedLocation != nil { if addr := cn.ProcurementProject.RealizedLocation.Address; addr != nil { t.CityName = addr.CityName t.PostalCode = addr.PostalZone t.RegionCode = addr.CountrySubentityCode if addr.Country != nil { t.CountryCode = addr.Country.IdentificationCode } } } // Set buyer organization buyerID := cn.GetBuyerID() if buyerID != "" { if buyerOrg := cn.GetOrganizationByID(buyerID); buyerOrg != nil { t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer") } } // Set review organization reviewID := cn.GetReviewOrganizationID() if reviewID != "" { if reviewOrg := cn.GetOrganizationByID(reviewID); reviewOrg != nil { t.ReviewOrganization = s.mapOrganization(reviewOrg, "review") } } // Set all organizations if orgs := cn.GetOrganizations(); orgs != nil { for _, org := range orgs.Organization { mappedOrg := s.mapOrganization(&org, "") if mappedOrg != nil { t.Organizations = append(t.Organizations, *mappedOrg) } } } // Set selection criteria if criteria := cn.GetSelectionCriteria(); criteria != nil { for _, criterion := range criteria { t.SelectionCriteria = append(t.SelectionCriteria, tender.SelectionCriterion{ TypeCode: criterion.CriterionTypeCode, Description: criterion.Description, LanguageID: criterion.LanguageID, }) } } // Set official languages if languages := cn.GetOfficialLanguages(); languages != nil { t.OfficialLanguages = languages } // Generate unique tender ID t.TenderID = s.generateTenderID(t) return t } // mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *tender.Tender { now := time.Now().Unix() t := &tender.Tender{ ContractNoticeID: can.ID, ContractFolderID: can.ContractFolderID, NoticeTypeCode: can.NoticeTypeCode, NoticeSubTypeCode: can.GetNoticeSubType(), NoticeLanguageCode: can.NoticeLanguageCode, IssueDate: s.parseDateToUnix(can.IssueDate), IssueTime: s.parseTimeToUnix(can.IssueTime), GazetteID: can.GetOJSID(), Title: can.GetProcurementTitle(), Description: can.GetProcurementDescription(), ProcurementTypeCode: can.GetContractNature(), ProcedureCode: can.GetProcedureCode(), MainClassification: can.GetMainClassification(), Status: tender.TenderStatusAwarded, // Award notices are for completed tenders Source: tender.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, ProcessingMetadata: tender.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", }, } // Set publication information if pubInfo := can.GetPublicationInfo(); pubInfo != nil { t.NoticePublicationID = pubInfo.NoticePublicationID t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate) t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID) } // Set total award value if amount, currency := can.GetTotalAwardValue(); amount != "" { if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil { t.EstimatedValue = parsedAmount } t.Currency = currency } // Set location information from main procurement project if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil { if addr := can.ProcurementProject.RealizedLocation.Address; addr != nil { t.CityName = addr.CityName t.PostalCode = addr.PostalZone t.RegionCode = addr.CountrySubentityCode if addr.Country != nil { t.CountryCode = addr.Country.IdentificationCode } } } // Set buyer organization buyerID := can.GetBuyerID() if buyerID != "" { if buyerOrg := can.GetOrganizationByID(buyerID); buyerOrg != nil { t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer") } } // Set all organizations if orgs := can.GetOrganizations(); orgs != nil { for _, org := range orgs.Organization { mappedOrg := s.mapOrganization(&org, "") if mappedOrg != nil { t.Organizations = append(t.Organizations, *mappedOrg) } } } // Generate unique tender ID t.TenderID = s.generateTenderID(t) return t } // mapOrganization maps a TED Organization to tender Organization func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender.Organization { if tedOrg == nil || tedOrg.Company == nil { return nil } company := tedOrg.Company org := &tender.Organization{ Role: role, } // Set organization ID and name if company.PartyIdentification != nil { org.ID = company.PartyIdentification.ID } if company.PartyName != nil { org.Name = company.PartyName.Name } // Set company ID if company.PartyLegalEntity != nil { org.CompanyID = company.PartyLegalEntity.CompanyID } // Set website org.WebsiteURI = company.WebsiteURI // Set contact information if company.Contact != nil { org.ContactName = company.Contact.Name org.ContactTelephone = company.Contact.Telephone org.ContactEmail = company.Contact.ElectronicMail org.ContactFax = company.Contact.Telefax } // Set address if company.PostalAddress != nil { org.Address = tender.Address{ StreetName: company.PostalAddress.StreetName, CityName: company.PostalAddress.CityName, PostalZone: company.PostalAddress.PostalZone, CountrySubentityCode: company.PostalAddress.CountrySubentityCode, Department: company.PostalAddress.Department, } if company.PostalAddress.Country != nil { org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode } } return org } // generateTenderID generates a unique tender ID using the format: // {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE} func (s *TEDScraper) generateTenderID(t *tender.Tender) string { var parts []string // Source (TED) parts = append(parts, "1") // TED code (Notice Publication ID, first 8 chars) tedCode := "00000000" if t.NoticePublicationID != "" { tedCode = s.padOrTruncate(t.NoticePublicationID, 8) } parts = append(parts, tedCode) // Buyer code (first 8 chars of buyer organization ID, or 8 zeros if not available) // buyerCode := "00000000" // if t.BuyerOrganization != nil && t.BuyerOrganization.ID != "" { // buyerCode = s.padOrTruncate(t.BuyerOrganization.ID, 8) // } // parts = append(parts, buyerCode) // Date (YYYYMMDD format from issue date) parts = append(parts, s.padOrTruncate(s.UnixToDateString(t.IssueDate), 4)) // Location code (country code + region code, padded to 6 chars) locationCode := "EU" if t.CountryCode != "" { locationCode = t.CountryCode } parts = append(parts, locationCode) return strings.Join(parts, "") } // padOrTruncate pads a string with zeros or truncates it to the specified length func (s *TEDScraper) padOrTruncate(str string, length int) string { // Remove any non-alphanumeric characters and convert to uppercase clean := strings.ToUpper(strings.ReplaceAll(str, "-", "")) clean = strings.ReplaceAll(clean, " ", "") if len(clean) >= length { return clean[:length] } // Pad with zeros return clean + strings.Repeat("0", length-len(clean)) } // generateTenderURL generates the TED tender URL from NoticePublicationID func (s *TEDScraper) generateTenderURL(noticePublicationID string) string { if noticePublicationID == "" { return "" } // Remove leading zeros and format for URL // Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025 parts := strings.Split(noticePublicationID, "-") if len(parts) != 2 { return "" } // Convert to int and back to string to remove leading zeros if num, err := strconv.Atoi(parts[0]); err == nil { cleanID := fmt.Sprintf("%d-%s", num, parts[1]) return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID) } return "" } // parseDate parses various date formats used in TED XML func (s *TEDScraper) parseDate(dateStr string) (time.Time, error) { if dateStr == "" { return time.Time{}, fmt.Errorf("empty date string") } // Common TED date formats formats := []string{ "2006-01-02+07:00", "2006-01-02-07:00", "2006-01-02T15:04:05+07:00", "2006-01-02T15:04:05-07:00", "2006-01-02T15:04:05Z", "2006-01-02Z", "2006-01-02", time.RFC3339, time.RFC3339Nano, } for _, format := range formats { if parsedTime, err := time.Parse(format, dateStr); err == nil { return parsedTime, nil } } return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr) } // parseDateToUnix converts string date to Unix milliseconds (int64) func (s *TEDScraper) parseDateToUnix(dateStr string) int64 { if dateStr == "" { return 0 } if parsedTime, err := s.parseDate(dateStr); err == nil { return parsedTime.Unix() } return 0 } // parseTimeToUnix converts string time to Unix milliseconds (int64) func (s *TEDScraper) parseTimeToUnix(timeStr string) int64 { if timeStr == "" { return 0 } // If time string is just time without date, use today's date if !strings.Contains(timeStr, "T") && !strings.Contains(timeStr, " ") { // Assume it's just a time like "15:04:05" today := time.Now().Format("2006-01-02") timeStr = fmt.Sprintf("%sT%s", today, timeStr) } if parsedTime, err := s.parseDate(timeStr); err == nil { return parsedTime.Unix() } return 0 } // UnixToDateString converts Unix milliseconds to date string for TenderID generation func (s *TEDScraper) UnixToDateString(u int64) string { if u == 0 { return "00000000" } return time.Unix(u, 0).Format("20060102") } func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time { deadline := publicationDate workingHoursAdded := 0 for workingHoursAdded < 48 { deadline = deadline.Add(1 * time.Hour) if deadline.Weekday() != time.Saturday && deadline.Weekday() != time.Sunday { workingHoursAdded++ } } return deadline } func (s *TEDScraper) calculateApplicationDeadline(tenderDeadline time.Time) time.Time { applicationDeadline := tenderDeadline workingDaysSubtracted := 0 for workingDaysSubtracted < 7 { applicationDeadline = applicationDeadline.AddDate(0, 0, -1) if applicationDeadline.Weekday() != time.Saturday && applicationDeadline.Weekday() != time.Sunday { workingDaysSubtracted++ } } return applicationDeadline } func (s *TEDScraper) adjustToWorkingDay(targetDate time.Time, forward bool) time.Time { current := targetDate for current.Weekday() == time.Saturday || current.Weekday() == time.Sunday { if forward { current = current.AddDate(0, 0, 1) } else { current = current.AddDate(0, 0, -1) } } return current } // mapTEDStatusToTenderStatus maps TED notice status to tender status func (s *TEDScraper) mapTEDStatusToTenderStatus(tedStatus NoticeStatus) tender.TenderStatus { switch tedStatus { case NoticeStatusDraft: return tender.TenderStatusDraft case NoticeStatusPublished: return tender.TenderStatusPublished case NoticeStatusCancelled: return tender.TenderStatusCancelled case NoticeStatusAwarded: return tender.TenderStatusAwarded case NoticeStatusClosed: return tender.TenderStatusClosed case NoticeStatusModified: return tender.TenderStatusModified case NoticeStatusSuspended: return tender.TenderStatusSuspended default: // Default to active for unknown statuses, but log a warning s.logger.Warn("Unknown TED notice status, defaulting to active", map[string]interface{}{ "ted_status": string(tedStatus), }) return tender.TenderStatusActive } } // applyStatusSpecificInformation applies status-specific information to the tender entity func (s *TEDScraper) applyStatusSpecificInformation(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) { if tenderEntity == nil || parsedDoc == nil { return } // Map TED status to tender status tenderEntity.Status = s.mapTEDStatusToTenderStatus(parsedDoc.Status) // Get the document to access status-specific methods document := parsedDoc.GetDocument() if document == nil { return } // Apply status-specific information based on document type switch parsedDoc.Type { case DocumentTypeContractNotice: if cn, ok := document.(*ContractNotice); ok { s.applyContractNoticeStatusInfo(tenderEntity, cn, parsedDoc.Status) } case DocumentTypeContractAwardNotice: if can, ok := document.(*ContractAwardNotice); ok { s.applyContractAwardNoticeStatusInfo(tenderEntity, can, parsedDoc.Status) } default: // For other document types, we rely on the parsed status s.logger.Debug("Applied basic status information for document type", map[string]interface{}{ "document_type": string(parsedDoc.Type), "status": string(parsedDoc.Status), }) } } // applyContractNoticeStatusInfo applies status-specific information from ContractNotice func (s *TEDScraper) applyContractNoticeStatusInfo(tenderEntity *tender.Tender, cn *ContractNotice, status NoticeStatus) { switch status { case NoticeStatusCancelled: if cancellation := cn.GetCancellationReason(); cancellation != nil { tenderEntity.CancellationReason = cancellation.Description if cancellation.ReasonCode != "" { tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description) } // Note: TED XML doesn't typically have cancellation date, so we use current time tenderEntity.CancellationDate = time.Now().Unix() } case NoticeStatusAwarded: if award := cn.GetAwardInformation(); award != nil { if award.AwardDate != "" { tenderEntity.AwardDate = s.parseDateToUnix(award.AwardDate) } if award.AwardedValue.Text != "" { if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil { tenderEntity.AwardedValue = value } } tenderEntity.ContractNumber = award.ContractNumber if award.WinningTenderer != nil && award.WinningTenderer.Company != nil { tenderEntity.WinningTenderer = s.mapOrganization(&Organization{ Company: award.WinningTenderer.Company, }, "winner") } } case NoticeStatusSuspended: if suspension := cn.GetSuspensionReason(); suspension != nil { tenderEntity.SuspensionReason = suspension.Description if suspension.ReasonCode != "" { tenderEntity.SuspensionReason = fmt.Sprintf("%s: %s", suspension.ReasonCode, suspension.Description) } tenderEntity.SuspensionDate = time.Now().Unix() } case NoticeStatusModified: if modifications := cn.GetModifications(); modifications != nil { for _, mod := range modifications { tenderMod := tender.TenderModification{ ModificationDate: s.parseDateToUnix(mod.ModificationDate), ModificationReason: mod.ModificationReason, Description: mod.Description, LanguageID: mod.LanguageID, } tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod) } } } } // applyContractAwardNoticeStatusInfo applies status-specific information from ContractAwardNotice func (s *TEDScraper) applyContractAwardNoticeStatusInfo(tenderEntity *tender.Tender, can *ContractAwardNotice, status NoticeStatus) { // Contract award notices are typically awarded status tenderEntity.Status = tender.TenderStatusAwarded // Set award date from tender result if can.TenderResult != nil && can.TenderResult.AwardDate != "" { tenderEntity.AwardDate = s.parseDateToUnix(can.TenderResult.AwardDate) } // Get award information from extensions if available if award := can.GetAwardInformation(); award != nil { if award.AwardedValue.Text != "" { if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil { tenderEntity.AwardedValue = value } } tenderEntity.ContractNumber = award.ContractNumber } // Get total award value from notice result if amount, currency := can.GetTotalAwardValue(); amount != "" { if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil { tenderEntity.AwardedValue = parsedAmount tenderEntity.Currency = currency } } // Extract detailed awarded contractors information s.extractAwardedEntities(tenderEntity, can) // Handle other statuses if applicable switch status { case NoticeStatusCancelled: if cancellation := can.GetCancellationReason(); cancellation != nil { tenderEntity.Status = tender.TenderStatusCancelled tenderEntity.CancellationReason = cancellation.Description if cancellation.ReasonCode != "" { tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description) } tenderEntity.CancellationDate = time.Now().Unix() } case NoticeStatusModified: if modifications := can.GetModifications(); modifications != nil { tenderEntity.Status = tender.TenderStatusModified for _, mod := range modifications { tenderMod := tender.TenderModification{ ModificationDate: s.parseDateToUnix(mod.ModificationDate), ModificationReason: mod.ModificationReason, Description: mod.Description, LanguageID: mod.LanguageID, } tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod) } } } } // extractAwardedEntities extracts detailed information about awarded entities from a contract award notice func (s *TEDScraper) extractAwardedEntities(tenderEntity *tender.Tender, can *ContractAwardNotice) { awardedContractors := can.GetAwardedContractors() if len(awardedContractors) == 0 { return } s.logger.Info("Extracting awarded entities", map[string]interface{}{ "contract_notice_id": can.ID, "num_contractors": len(awardedContractors), }) for _, contractor := range awardedContractors { entity := tender.Awarded{ Name: contractor.Name, Address: contractor.Address, Country: contractor.Country, Currency: contractor.Currency, ContractID: contractor.ContractID, TenderID: contractor.TenderID, LotID: contractor.LotID, CompanyID: contractor.CompanyID, OrganizationID: contractor.OrganizationID, } // Parse amount if contractor.Amount != "" { if amount, err := strconv.ParseFloat(contractor.Amount, 64); err == nil { entity.Amount = amount } else { s.logger.Warn("Failed to parse award amount", map[string]interface{}{ "amount_string": contractor.Amount, "contractor": contractor.Name, "error": err.Error(), }) } } // Parse award date if contractor.AwardDate != "" { entity.AwardDate = s.parseDateToUnix(contractor.AwardDate) } else if contractor.IssueDate != "" { entity.AwardDate = s.parseDateToUnix(contractor.IssueDate) } // Calculate share percentage if there are multiple contractors totalAmount := tenderEntity.AwardedValue if totalAmount > 0 && entity.Amount > 0 { entity.Share = (entity.Amount / totalAmount) * 100 } tenderEntity.AwardedEntities = append(tenderEntity.AwardedEntities, entity) s.logger.Debug("Added awarded entity", map[string]interface{}{ "name": entity.Name, "amount": entity.Amount, "currency": entity.Currency, "share": entity.Share, "contract_id": entity.ContractID, }) } s.logger.Info("Successfully extracted awarded entities", map[string]interface{}{ "total_entities": len(tenderEntity.AwardedEntities), "total_value": tenderEntity.AwardedValue, "currency": tenderEntity.Currency, }) } // ScrapeByDateRange scrapes TED XML files within a specified date range func (s *TEDScraper) ScrapeByDateRange(ctx context.Context, config DateRangeConfig) error { startTime := time.Now() s.logger.Info("Starting TED XML scraping by date range", map[string]interface{}{ "start_date": config.StartDate.Format("2006-01-02"), "end_date": config.EndDate.Format("2006-01-02"), "mode": string(config.Mode), "timestamp": startTime.Unix(), }) // Validate date range if config.EndDate.Before(config.StartDate) { return fmt.Errorf("end date cannot be before start date") } // Calculate the date range in days daysInRange := int(config.EndDate.Sub(config.StartDate).Hours()/24) + 1 if daysInRange > 365 { return fmt.Errorf("date range too large: maximum 365 days allowed") } // Create scraping job job := &ScrapingJob{ Type: fmt.Sprintf("ted_%s", config.Mode), Status: "running", StartedAt: startTime.Unix(), Progress: JobProgress{}, Results: JobResults{}, Config: map[string]interface{}{ "base_url": s.config.BaseURL, "start_date": config.StartDate.Format("2006-01-02"), "end_date": config.EndDate.Format("2006-01-02"), "mode": string(config.Mode), }, } // Save job if err := s.saveScrapingJob(ctx, job); err != nil { s.logger.Error("Failed to save scraping job", map[string]interface{}{ "error": err.Error(), }) } // Calculate files to download based on date range filesToDownload := s.calculateFilesInDateRange(config.StartDate, config.EndDate) if len(filesToDownload) == 0 { s.logger.Info("No files to download in date range", map[string]interface{}{ "start_date": config.StartDate.Format("2006-01-02"), "end_date": config.EndDate.Format("2006-01-02"), }) // Complete the job s.completeJob(ctx, job, nil, nil) return nil } // Update job progress job.Progress.TotalFiles = len(filesToDownload) s.saveScrapingJob(ctx, job) var lastSuccessfulFile *FileInfo // Download and process files for i, fileInfo := range filesToDownload { job.Progress.CurrentFile = fmt.Sprintf("%d%05d", fileInfo.Year, fileInfo.Number) s.saveScrapingJob(ctx, job) fileResult, err := s.downloadAndProcess(ctx, fileInfo) if err != nil { s.logger.Error("Failed to download and process file", map[string]interface{}{ "year": fileInfo.Year, "number": fileInfo.Number, "error": err.Error(), }) job.Progress.FailedFiles++ job.Results.ErrorCount++ } else { job.Progress.ProcessedFiles++ job.Results.TotalTenders += fileResult.ProcessedCount job.Results.NewTenders += fileResult.NewTenders job.Results.UpdatedTenders += fileResult.UpdatedTenders if len(fileResult.Errors) > 0 { job.Results.ErrorCount += len(fileResult.Errors) } lastSuccessfulFile = &fileInfo } // Update progress percentage job.Progress.PercentComplete = float64(i+1) / float64(len(filesToDownload)) * 100 s.saveScrapingJob(ctx, job) // Check context for cancellation select { case <-ctx.Done(): s.completeJob(ctx, job, nil, lastSuccessfulFile) return ctx.Err() default: // Continue processing } } // Complete the job s.completeJob(ctx, job, nil, lastSuccessfulFile) s.logger.Info("TED XML scraping by date range completed", map[string]interface{}{ "files_processed": job.Progress.ProcessedFiles, "tenders_created": job.Results.NewTenders, "tenders_updated": job.Results.UpdatedTenders, "error_count": job.Results.ErrorCount, "success_rate": job.Results.SuccessRate, }) return nil } // calculateFilesInDateRange calculates which files need to be downloaded for a date range func (s *TEDScraper) calculateFilesInDateRange(startDate, endDate time.Time) []FileInfo { var files []FileInfo // TED files are published roughly once per working day // We estimate file numbers based on working days from start of year for year := startDate.Year(); year <= endDate.Year(); year++ { yearStart := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC) yearEnd := time.Date(year, 12, 31, 23, 59, 59, 0, time.UTC) // Determine the effective start and end dates for this year effectiveStart := startDate if yearStart.After(startDate) { effectiveStart = yearStart } effectiveEnd := endDate if yearEnd.Before(endDate) { effectiveEnd = yearEnd } // Skip if no overlap with this year if effectiveStart.After(effectiveEnd) { continue } // Calculate approximate file range for this year startDayOfYear := effectiveStart.YearDay() endDayOfYear := effectiveEnd.YearDay() // Estimate file numbers (roughly 0.7 files per day, accounting for weekends) startFileNum := int(float64(startDayOfYear) * 0.7) endFileNum := int(float64(endDayOfYear) * 0.7) // Ensure minimum range if startFileNum < 1 { startFileNum = 1 } if endFileNum < startFileNum { endFileNum = startFileNum } // Add some buffer startFileNum = max(1, startFileNum-2) endFileNum = min(366, endFileNum+2) // Max ~366 files per year s.logger.Info("Calculating file range for year", map[string]interface{}{ "year": year, "start_file_num": startFileNum, "end_file_num": endFileNum, "start_date": effectiveStart.Format("2006-01-02"), "end_date": effectiveEnd.Format("2006-01-02"), }) for number := startFileNum; number <= endFileNum; number++ { fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number) ojsID := fmt.Sprintf("S %03d / %d", number, year) files = append(files, FileInfo{ Year: year, Number: number, URL: fileURL, OJS: ojsID, }) } } s.logger.Info("Calculated files for date range", map[string]interface{}{ "total_files": len(files), "start_date": startDate.Format("2006-01-02"), "end_date": endDate.Format("2006-01-02"), }) return files } // Helper function for max func max(a, b int) int { if a > b { return a } return b } // Helper function for min func min(a, b int) int { if a < b { return a } return b } // applyAwardLogic applies award-specific logic based on document type and content func (s *TEDScraper) applyAwardLogic(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) { if tenderEntity == nil || parsedDoc == nil { return } // Only set award information if the tender has a confirmed winner hasConfirmedWinner := s.hasConfirmedWinner(parsedDoc) s.logger.Debug("Applying award logic", map[string]interface{}{ "document_type": string(parsedDoc.Type), "status": string(parsedDoc.Status), "has_confirmed_winner": hasConfirmedWinner, }) if !hasConfirmedWinner { // Clear any award date and winner information for tenders without confirmed winners tenderEntity.AwardDate = 0 tenderEntity.AwardedValue = 0 tenderEntity.WinningTenderer = nil tenderEntity.AwardedEntities = nil tenderEntity.ContractNumber = "" s.logger.Debug("Cleared award information for tender without confirmed winner", map[string]interface{}{ "contract_notice_id": tenderEntity.ContractNoticeID, }) return } // Process award information for tenders with confirmed winners s.logger.Info("Processing award information for tender with confirmed winner", map[string]interface{}{ "contract_notice_id": tenderEntity.ContractNoticeID, "document_type": string(parsedDoc.Type), }) } // hasConfirmedWinner checks if the tender has a confirmed winner based on the document content func (s *TEDScraper) hasConfirmedWinner(parsedDoc *ParsedDocument) bool { if parsedDoc == nil { return false } // Check for award-specific document types switch parsedDoc.Type { case DocumentTypeContractAwardNotice, DocumentTypeResultNotice: // These document types typically indicate awarded tenders if parsedDoc.Status == NoticeStatusAwarded { return true } // Check for specific award indicators in the document document := parsedDoc.GetDocument() if document != nil { return s.checkForAwardIndicators(document, parsedDoc.Type) } case DocumentTypeContractNotice: // Contract notices may have award information if they're updated with results if parsedDoc.Status == NoticeStatusAwarded { if cn := parsedDoc.ContractNotice; cn != nil { return s.checkForAwardIndicators(cn, parsedDoc.Type) } } } return false } // checkForAwardIndicators checks for specific award indicators in the document func (s *TEDScraper) checkForAwardIndicators(document TEDDocument, docType DocumentType) bool { switch docType { case DocumentTypeContractAwardNotice: if can, ok := document.(*ContractAwardNotice); ok { // Check for tender result with award date if can.TenderResult != nil && can.TenderResult.AwardDate != "" { return true } // Check for awarded contractors awardedContractors := can.GetAwardedContractors() if len(awardedContractors) > 0 { return true } // Check for award information in extensions if award := can.GetAwardInformation(); award != nil && award.AwardDate != "" { return true } } case DocumentTypeContractNotice: if cn, ok := document.(*ContractNotice); ok { // Check for award information if award := cn.GetAwardInformation(); award != nil && award.AwardDate != "" { return true } } case DocumentTypeResultNotice: // Result notices typically contain award information return true } return false } // mergeExistingTenderData merges new data with existing tender data, preserving important existing information func (s *TEDScraper) mergeExistingTenderData(existing *tender.Tender, updated *tender.Tender) { // Preserve creation timestamp and original metadata updated.CreatedAt = existing.CreatedAt // Merge processing metadata if existing.ProcessingMetadata.ScrapedAt != 0 { // Keep original scraped time, update processed time updated.ProcessingMetadata.ScrapedAt = existing.ProcessingMetadata.ScrapedAt } // Preserve existing award information if the new document doesn't have confirmed winners // but existing data does (to avoid losing award data on subsequent updates) if existing.AwardDate != 0 && updated.AwardDate == 0 && len(existing.AwardedEntities) > 0 { s.logger.Info("Preserving existing award information", map[string]interface{}{ "contract_notice_id": existing.ContractNoticeID, "existing_award_date": existing.AwardDate, "awarded_entities_count": len(existing.AwardedEntities), }) updated.AwardDate = existing.AwardDate updated.AwardedValue = existing.AwardedValue updated.WinningTenderer = existing.WinningTenderer updated.AwardedEntities = existing.AwardedEntities updated.ContractNumber = existing.ContractNumber } // Merge modifications (append new modifications to existing ones) if len(existing.Modifications) > 0 { // Create a map to avoid duplicate modifications modMap := make(map[string]tender.TenderModification) // Add existing modifications for _, mod := range existing.Modifications { key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason) modMap[key] = mod } // Add new modifications for _, mod := range updated.Modifications { key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason) modMap[key] = mod } // Convert back to slice updated.Modifications = make([]tender.TenderModification, 0, len(modMap)) for _, mod := range modMap { updated.Modifications = append(updated.Modifications, mod) } } }