Enhance Tender Entity and TED Parser with Status Management and Upsert Logic
- Added new fields to the Tender entity for handling cancellation, award, and suspension details, improving the entity's capability to manage tender statuses. - Implemented a DetectNoticeStatus method in the TEDParser to identify the status of TED notices from XML data, enhancing the parser's functionality. - Updated the ParseXML method to include notice status detection, ensuring accurate status representation in parsed documents. - Introduced upsert logic in the TED scraper to handle tender records more effectively, allowing for updates or creation based on existing ContractIDs. - Enhanced logging and error handling throughout the scraping process to improve traceability and robustness. - Added a comprehensive usage example document to illustrate the new features and usage patterns for the TED XML parser and scraper.
This commit is contained in:
n.nakhostin
+706
-36
@@ -64,11 +64,11 @@ type JobResults struct {
|
||||
// ScrapingState represents the current state of scraping
|
||||
type ScrapingState struct {
|
||||
mongo.Model
|
||||
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
|
||||
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
|
||||
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"`
|
||||
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"`
|
||||
IsRunning bool `bson:"is_running" json:"is_running"`
|
||||
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
|
||||
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
|
||||
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"`
|
||||
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"`
|
||||
IsRunning bool `bson:"is_running" json:"is_running"`
|
||||
CurrentJobID bson.ObjectID `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
|
||||
}
|
||||
|
||||
@@ -85,6 +85,22 @@ type Config struct {
|
||||
ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
|
||||
}
|
||||
|
||||
// ScrapingMode represents the type of scraping operation
|
||||
type ScrapingMode string
|
||||
|
||||
const (
|
||||
ScrapingModeDaily ScrapingMode = "daily" // Daily bulk fetch mode
|
||||
ScrapingModeManual ScrapingMode = "manual" // Manual date range mode
|
||||
ScrapingModeRealtime ScrapingMode = "realtime" // Real-time processing mode
|
||||
)
|
||||
|
||||
// DateRangeConfig holds configuration for manual date range scraping
|
||||
type DateRangeConfig struct {
|
||||
StartDate time.Time `json:"start_date"`
|
||||
EndDate time.Time `json:"end_date"`
|
||||
Mode ScrapingMode `json:"mode"`
|
||||
}
|
||||
|
||||
// NewTEDScraper creates a new TED scraper instance
|
||||
func NewTEDScraper(
|
||||
config *Config,
|
||||
@@ -245,7 +261,8 @@ func (s *TEDScraper) ScrapeLatest(ctx context.Context) error {
|
||||
} else {
|
||||
job.Progress.ProcessedFiles++
|
||||
job.Results.TotalTenders += fileResult.ProcessedCount
|
||||
job.Results.NewTenders += fileResult.SuccessCount
|
||||
job.Results.NewTenders += fileResult.NewTenders
|
||||
job.Results.UpdatedTenders += fileResult.UpdatedTenders
|
||||
if len(fileResult.Errors) > 0 {
|
||||
job.Results.ErrorCount += len(fileResult.Errors)
|
||||
}
|
||||
@@ -272,6 +289,8 @@ func (s *TEDScraper) ScrapeLatest(ctx context.Context) error {
|
||||
s.logger.Info("TED XML scraping completed", map[string]interface{}{
|
||||
"files_processed": job.Progress.ProcessedFiles,
|
||||
"tenders_created": job.Results.NewTenders,
|
||||
"tenders_updated": job.Results.UpdatedTenders,
|
||||
"total_tenders": job.Results.TotalTenders,
|
||||
"error_count": job.Results.ErrorCount,
|
||||
"success_rate": job.Results.SuccessRate,
|
||||
})
|
||||
@@ -347,6 +366,8 @@ type FileProcessingResult struct {
|
||||
Number int `json:"number"`
|
||||
ProcessedCount int `json:"processed_count"`
|
||||
SuccessCount int `json:"success_count"`
|
||||
NewTenders int `json:"new_tenders"`
|
||||
UpdatedTenders int `json:"updated_tenders"`
|
||||
Errors []string `json:"errors"`
|
||||
ProcessedAt int64 `json:"processed_at"`
|
||||
}
|
||||
@@ -527,9 +548,17 @@ func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fil
|
||||
}
|
||||
|
||||
// Parse and process XML content
|
||||
if err := s.processXMLContent(ctx, content, fileName, fileInfo, result); err != nil {
|
||||
processResult, err := s.processXMLContent(ctx, content, fileName, fileInfo)
|
||||
if err != nil {
|
||||
errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err)
|
||||
result.Errors = append(result.Errors, errMsg)
|
||||
} else {
|
||||
result.SuccessCount++
|
||||
if processResult.IsNew {
|
||||
result.NewTenders++
|
||||
} else {
|
||||
result.UpdatedTenders++
|
||||
}
|
||||
}
|
||||
|
||||
result.ProcessedCount++
|
||||
@@ -556,8 +585,15 @@ func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fil
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// XMLProcessResult represents the result of processing a single XML file
|
||||
type XMLProcessResult struct {
|
||||
IsNew bool `json:"is_new"`
|
||||
TenderID string `json:"tender_id"`
|
||||
ContractID string `json:"contract_id"`
|
||||
}
|
||||
|
||||
// processXMLContent parses XML content and creates/updates tender
|
||||
func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo, result *FileProcessingResult) error {
|
||||
func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo) (*XMLProcessResult, error) {
|
||||
s.logger.Info("Starting XML processing", map[string]interface{}{
|
||||
"file_name": fileName,
|
||||
"file_size": len(content),
|
||||
@@ -570,7 +606,7 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
|
||||
"file_name": fileName,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Errorf("failed to parse XML: %w", err)
|
||||
return nil, fmt.Errorf("failed to parse XML: %w", err)
|
||||
}
|
||||
|
||||
// Get the underlying document for ID and validation
|
||||
@@ -612,33 +648,45 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
|
||||
sourceURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, fileInfo.Year, fileInfo.Number)
|
||||
tenderEntity := s.mapToTenderFromParsedDoc(parsedDoc, sourceURL, fileName)
|
||||
|
||||
// Handle status-specific information and award logic
|
||||
s.applyStatusSpecificInformation(tenderEntity, parsedDoc)
|
||||
s.applyAwardLogic(tenderEntity, parsedDoc)
|
||||
|
||||
s.logger.Info("Mapped to tender entity", map[string]interface{}{
|
||||
"file_name": fileName,
|
||||
"tender_id": tenderEntity.TenderID,
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
"title": tenderEntity.Title,
|
||||
"status": string(tenderEntity.Status),
|
||||
})
|
||||
|
||||
// Try to find existing tender by contract notice ID
|
||||
// Try to find existing tender by contract notice ID (ContractID)
|
||||
s.logger.Info("Checking for existing tender", map[string]interface{}{
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
})
|
||||
|
||||
existingTender, err := s.findTenderByContractNoticeID(ctx, tenderEntity.ContractNoticeID)
|
||||
if err != nil {
|
||||
if err != nil && err.Error() != "document not found" {
|
||||
s.logger.Warn("Error checking for existing tender", map[string]interface{}{
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
var processResult *XMLProcessResult
|
||||
|
||||
if err == nil && existingTender != nil {
|
||||
// Update existing tender
|
||||
s.logger.Info("Found existing tender, updating", map[string]interface{}{
|
||||
"existing_tender_id": existingTender.ID,
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
"old_status": string(existingTender.Status),
|
||||
"new_status": string(tenderEntity.Status),
|
||||
})
|
||||
|
||||
// Merge updates with existing data
|
||||
s.mergeExistingTenderData(existingTender, tenderEntity)
|
||||
|
||||
tenderEntity.ID = existingTender.ID
|
||||
tenderEntity.CreatedAt = existingTender.CreatedAt
|
||||
tenderEntity.UpdatedAt = time.Now().Unix()
|
||||
@@ -649,17 +697,24 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
|
||||
"tender_id": tenderEntity.ID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Errorf("failed to update existing tender: %w", err)
|
||||
return nil, fmt.Errorf("failed to update existing tender: %w", err)
|
||||
}
|
||||
|
||||
s.logger.Info("Successfully updated tender", map[string]interface{}{
|
||||
"tender_id": tenderEntity.ID,
|
||||
})
|
||||
|
||||
processResult = &XMLProcessResult{
|
||||
IsNew: false,
|
||||
TenderID: tenderEntity.TenderID,
|
||||
ContractID: tenderEntity.ContractNoticeID,
|
||||
}
|
||||
} else {
|
||||
// Create new tender
|
||||
s.logger.Info("Creating new tender", map[string]interface{}{
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
"tender_id": tenderEntity.TenderID,
|
||||
"status": string(tenderEntity.Status),
|
||||
})
|
||||
|
||||
tenderEntity.CreatedAt = time.Now().Unix()
|
||||
@@ -672,25 +727,30 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
|
||||
"tender_id": tenderEntity.TenderID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Errorf("failed to create tender: %w", err)
|
||||
return nil, fmt.Errorf("failed to create tender: %w", err)
|
||||
}
|
||||
|
||||
s.logger.Info("Successfully created new tender", map[string]interface{}{
|
||||
"tender_id": tenderEntity.TenderID,
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
})
|
||||
}
|
||||
|
||||
result.SuccessCount++
|
||||
processResult = &XMLProcessResult{
|
||||
IsNew: true,
|
||||
TenderID: tenderEntity.TenderID,
|
||||
ContractID: tenderEntity.ContractNoticeID,
|
||||
}
|
||||
}
|
||||
|
||||
s.logger.Debug("Successfully processed XML file", map[string]interface{}{
|
||||
"file_name": fileName,
|
||||
"document_type": string(parsedDoc.Type),
|
||||
"document_id": documentID,
|
||||
"tender_id": tenderEntity.TenderID,
|
||||
"is_new": processResult.IsNew,
|
||||
})
|
||||
|
||||
return nil
|
||||
return processResult, nil
|
||||
}
|
||||
|
||||
// Helper methods for database operations
|
||||
@@ -940,8 +1000,8 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
|
||||
NoticeTypeCode: cn.GetNoticeSubType(),
|
||||
NoticeSubTypeCode: cn.GetNoticeSubType(),
|
||||
NoticeLanguageCode: cn.NoticeLanguageCode,
|
||||
IssueDate: s.parseDateToUnixMilli(cn.IssueDate),
|
||||
IssueTime: s.parseTimeToUnixMilli(cn.IssueTime),
|
||||
IssueDate: s.parseDateToUnix(cn.IssueDate),
|
||||
IssueTime: s.parseTimeToUnix(cn.IssueTime),
|
||||
GazetteID: cn.GetOJSID(),
|
||||
Title: cn.GetProcurementTitle(),
|
||||
Description: cn.GetProcurementDescription(),
|
||||
@@ -964,7 +1024,7 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
|
||||
// Set publication information
|
||||
if pubInfo := cn.GetPublicationInfo(); pubInfo != nil {
|
||||
t.NoticePublicationID = pubInfo.NoticePublicationID
|
||||
t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate)
|
||||
t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate)
|
||||
t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
|
||||
}
|
||||
|
||||
@@ -993,17 +1053,17 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
|
||||
|
||||
// Set tender deadline
|
||||
if deadline := cn.GetTenderDeadline(); deadline != "" {
|
||||
t.TenderDeadline = s.parseDateToUnixMilli(deadline)
|
||||
t.TenderDeadline = s.parseDateToUnix(deadline)
|
||||
|
||||
if deadlineTime, err := s.parseDate(deadline); err == nil {
|
||||
applicationDeadline := s.calculateApplicationDeadline(deadlineTime)
|
||||
t.ApplicationDeadline = applicationDeadline.UnixMilli()
|
||||
t.ApplicationDeadline = applicationDeadline.Unix()
|
||||
}
|
||||
}
|
||||
|
||||
if pubDate, err := s.parseDate(cn.GetPublicationDate()); err == nil {
|
||||
submissionDeadline := s.calculateSubmissionDeadline(pubDate)
|
||||
t.SubmissionDeadline = submissionDeadline.UnixMilli()
|
||||
t.SubmissionDeadline = submissionDeadline.Unix()
|
||||
}
|
||||
|
||||
// Set SubmissionURL from TenderingTerms
|
||||
@@ -1089,8 +1149,8 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
|
||||
NoticeTypeCode: can.NoticeTypeCode,
|
||||
NoticeSubTypeCode: can.GetNoticeSubType(),
|
||||
NoticeLanguageCode: can.NoticeLanguageCode,
|
||||
IssueDate: s.parseDateToUnixMilli(can.IssueDate),
|
||||
IssueTime: s.parseTimeToUnixMilli(can.IssueTime),
|
||||
IssueDate: s.parseDateToUnix(can.IssueDate),
|
||||
IssueTime: s.parseTimeToUnix(can.IssueTime),
|
||||
GazetteID: can.GetOJSID(),
|
||||
Title: can.GetProcurementTitle(),
|
||||
Description: can.GetProcurementDescription(),
|
||||
@@ -1111,7 +1171,7 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
|
||||
// Set publication information
|
||||
if pubInfo := can.GetPublicationInfo(); pubInfo != nil {
|
||||
t.NoticePublicationID = pubInfo.NoticePublicationID
|
||||
t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate)
|
||||
t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate)
|
||||
t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
|
||||
}
|
||||
|
||||
@@ -1234,7 +1294,7 @@ func (s *TEDScraper) generateTenderID(t *tender.Tender) string {
|
||||
// parts = append(parts, buyerCode)
|
||||
|
||||
// Date (YYYYMMDD format from issue date)
|
||||
parts = append(parts, s.padOrTruncate(s.unixMilliToDateString(t.IssueDate), 4))
|
||||
parts = append(parts, s.padOrTruncate(s.UnixToDateString(t.IssueDate), 4))
|
||||
|
||||
// Location code (country code + region code, padded to 6 chars)
|
||||
locationCode := "EU"
|
||||
@@ -1310,21 +1370,21 @@ func (s *TEDScraper) parseDate(dateStr string) (time.Time, error) {
|
||||
return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr)
|
||||
}
|
||||
|
||||
// parseDateToUnixMilli converts string date to Unix milliseconds (int64)
|
||||
func (s *TEDScraper) parseDateToUnixMilli(dateStr string) int64 {
|
||||
// parseDateToUnix converts string date to Unix milliseconds (int64)
|
||||
func (s *TEDScraper) parseDateToUnix(dateStr string) int64 {
|
||||
if dateStr == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
if parsedTime, err := s.parseDate(dateStr); err == nil {
|
||||
return parsedTime.UnixMilli()
|
||||
return parsedTime.Unix()
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// parseTimeToUnixMilli converts string time to Unix milliseconds (int64)
|
||||
func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 {
|
||||
// parseTimeToUnix converts string time to Unix milliseconds (int64)
|
||||
func (s *TEDScraper) parseTimeToUnix(timeStr string) int64 {
|
||||
if timeStr == "" {
|
||||
return 0
|
||||
}
|
||||
@@ -1337,19 +1397,19 @@ func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 {
|
||||
}
|
||||
|
||||
if parsedTime, err := s.parseDate(timeStr); err == nil {
|
||||
return parsedTime.UnixMilli()
|
||||
return parsedTime.Unix()
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// unixMilliToDateString converts Unix milliseconds to date string for TenderID generation
|
||||
func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string {
|
||||
if unixMilli == 0 {
|
||||
// UnixToDateString converts Unix milliseconds to date string for TenderID generation
|
||||
func (s *TEDScraper) UnixToDateString(u int64) string {
|
||||
if u == 0 {
|
||||
return "00000000"
|
||||
}
|
||||
|
||||
return time.UnixMilli(unixMilli).Format("20060102")
|
||||
return time.Unix(u, 0).Format("20060102")
|
||||
}
|
||||
|
||||
func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time {
|
||||
@@ -1395,3 +1455,613 @@ func (s *TEDScraper) adjustToWorkingDay(targetDate time.Time, forward bool) time
|
||||
|
||||
return current
|
||||
}
|
||||
|
||||
// mapTEDStatusToTenderStatus maps TED notice status to tender status
|
||||
func (s *TEDScraper) mapTEDStatusToTenderStatus(tedStatus NoticeStatus) tender.TenderStatus {
|
||||
switch tedStatus {
|
||||
case NoticeStatusDraft:
|
||||
return tender.TenderStatusDraft
|
||||
case NoticeStatusPublished:
|
||||
return tender.TenderStatusPublished
|
||||
case NoticeStatusCancelled:
|
||||
return tender.TenderStatusCancelled
|
||||
case NoticeStatusAwarded:
|
||||
return tender.TenderStatusAwarded
|
||||
case NoticeStatusClosed:
|
||||
return tender.TenderStatusClosed
|
||||
case NoticeStatusModified:
|
||||
return tender.TenderStatusModified
|
||||
case NoticeStatusSuspended:
|
||||
return tender.TenderStatusSuspended
|
||||
default:
|
||||
// Default to active for unknown statuses, but log a warning
|
||||
s.logger.Warn("Unknown TED notice status, defaulting to active", map[string]interface{}{
|
||||
"ted_status": string(tedStatus),
|
||||
})
|
||||
return tender.TenderStatusActive
|
||||
}
|
||||
}
|
||||
|
||||
// applyStatusSpecificInformation applies status-specific information to the tender entity
|
||||
func (s *TEDScraper) applyStatusSpecificInformation(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) {
|
||||
if tenderEntity == nil || parsedDoc == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Map TED status to tender status
|
||||
tenderEntity.Status = s.mapTEDStatusToTenderStatus(parsedDoc.Status)
|
||||
|
||||
// Get the document to access status-specific methods
|
||||
document := parsedDoc.GetDocument()
|
||||
if document == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Apply status-specific information based on document type
|
||||
switch parsedDoc.Type {
|
||||
case DocumentTypeContractNotice:
|
||||
if cn, ok := document.(*ContractNotice); ok {
|
||||
s.applyContractNoticeStatusInfo(tenderEntity, cn, parsedDoc.Status)
|
||||
}
|
||||
case DocumentTypeContractAwardNotice:
|
||||
if can, ok := document.(*ContractAwardNotice); ok {
|
||||
s.applyContractAwardNoticeStatusInfo(tenderEntity, can, parsedDoc.Status)
|
||||
}
|
||||
default:
|
||||
// For other document types, we rely on the parsed status
|
||||
s.logger.Debug("Applied basic status information for document type", map[string]interface{}{
|
||||
"document_type": string(parsedDoc.Type),
|
||||
"status": string(parsedDoc.Status),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// applyContractNoticeStatusInfo applies status-specific information from ContractNotice
|
||||
func (s *TEDScraper) applyContractNoticeStatusInfo(tenderEntity *tender.Tender, cn *ContractNotice, status NoticeStatus) {
|
||||
switch status {
|
||||
case NoticeStatusCancelled:
|
||||
if cancellation := cn.GetCancellationReason(); cancellation != nil {
|
||||
tenderEntity.CancellationReason = cancellation.Description
|
||||
if cancellation.ReasonCode != "" {
|
||||
tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description)
|
||||
}
|
||||
// Note: TED XML doesn't typically have cancellation date, so we use current time
|
||||
tenderEntity.CancellationDate = time.Now().Unix()
|
||||
}
|
||||
|
||||
case NoticeStatusAwarded:
|
||||
if award := cn.GetAwardInformation(); award != nil {
|
||||
if award.AwardDate != "" {
|
||||
tenderEntity.AwardDate = s.parseDateToUnix(award.AwardDate)
|
||||
}
|
||||
if award.AwardedValue.Text != "" {
|
||||
if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil {
|
||||
tenderEntity.AwardedValue = value
|
||||
}
|
||||
}
|
||||
tenderEntity.ContractNumber = award.ContractNumber
|
||||
if award.WinningTenderer != nil && award.WinningTenderer.Company != nil {
|
||||
tenderEntity.WinningTenderer = s.mapOrganization(&Organization{
|
||||
Company: award.WinningTenderer.Company,
|
||||
}, "winner")
|
||||
}
|
||||
}
|
||||
|
||||
case NoticeStatusSuspended:
|
||||
if suspension := cn.GetSuspensionReason(); suspension != nil {
|
||||
tenderEntity.SuspensionReason = suspension.Description
|
||||
if suspension.ReasonCode != "" {
|
||||
tenderEntity.SuspensionReason = fmt.Sprintf("%s: %s", suspension.ReasonCode, suspension.Description)
|
||||
}
|
||||
tenderEntity.SuspensionDate = time.Now().Unix()
|
||||
}
|
||||
|
||||
case NoticeStatusModified:
|
||||
if modifications := cn.GetModifications(); modifications != nil {
|
||||
for _, mod := range modifications {
|
||||
tenderMod := tender.TenderModification{
|
||||
ModificationDate: s.parseDateToUnix(mod.ModificationDate),
|
||||
ModificationReason: mod.ModificationReason,
|
||||
Description: mod.Description,
|
||||
LanguageID: mod.LanguageID,
|
||||
}
|
||||
tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// applyContractAwardNoticeStatusInfo applies status-specific information from ContractAwardNotice
|
||||
func (s *TEDScraper) applyContractAwardNoticeStatusInfo(tenderEntity *tender.Tender, can *ContractAwardNotice, status NoticeStatus) {
|
||||
// Contract award notices are typically awarded status
|
||||
tenderEntity.Status = tender.TenderStatusAwarded
|
||||
|
||||
// Set award date from tender result
|
||||
if can.TenderResult != nil && can.TenderResult.AwardDate != "" {
|
||||
tenderEntity.AwardDate = s.parseDateToUnix(can.TenderResult.AwardDate)
|
||||
}
|
||||
|
||||
// Get award information from extensions if available
|
||||
if award := can.GetAwardInformation(); award != nil {
|
||||
if award.AwardedValue.Text != "" {
|
||||
if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil {
|
||||
tenderEntity.AwardedValue = value
|
||||
}
|
||||
}
|
||||
tenderEntity.ContractNumber = award.ContractNumber
|
||||
}
|
||||
|
||||
// Get total award value from notice result
|
||||
if amount, currency := can.GetTotalAwardValue(); amount != "" {
|
||||
if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
|
||||
tenderEntity.AwardedValue = parsedAmount
|
||||
tenderEntity.Currency = currency
|
||||
}
|
||||
}
|
||||
|
||||
// Extract detailed awarded contractors information
|
||||
s.extractAwardedEntities(tenderEntity, can)
|
||||
|
||||
// Handle other statuses if applicable
|
||||
switch status {
|
||||
case NoticeStatusCancelled:
|
||||
if cancellation := can.GetCancellationReason(); cancellation != nil {
|
||||
tenderEntity.Status = tender.TenderStatusCancelled
|
||||
tenderEntity.CancellationReason = cancellation.Description
|
||||
if cancellation.ReasonCode != "" {
|
||||
tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description)
|
||||
}
|
||||
tenderEntity.CancellationDate = time.Now().Unix()
|
||||
}
|
||||
|
||||
case NoticeStatusModified:
|
||||
if modifications := can.GetModifications(); modifications != nil {
|
||||
tenderEntity.Status = tender.TenderStatusModified
|
||||
for _, mod := range modifications {
|
||||
tenderMod := tender.TenderModification{
|
||||
ModificationDate: s.parseDateToUnix(mod.ModificationDate),
|
||||
ModificationReason: mod.ModificationReason,
|
||||
Description: mod.Description,
|
||||
LanguageID: mod.LanguageID,
|
||||
}
|
||||
tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// extractAwardedEntities extracts detailed information about awarded entities from a contract award notice
|
||||
func (s *TEDScraper) extractAwardedEntities(tenderEntity *tender.Tender, can *ContractAwardNotice) {
|
||||
awardedContractors := can.GetAwardedContractors()
|
||||
if len(awardedContractors) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
s.logger.Info("Extracting awarded entities", map[string]interface{}{
|
||||
"contract_notice_id": can.ID,
|
||||
"num_contractors": len(awardedContractors),
|
||||
})
|
||||
|
||||
for _, contractor := range awardedContractors {
|
||||
entity := tender.AwardedEntity{
|
||||
Name: contractor.Name,
|
||||
Address: contractor.Address,
|
||||
Country: contractor.Country,
|
||||
Currency: contractor.Currency,
|
||||
ContractID: contractor.ContractID,
|
||||
TenderID: contractor.TenderID,
|
||||
LotID: contractor.LotID,
|
||||
CompanyID: contractor.CompanyID,
|
||||
OrganizationID: contractor.OrganizationID,
|
||||
}
|
||||
|
||||
// Parse amount
|
||||
if contractor.Amount != "" {
|
||||
if amount, err := strconv.ParseFloat(contractor.Amount, 64); err == nil {
|
||||
entity.Amount = amount
|
||||
} else {
|
||||
s.logger.Warn("Failed to parse award amount", map[string]interface{}{
|
||||
"amount_string": contractor.Amount,
|
||||
"contractor": contractor.Name,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Parse award date
|
||||
if contractor.AwardDate != "" {
|
||||
entity.AwardDate = s.parseDateToUnix(contractor.AwardDate)
|
||||
} else if contractor.IssueDate != "" {
|
||||
entity.AwardDate = s.parseDateToUnix(contractor.IssueDate)
|
||||
}
|
||||
|
||||
// Calculate share percentage if there are multiple contractors
|
||||
totalAmount := tenderEntity.AwardedValue
|
||||
if totalAmount > 0 && entity.Amount > 0 {
|
||||
entity.Share = (entity.Amount / totalAmount) * 100
|
||||
}
|
||||
|
||||
tenderEntity.AwardedEntities = append(tenderEntity.AwardedEntities, entity)
|
||||
|
||||
s.logger.Debug("Added awarded entity", map[string]interface{}{
|
||||
"name": entity.Name,
|
||||
"amount": entity.Amount,
|
||||
"currency": entity.Currency,
|
||||
"share": entity.Share,
|
||||
"contract_id": entity.ContractID,
|
||||
})
|
||||
}
|
||||
|
||||
s.logger.Info("Successfully extracted awarded entities", map[string]interface{}{
|
||||
"total_entities": len(tenderEntity.AwardedEntities),
|
||||
"total_value": tenderEntity.AwardedValue,
|
||||
"currency": tenderEntity.Currency,
|
||||
})
|
||||
}
|
||||
|
||||
// ScrapeByDateRange scrapes TED XML files within a specified date range
|
||||
func (s *TEDScraper) ScrapeByDateRange(ctx context.Context, config DateRangeConfig) error {
|
||||
startTime := time.Now()
|
||||
|
||||
s.logger.Info("Starting TED XML scraping by date range", map[string]interface{}{
|
||||
"start_date": config.StartDate.Format("2006-01-02"),
|
||||
"end_date": config.EndDate.Format("2006-01-02"),
|
||||
"mode": string(config.Mode),
|
||||
"timestamp": startTime.Unix(),
|
||||
})
|
||||
|
||||
// Validate date range
|
||||
if config.EndDate.Before(config.StartDate) {
|
||||
return fmt.Errorf("end date cannot be before start date")
|
||||
}
|
||||
|
||||
// Calculate the date range in days
|
||||
daysInRange := int(config.EndDate.Sub(config.StartDate).Hours()/24) + 1
|
||||
if daysInRange > 365 {
|
||||
return fmt.Errorf("date range too large: maximum 365 days allowed")
|
||||
}
|
||||
|
||||
// Create scraping job
|
||||
job := &ScrapingJob{
|
||||
Type: fmt.Sprintf("ted_%s", config.Mode),
|
||||
Status: "running",
|
||||
StartedAt: startTime.Unix(),
|
||||
Progress: JobProgress{},
|
||||
Results: JobResults{},
|
||||
Config: map[string]interface{}{
|
||||
"base_url": s.config.BaseURL,
|
||||
"start_date": config.StartDate.Format("2006-01-02"),
|
||||
"end_date": config.EndDate.Format("2006-01-02"),
|
||||
"mode": string(config.Mode),
|
||||
},
|
||||
}
|
||||
|
||||
// Save job
|
||||
if err := s.saveScrapingJob(ctx, job); err != nil {
|
||||
s.logger.Error("Failed to save scraping job", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Calculate files to download based on date range
|
||||
filesToDownload := s.calculateFilesInDateRange(config.StartDate, config.EndDate)
|
||||
if len(filesToDownload) == 0 {
|
||||
s.logger.Info("No files to download in date range", map[string]interface{}{
|
||||
"start_date": config.StartDate.Format("2006-01-02"),
|
||||
"end_date": config.EndDate.Format("2006-01-02"),
|
||||
})
|
||||
|
||||
// Complete the job
|
||||
s.completeJob(ctx, job, nil, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update job progress
|
||||
job.Progress.TotalFiles = len(filesToDownload)
|
||||
s.saveScrapingJob(ctx, job)
|
||||
|
||||
var lastSuccessfulFile *FileInfo
|
||||
|
||||
// Download and process files
|
||||
for i, fileInfo := range filesToDownload {
|
||||
job.Progress.CurrentFile = fmt.Sprintf("%d%05d", fileInfo.Year, fileInfo.Number)
|
||||
s.saveScrapingJob(ctx, job)
|
||||
|
||||
fileResult, err := s.downloadAndProcess(ctx, fileInfo)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to download and process file", map[string]interface{}{
|
||||
"year": fileInfo.Year,
|
||||
"number": fileInfo.Number,
|
||||
"error": err.Error(),
|
||||
})
|
||||
job.Progress.FailedFiles++
|
||||
job.Results.ErrorCount++
|
||||
} else {
|
||||
job.Progress.ProcessedFiles++
|
||||
job.Results.TotalTenders += fileResult.ProcessedCount
|
||||
job.Results.NewTenders += fileResult.NewTenders
|
||||
job.Results.UpdatedTenders += fileResult.UpdatedTenders
|
||||
if len(fileResult.Errors) > 0 {
|
||||
job.Results.ErrorCount += len(fileResult.Errors)
|
||||
}
|
||||
lastSuccessfulFile = &fileInfo
|
||||
}
|
||||
|
||||
// Update progress percentage
|
||||
job.Progress.PercentComplete = float64(i+1) / float64(len(filesToDownload)) * 100
|
||||
s.saveScrapingJob(ctx, job)
|
||||
|
||||
// Check context for cancellation
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
s.completeJob(ctx, job, nil, lastSuccessfulFile)
|
||||
return ctx.Err()
|
||||
default:
|
||||
// Continue processing
|
||||
}
|
||||
}
|
||||
|
||||
// Complete the job
|
||||
s.completeJob(ctx, job, nil, lastSuccessfulFile)
|
||||
|
||||
s.logger.Info("TED XML scraping by date range completed", map[string]interface{}{
|
||||
"files_processed": job.Progress.ProcessedFiles,
|
||||
"tenders_created": job.Results.NewTenders,
|
||||
"tenders_updated": job.Results.UpdatedTenders,
|
||||
"error_count": job.Results.ErrorCount,
|
||||
"success_rate": job.Results.SuccessRate,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// calculateFilesInDateRange calculates which files need to be downloaded for a date range
|
||||
func (s *TEDScraper) calculateFilesInDateRange(startDate, endDate time.Time) []FileInfo {
|
||||
var files []FileInfo
|
||||
|
||||
// TED files are published roughly once per working day
|
||||
// We estimate file numbers based on working days from start of year
|
||||
|
||||
for year := startDate.Year(); year <= endDate.Year(); year++ {
|
||||
yearStart := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
yearEnd := time.Date(year, 12, 31, 23, 59, 59, 0, time.UTC)
|
||||
|
||||
// Determine the effective start and end dates for this year
|
||||
effectiveStart := startDate
|
||||
if yearStart.After(startDate) {
|
||||
effectiveStart = yearStart
|
||||
}
|
||||
|
||||
effectiveEnd := endDate
|
||||
if yearEnd.Before(endDate) {
|
||||
effectiveEnd = yearEnd
|
||||
}
|
||||
|
||||
// Skip if no overlap with this year
|
||||
if effectiveStart.After(effectiveEnd) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Calculate approximate file range for this year
|
||||
startDayOfYear := effectiveStart.YearDay()
|
||||
endDayOfYear := effectiveEnd.YearDay()
|
||||
|
||||
// Estimate file numbers (roughly 0.7 files per day, accounting for weekends)
|
||||
startFileNum := int(float64(startDayOfYear) * 0.7)
|
||||
endFileNum := int(float64(endDayOfYear) * 0.7)
|
||||
|
||||
// Ensure minimum range
|
||||
if startFileNum < 1 {
|
||||
startFileNum = 1
|
||||
}
|
||||
if endFileNum < startFileNum {
|
||||
endFileNum = startFileNum
|
||||
}
|
||||
|
||||
// Add some buffer
|
||||
startFileNum = max(1, startFileNum-2)
|
||||
endFileNum = min(366, endFileNum+2) // Max ~366 files per year
|
||||
|
||||
s.logger.Info("Calculating file range for year", map[string]interface{}{
|
||||
"year": year,
|
||||
"start_file_num": startFileNum,
|
||||
"end_file_num": endFileNum,
|
||||
"start_date": effectiveStart.Format("2006-01-02"),
|
||||
"end_date": effectiveEnd.Format("2006-01-02"),
|
||||
})
|
||||
|
||||
for number := startFileNum; number <= endFileNum; number++ {
|
||||
fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number)
|
||||
ojsID := fmt.Sprintf("S %03d / %d", number, year)
|
||||
|
||||
files = append(files, FileInfo{
|
||||
Year: year,
|
||||
Number: number,
|
||||
URL: fileURL,
|
||||
OJS: ojsID,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
s.logger.Info("Calculated files for date range", map[string]interface{}{
|
||||
"total_files": len(files),
|
||||
"start_date": startDate.Format("2006-01-02"),
|
||||
"end_date": endDate.Format("2006-01-02"),
|
||||
})
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// Helper function for max
|
||||
func max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Helper function for min
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// applyAwardLogic applies award-specific logic based on document type and content
|
||||
func (s *TEDScraper) applyAwardLogic(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) {
|
||||
if tenderEntity == nil || parsedDoc == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Only set award information if the tender has a confirmed winner
|
||||
hasConfirmedWinner := s.hasConfirmedWinner(parsedDoc)
|
||||
|
||||
s.logger.Debug("Applying award logic", map[string]interface{}{
|
||||
"document_type": string(parsedDoc.Type),
|
||||
"status": string(parsedDoc.Status),
|
||||
"has_confirmed_winner": hasConfirmedWinner,
|
||||
})
|
||||
|
||||
if !hasConfirmedWinner {
|
||||
// Clear any award date and winner information for tenders without confirmed winners
|
||||
tenderEntity.AwardDate = 0
|
||||
tenderEntity.AwardedValue = 0
|
||||
tenderEntity.WinningTenderer = nil
|
||||
tenderEntity.AwardedEntities = nil
|
||||
tenderEntity.ContractNumber = ""
|
||||
|
||||
s.logger.Debug("Cleared award information for tender without confirmed winner", map[string]interface{}{
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Process award information for tenders with confirmed winners
|
||||
s.logger.Info("Processing award information for tender with confirmed winner", map[string]interface{}{
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
"document_type": string(parsedDoc.Type),
|
||||
})
|
||||
}
|
||||
|
||||
// hasConfirmedWinner checks if the tender has a confirmed winner based on the document content
|
||||
func (s *TEDScraper) hasConfirmedWinner(parsedDoc *ParsedDocument) bool {
|
||||
if parsedDoc == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for award-specific document types
|
||||
switch parsedDoc.Type {
|
||||
case DocumentTypeContractAwardNotice, DocumentTypeResultNotice:
|
||||
// These document types typically indicate awarded tenders
|
||||
if parsedDoc.Status == NoticeStatusAwarded {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for specific award indicators in the document
|
||||
document := parsedDoc.GetDocument()
|
||||
if document != nil {
|
||||
return s.checkForAwardIndicators(document, parsedDoc.Type)
|
||||
}
|
||||
|
||||
case DocumentTypeContractNotice:
|
||||
// Contract notices may have award information if they're updated with results
|
||||
if parsedDoc.Status == NoticeStatusAwarded {
|
||||
if cn := parsedDoc.ContractNotice; cn != nil {
|
||||
return s.checkForAwardIndicators(cn, parsedDoc.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// checkForAwardIndicators checks for specific award indicators in the document
|
||||
func (s *TEDScraper) checkForAwardIndicators(document TEDDocument, docType DocumentType) bool {
|
||||
switch docType {
|
||||
case DocumentTypeContractAwardNotice:
|
||||
if can, ok := document.(*ContractAwardNotice); ok {
|
||||
// Check for tender result with award date
|
||||
if can.TenderResult != nil && can.TenderResult.AwardDate != "" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for awarded contractors
|
||||
awardedContractors := can.GetAwardedContractors()
|
||||
if len(awardedContractors) > 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for award information in extensions
|
||||
if award := can.GetAwardInformation(); award != nil && award.AwardDate != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
case DocumentTypeContractNotice:
|
||||
if cn, ok := document.(*ContractNotice); ok {
|
||||
// Check for award information
|
||||
if award := cn.GetAwardInformation(); award != nil && award.AwardDate != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
case DocumentTypeResultNotice:
|
||||
// Result notices typically contain award information
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// mergeExistingTenderData merges new data with existing tender data, preserving important existing information
|
||||
func (s *TEDScraper) mergeExistingTenderData(existing *tender.Tender, updated *tender.Tender) {
|
||||
// Preserve creation timestamp and original metadata
|
||||
updated.CreatedAt = existing.CreatedAt
|
||||
|
||||
// Merge processing metadata
|
||||
if existing.ProcessingMetadata.ScrapedAt != 0 {
|
||||
// Keep original scraped time, update processed time
|
||||
updated.ProcessingMetadata.ScrapedAt = existing.ProcessingMetadata.ScrapedAt
|
||||
}
|
||||
|
||||
// Preserve existing award information if the new document doesn't have confirmed winners
|
||||
// but existing data does (to avoid losing award data on subsequent updates)
|
||||
if existing.AwardDate != 0 && updated.AwardDate == 0 && len(existing.AwardedEntities) > 0 {
|
||||
s.logger.Info("Preserving existing award information", map[string]interface{}{
|
||||
"contract_notice_id": existing.ContractNoticeID,
|
||||
"existing_award_date": existing.AwardDate,
|
||||
"awarded_entities_count": len(existing.AwardedEntities),
|
||||
})
|
||||
|
||||
updated.AwardDate = existing.AwardDate
|
||||
updated.AwardedValue = existing.AwardedValue
|
||||
updated.WinningTenderer = existing.WinningTenderer
|
||||
updated.AwardedEntities = existing.AwardedEntities
|
||||
updated.ContractNumber = existing.ContractNumber
|
||||
}
|
||||
|
||||
// Merge modifications (append new modifications to existing ones)
|
||||
if len(existing.Modifications) > 0 {
|
||||
// Create a map to avoid duplicate modifications
|
||||
modMap := make(map[string]tender.TenderModification)
|
||||
|
||||
// Add existing modifications
|
||||
for _, mod := range existing.Modifications {
|
||||
key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason)
|
||||
modMap[key] = mod
|
||||
}
|
||||
|
||||
// Add new modifications
|
||||
for _, mod := range updated.Modifications {
|
||||
key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason)
|
||||
modMap[key] = mod
|
||||
}
|
||||
|
||||
// Convert back to slice
|
||||
updated.Modifications = make([]tender.TenderModification, 0, len(modMap))
|
||||
for _, mod := range modMap {
|
||||
updated.Modifications = append(updated.Modifications, mod)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user