diff --git a/ted/scraper.go b/ted/scraper.go index 143d3e2..76b011c 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -323,6 +323,24 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file t := s.mapToTenderFromParsedDoc(notice, "", fileName) t.ContentXML = string(content) + // Check if tender already exists to prevent duplicates + existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID) + if err != nil { + s.logger.Error("Failed to check existing tender", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + "error": err.Error(), + }) + return fmt.Errorf("failed to check existing tender: %w", err) + } + + if existingTender != nil { + s.logger.Info("Tender already exists, skipping duplicate", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + "existing_id": existingTender.ID, + }) + return nil + } + // Create new tender s.logger.Info("Creating new tender", map[string]interface{}{ "contract_notice_id": t.ContractNoticeID,