mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance Scraper Dockerfile and Notice Entity

Browse Source 
- Added an entrypoint script to the scraper Dockerfile to handle one-time scraping with date range support.
- Introduced a new ContentXML field in the Notice entity to store XML content, improving data structure for notices.
- Updated the scraper logic to map XML content to the tender entity, enhancing data processing capabilities.
This commit is contained in:
Nima Nakhostin
2025-11-23 13:27:04 +03:30
parent 61a4465f50
commit 1f6d3fd1cb
3 changed files with 14 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ted/scraper.go
+2 -31
View File
@@ -311,38 +311,9 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
// Map to tender entity
t := s.mapToTenderFromParsedDoc(notice, "", fileName)
t.ContentXML = string(content)
// existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID)
// if err != nil {
// s.logger.Warn("Error checking for existing tender", map[string]interface{}{
// "contract_notice_id": t.ContractNoticeID,
// "error": err.Error(),
// })
// }
// if err == nil && existingTender != nil {
// // Update existing tender
// s.logger.Info("Found existing tender, updating", map[string]interface{}{
// "existing_tender_id": existingTender.ID.Hex(),
// "contract_notice_id": t.ContractNoticeID,
// })
// t.ID = existingTender.ID
// t.UpdatedAt = time.Now().Unix()
// err = s.tenderRepo.Update(ctx, t)
// if err != nil {
// s.logger.Error("Failed to update existing tender", map[string]interface{}{
// "tender_id": t.ID.Hex(),
// "error": err.Error(),
// })
// return fmt.Errorf("failed to update existing tender: %w", err)
// }
// s.logger.Info("Successfully updated tender", map[string]interface{}{
// "tender_id": t.ID.Hex(),
// })
// } else {
// // Create new tender
// Create new tender
s.logger.Info("Creating new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
})