package workers import ( "context" "time" "tm/internal/tender" "tm/pkg/logger" "tm/pkg/mongo" "tm/pkg/scraper" ) // DocumentScraperWorker handles document scraping for tenders type DocumentScraperWorker struct { Mongo *mongo.ConnectionManager Logger logger.Logger TenderRepo tender.TenderRepository Scraper scraper.SDK } // NewDocumentScraperWorker creates a new document scraper worker func NewDocumentScraperWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, scraper scraper.SDK) *DocumentScraperWorker { logger.Info("Creating new document scraper worker", map[string]interface{}{}) return &DocumentScraperWorker{ Mongo: mongo, Logger: logger, TenderRepo: tenderRepo, Scraper: scraper, } } // Run starts the document scraping process func (w *DocumentScraperWorker) Run() { w.Logger.Info("Document scraper worker started", map[string]interface{}{}) limit := 10 skip := 0 for { w.Logger.Info("Document scraper worker checking for tenders", map[string]interface{}{ "skip": skip, "limit": limit, }) tenders, totalCount, err := w.TenderRepo.GetUnScrapedTenders(context.Background(), limit, skip) if err != nil { w.Logger.Error("Failed to get un-scraped tenders", map[string]interface{}{ "error": err.Error(), }) continue } w.Logger.Info("Found tenders for document scraping", map[string]interface{}{ "count": len(tenders), "total_count": totalCount, "skip": skip, }) if len(tenders) == 0 { w.Logger.Info("No more tenders to process for document scraping", map[string]interface{}{}) break } w.Logger.Info("Processing tenders for document scraping", map[string]interface{}{ "tender_count": len(tenders), }) for _, t := range tenders { w.Logger.Info("Processing tender for document scraping", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, }) w.scrapeDocumentsForTender(&t) } skip += limit } w.Logger.Info("Document scraper worker completed", map[string]interface{}{}) } // scrapeDocumentsForTender scrapes documents for a single tender func (w *DocumentScraperWorker) scrapeDocumentsForTender(t *tender.Tender) { w.Logger.Info("Starting document scraping for tender", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "document_uri": t.DocumentURI, }) // Skip if no NoticePublicationID if t.NoticePublicationID == "" { w.Logger.Warn("Skipping document scraping - no NoticePublicationID", map[string]interface{}{ "tender_id": t.ID.Hex(), }) return } // Use DocumentURI as NoticeURL for the scraper noticeURL := t.DocumentURI if noticeURL == "" { w.Logger.Warn("Skipping document scraping - no DocumentURI available", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, }) return } // Call scraper SDK scrapeReq := &scraper.ScrapeRequest{ NoticeID: t.NoticePublicationID, // Use NoticePublicationID as notice_id NoticeURL: noticeURL, // Use DocumentURI as notice_url } w.Logger.Info("Starting document scraping for tender", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "document_uri": noticeURL, }) response, err := w.Scraper.ScrapeDocuments(context.Background(), scrapeReq) if err != nil { w.Logger.Error("Failed to scrape documents for tender", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "error": err.Error(), }) // Mark as processed with error (don't retry indefinitely) t.ProcessingMetadata.DocumentsScraped = false t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix() w.updateTenderScrapingStatus(t) return } if !response.Success { w.Logger.Warn("Document scraping failed for tender", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "errors": response.Errors, }) // Mark as processed with failure t.ProcessingMetadata.DocumentsScraped = false t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix() w.updateTenderScrapingStatus(t) return } // Store scraped documents information scrapedAt := time.Now().Unix() t.ScrapedDocuments = make([]tender.ScrapedDocument, len(response.UploadedFiles)) for i, uploadedFile := range response.UploadedFiles { t.ScrapedDocuments[i] = tender.ScrapedDocument{ Filename: uploadedFile.Filename, ObjectName: uploadedFile.ObjectName, BucketName: "opplens", Size: uploadedFile.Size, ScrapedAt: scrapedAt, } } // Mark as successfully scraped t.ProcessingMetadata.DocumentsScraped = true t.ProcessingMetadata.DocumentsScrapedAt = scrapedAt err = w.updateTenderScrapingStatus(t) if err != nil { w.Logger.Error("Failed to update tender scraping status", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "error": err.Error(), }) return } w.Logger.Info("Document scraping completed successfully for tender", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "uploaded_count": response.UploadedCount, "bucket_count": len(response.Buckets), }) } // updateTenderScrapingStatus updates the tender's document scraping metadata func (w *DocumentScraperWorker) updateTenderScrapingStatus(t *tender.Tender) error { err := w.TenderRepo.Update(context.Background(), t) if err != nil { w.Logger.Error("Failed to update tender scraping status", map[string]interface{}{ "tender_id": t.ID.Hex(), "error": err.Error(), }) return err } w.Logger.Debug("Updated tender scraping status", map[string]interface{}{ "tender_id": t.ID.Hex(), "documents_scraped": t.ProcessingMetadata.DocumentsScraped, "documents_scraped_at": t.ProcessingMetadata.DocumentsScrapedAt, }) return nil }