Implemented the integration with scraper python server, tenders summarise, and some fixes.

2025-12-27 12:47:08 +03:30
parent e4da3b5bb2
commit f6bdcc6d7c
18 changed files with 1054 additions and 45 deletions
@@ -0,0 +1,200 @@
+package workers
+
+import (
+	"context"
+	"time"
+	"tm/internal/tender"
+	"tm/pkg/logger"
+	"tm/pkg/mongo"
+	"tm/pkg/scraper"
+)
+
+// DocumentScraperWorker handles document scraping for tenders
+type DocumentScraperWorker struct {
+	Mongo      *mongo.ConnectionManager
+	Logger     logger.Logger
+	TenderRepo tender.TenderRepository
+	Scraper    scraper.SDK
+}
+
+// NewDocumentScraperWorker creates a new document scraper worker
+func NewDocumentScraperWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, scraper scraper.SDK) *DocumentScraperWorker {
+	logger.Info("Creating new document scraper worker", map[string]interface{}{})
+	return &DocumentScraperWorker{
+		Mongo:      mongo,
+		Logger:     logger,
+		TenderRepo: tenderRepo,
+		Scraper:    scraper,
+	}
+}
+
+// Run starts the document scraping process
+func (w *DocumentScraperWorker) Run() {
+	w.Logger.Info("Document scraper worker started", map[string]interface{}{})
+
+	limit := 10
+	skip := 0
+	for {
+		w.Logger.Info("Document scraper worker checking for tenders", map[string]interface{}{
+			"skip":  skip,
+			"limit": limit,
+		})
+		tenders, totalCount, err := w.TenderRepo.GetUnScrapedTenders(context.Background(), limit, skip)
+		if err != nil {
+			w.Logger.Error("Failed to get un-scraped tenders", map[string]interface{}{
+				"error": err.Error(),
+			})
+			continue
+		}
+
+		w.Logger.Info("Found tenders for document scraping", map[string]interface{}{
+			"count":       len(tenders),
+			"total_count": totalCount,
+			"skip":        skip,
+		})
+
+		if len(tenders) == 0 {
+			w.Logger.Info("No more tenders to process for document scraping", map[string]interface{}{})
+			break
+		}
+
+		w.Logger.Info("Processing tenders for document scraping", map[string]interface{}{
+			"tender_count": len(tenders),
+		})
+
+		for _, t := range tenders {
+			w.Logger.Info("Processing tender for document scraping", map[string]interface{}{
+				"tender_id":             t.ID.Hex(),
+				"notice_publication_id": t.NoticePublicationID,
+			})
+
+			w.scrapeDocumentsForTender(&t)
+		}
+
+		skip += limit
+	}
+
+	w.Logger.Info("Document scraper worker completed", map[string]interface{}{})
+}
+
+// scrapeDocumentsForTender scrapes documents for a single tender
+func (w *DocumentScraperWorker) scrapeDocumentsForTender(t *tender.Tender) {
+	w.Logger.Info("Starting document scraping for tender", map[string]interface{}{
+		"tender_id":             t.ID.Hex(),
+		"notice_publication_id": t.NoticePublicationID,
+		"document_uri":          t.DocumentURI,
+	})
+
+	// Skip if no NoticePublicationID
+	if t.NoticePublicationID == "" {
+		w.Logger.Warn("Skipping document scraping - no NoticePublicationID", map[string]interface{}{
+			"tender_id": t.ID.Hex(),
+		})
+		return
+	}
+
+	// Use DocumentURI as NoticeURL for the scraper
+	noticeURL := t.DocumentURI
+	if noticeURL == "" {
+		w.Logger.Warn("Skipping document scraping - no DocumentURI available", map[string]interface{}{
+			"tender_id":             t.ID.Hex(),
+			"notice_publication_id": t.NoticePublicationID,
+		})
+		return
+	}
+
+	// Call scraper SDK
+	scrapeReq := &scraper.ScrapeRequest{
+		NoticeID:  t.NoticePublicationID, // Use NoticePublicationID as notice_id
+		NoticeURL: noticeURL,             // Use DocumentURI as notice_url
+	}
+
+	w.Logger.Info("Starting document scraping for tender", map[string]interface{}{
+		"tender_id":             t.ID.Hex(),
+		"notice_publication_id": t.NoticePublicationID,
+		"document_uri":          noticeURL,
+	})
+
+	response, err := w.Scraper.ScrapeDocuments(context.Background(), scrapeReq)
+	if err != nil {
+		w.Logger.Error("Failed to scrape documents for tender", map[string]interface{}{
+			"tender_id":             t.ID.Hex(),
+			"notice_publication_id": t.NoticePublicationID,
+			"error":                 err.Error(),
+		})
+
+		// Mark as processed with error (don't retry indefinitely)
+		t.ProcessingMetadata.DocumentsScraped = false
+		t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix()
+		w.updateTenderScrapingStatus(t)
+		return
+	}
+
+	if !response.Success {
+		w.Logger.Warn("Document scraping failed for tender", map[string]interface{}{
+			"tender_id":             t.ID.Hex(),
+			"notice_publication_id": t.NoticePublicationID,
+			"errors":                response.Errors,
+		})
+
+		// Mark as processed with failure
+		t.ProcessingMetadata.DocumentsScraped = false
+		t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix()
+		w.updateTenderScrapingStatus(t)
+		return
+	}
+
+	// Store scraped documents information
+	scrapedAt := time.Now().Unix()
+	t.ScrapedDocuments = make([]tender.ScrapedDocument, len(response.UploadedFiles))
+	for i, uploadedFile := range response.UploadedFiles {
+		t.ScrapedDocuments[i] = tender.ScrapedDocument{
+			Filename:   uploadedFile.Filename,
+			ObjectName: uploadedFile.ObjectName,
+			BucketName: "opplens",
+			Size:       uploadedFile.Size,
+			ScrapedAt:  scrapedAt,
+		}
+	}
+
+	// Mark as successfully scraped
+	t.ProcessingMetadata.DocumentsScraped = true
+	t.ProcessingMetadata.DocumentsScrapedAt = scrapedAt
+
+	err = w.updateTenderScrapingStatus(t)
+	if err != nil {
+		w.Logger.Error("Failed to update tender scraping status", map[string]interface{}{
+			"tender_id":             t.ID.Hex(),
+			"notice_publication_id": t.NoticePublicationID,
+			"error":                 err.Error(),
+		})
+		return
+	}
+
+	w.Logger.Info("Document scraping completed successfully for tender", map[string]interface{}{
+		"tender_id":             t.ID.Hex(),
+		"notice_publication_id": t.NoticePublicationID,
+		"uploaded_count":        response.UploadedCount,
+		"bucket_count":          len(response.Buckets),
+	})
+}
+
+// updateTenderScrapingStatus updates the tender's document scraping metadata
+func (w *DocumentScraperWorker) updateTenderScrapingStatus(t *tender.Tender) error {
+	err := w.TenderRepo.Update(context.Background(), t)
+	if err != nil {
+		w.Logger.Error("Failed to update tender scraping status", map[string]interface{}{
+			"tender_id": t.ID.Hex(),
+			"error":     err.Error(),
+		})
+		return err
+	}
+
+	w.Logger.Debug("Updated tender scraping status", map[string]interface{}{
+		"tender_id":            t.ID.Hex(),
+		"documents_scraped":    t.ProcessingMetadata.DocumentsScraped,
+		"documents_scraped_at": t.ProcessingMetadata.DocumentsScrapedAt,
+	})
+
+	return nil
+}