tm_back/cmd/worker/workers/scraper.go

package workers

import (
	"context"
	"time"
	"tm/internal/tender"
	"tm/pkg/logger"
	"tm/pkg/mongo"
	"tm/pkg/scraper"
)

// DocumentScraperWorker handles document scraping for tenders
type DocumentScraperWorker struct {
	Mongo      *mongo.ConnectionManager
	Logger     logger.Logger
	TenderRepo tender.TenderRepository
	Scraper    scraper.SDK
}

// NewDocumentScraperWorker creates a new document scraper worker
func NewDocumentScraperWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, scraper scraper.SDK) *DocumentScraperWorker {
	logger.Info("Creating new document scraper worker", map[string]interface{}{})
	return &DocumentScraperWorker{
		Mongo:      mongo,
		Logger:     logger,
		TenderRepo: tenderRepo,
		Scraper:    scraper,
	}
}

// Run starts the document scraping process
func (w *DocumentScraperWorker) Run() {
	w.Logger.Info("Document scraper worker started", map[string]interface{}{})

	limit := 10
	skip := 0
	for {
		w.Logger.Info("Document scraper worker checking for tenders", map[string]interface{}{
			"skip":  skip,
			"limit": limit,
		})
		tenders, totalCount, err := w.TenderRepo.GetUnScrapedTenders(context.Background(), limit, skip)
		if err != nil {
			w.Logger.Error("Failed to get un-scraped tenders", map[string]interface{}{
				"error": err.Error(),
			})
			continue
		}

		w.Logger.Info("Found tenders for document scraping", map[string]interface{}{
			"count":       len(tenders),
			"total_count": totalCount,
			"skip":        skip,
		})

		if len(tenders) == 0 {
			w.Logger.Info("No more tenders to process for document scraping", map[string]interface{}{})
			break
		}

		w.Logger.Info("Processing tenders for document scraping", map[string]interface{}{
			"tender_count": len(tenders),
		})

		for _, t := range tenders {
			w.Logger.Info("Processing tender for document scraping", map[string]interface{}{
				"tender_id":             t.ID.Hex(),
				"notice_publication_id": t.NoticePublicationID,
			})

			w.scrapeDocumentsForTender(&t)
		}

		skip += limit
	}

	w.Logger.Info("Document scraper worker completed", map[string]interface{}{})
}

// scrapeDocumentsForTender scrapes documents for a single tender
func (w *DocumentScraperWorker) scrapeDocumentsForTender(t *tender.Tender) {
	w.Logger.Info("Starting document scraping for tender", map[string]interface{}{
		"tender_id":             t.ID.Hex(),
		"notice_publication_id": t.NoticePublicationID,
		"document_uri":          t.DocumentURI,
	})

	// Skip if no NoticePublicationID
	if t.NoticePublicationID == "" {
		w.Logger.Warn("Skipping document scraping - no NoticePublicationID", map[string]interface{}{
			"tender_id": t.ID.Hex(),
		})
		return
	}

	// Use DocumentURI as NoticeURL for the scraper
	noticeURL := t.DocumentURI
	if noticeURL == "" {
		w.Logger.Warn("Skipping document scraping - no DocumentURI available", map[string]interface{}{
			"tender_id":             t.ID.Hex(),
			"notice_publication_id": t.NoticePublicationID,
		})
		return
	}

	// Call scraper SDK
	scrapeReq := &scraper.ScrapeRequest{
		NoticeID:  t.NoticePublicationID, // Use NoticePublicationID as notice_id
		NoticeURL: noticeURL,             // Use DocumentURI as notice_url
	}

	w.Logger.Info("Starting document scraping for tender", map[string]interface{}{
		"tender_id":             t.ID.Hex(),
		"notice_publication_id": t.NoticePublicationID,
		"document_uri":          noticeURL,
	})

	response, err := w.Scraper.ScrapeDocuments(context.Background(), scrapeReq)
	if err != nil {
		w.Logger.Error("Failed to scrape documents for tender", map[string]interface{}{
			"tender_id":             t.ID.Hex(),
			"notice_publication_id": t.NoticePublicationID,
			"error":                 err.Error(),
		})

		// Mark as processed with error (don't retry indefinitely)
		t.ProcessingMetadata.DocumentsScraped = false
		t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix()
		w.updateTenderScrapingStatus(t)
		return
	}

	if !response.Success {
		w.Logger.Warn("Document scraping failed for tender", map[string]interface{}{
			"tender_id":             t.ID.Hex(),
			"notice_publication_id": t.NoticePublicationID,
			"errors":                response.Errors,
		})

		// Mark as processed with failure
		t.ProcessingMetadata.DocumentsScraped = false
		t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix()
		w.updateTenderScrapingStatus(t)
		return
	}

	// Store scraped documents information
	scrapedAt := time.Now().Unix()
	t.ScrapedDocuments = make([]tender.ScrapedDocument, len(response.UploadedFiles))
	for i, uploadedFile := range response.UploadedFiles {
		t.ScrapedDocuments[i] = tender.ScrapedDocument{
			Filename:   uploadedFile.Filename,
			ObjectName: uploadedFile.ObjectName,
			BucketName: "opplens",
			Size:       uploadedFile.Size,
			ScrapedAt:  scrapedAt,
		}
	}

	// Mark as successfully scraped
	t.ProcessingMetadata.DocumentsScraped = true
	t.ProcessingMetadata.DocumentsScrapedAt = scrapedAt

	err = w.updateTenderScrapingStatus(t)
	if err != nil {
		w.Logger.Error("Failed to update tender scraping status", map[string]interface{}{
			"tender_id":             t.ID.Hex(),
			"notice_publication_id": t.NoticePublicationID,
			"error":                 err.Error(),
		})
		return
	}

	w.Logger.Info("Document scraping completed successfully for tender", map[string]interface{}{
		"tender_id":             t.ID.Hex(),
		"notice_publication_id": t.NoticePublicationID,
		"uploaded_count":        response.UploadedCount,
		"bucket_count":          len(response.Buckets),
	})
}

// updateTenderScrapingStatus updates the tender's document scraping metadata
func (w *DocumentScraperWorker) updateTenderScrapingStatus(t *tender.Tender) error {
	err := w.TenderRepo.Update(context.Background(), t)
	if err != nil {
		w.Logger.Error("Failed to update tender scraping status", map[string]interface{}{
			"tender_id": t.ID.Hex(),
			"error":     err.Error(),
		})
		return err
	}

	w.Logger.Debug("Updated tender scraping status", map[string]interface{}{
		"tender_id":            t.ID.Hex(),
		"documents_scraped":    t.ProcessingMetadata.DocumentsScraped,
		"documents_scraped_at": t.ProcessingMetadata.DocumentsScrapedAt,
	})

	return nil
}