Enhance AI pipeline service with document metadata synchronization

- Updated the AI pipeline service to include a new `ScrapedDocumentMetadataSyncer` interface for persisting scraped document metadata onto tender records.
- Modified the `NewService` function to accept the new metadata syncer dependency.
- Implemented synchronization of scraped document metadata in the `ScrapeDocuments` and `Run` methods.
- Enhanced the tender service to enrich search filters based on scraped documents and added a new method for syncing scraped documents from storage.
- Updated the `SearchForm` to include `ContractFolderIDsWithDocuments` for better handling of scraped documents in queries.

This update improves the integration of scraped document handling within the AI pipeline, enhancing data consistency and operational efficiency in the tender management system.

internal/ai_pipeline/service.go

@@ -46,16 +46,23 @@ type Service interface {
 	GetMinioStats(ctx context.Context) (ai_summarizer.PipelineMinioStatsResponse, error)
 }
 
+// ScrapedDocumentMetadataSyncer persists MinIO scraped-document metadata onto tender records.
+type ScrapedDocumentMetadataSyncer interface {
+	SyncScrapedDocumentsFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) error
+}
+
 type service struct {
-	client Client
-	logger logger.Logger
+	client         Client
+	logger         logger.Logger
+	metadataSyncer ScrapedDocumentMetadataSyncer
 }
 
 // NewService creates an AI pipeline admin service.
-func NewService(client Client, log logger.Logger) Service {
+func NewService(client Client, log logger.Logger, metadataSyncer ScrapedDocumentMetadataSyncer) Service {
 	return &service{
-		client: client,
-		logger: log,
+		client:         client,
+		logger:         log,
+		metadataSyncer: metadataSyncer,
 	}
 }
 
@@ -89,6 +96,10 @@ func (s *service) ScrapeDocuments(ctx context.Context, form ScrapeDocumentsForm)
 		return nil, fmt.Errorf("scrape documents failed: %w", err)
 	}
 
+	if resp != nil && resp.Success && resp.FilesDownloaded > 0 {
+		s.syncScrapedDocumentMetadata(ctx, form.ContractFolderID, form.NoticePublicationID)
+	}
+
 	return resp, nil
 }
 
@@ -222,6 +233,8 @@ func (s *service) Run(ctx context.Context, form PipelineRunForm) (map[string]int
 		return nil, fmt.Errorf("pipeline run failed: %w", err)
 	}
 
+	s.syncScrapedDocumentMetadata(ctx, req.ContractFolderID, req.NoticePublicationID)
+
 	return resp, nil
 }
 
@@ -398,3 +411,16 @@ func validateTenderBatch(forms []TenderRefForm) ([]ai_summarizer.TenderRef, erro
 	}
 	return toTenderRefs(forms), nil
 }
+
+func (s *service) syncScrapedDocumentMetadata(ctx context.Context, contractFolderID, noticePublicationID string) {
+	if s.metadataSyncer == nil {
+		return
+	}
+	if err := s.metadataSyncer.SyncScrapedDocumentsFromStorage(ctx, contractFolderID, noticePublicationID); err != nil {
+		s.logger.Warn("Failed to sync scraped document metadata to Mongo", map[string]interface{}{
+			"contract_folder_id":    contractFolderID,
+			"notice_publication_id": noticePublicationID,
+			"error":                 err.Error(),
+		})
+	}
+}