Implemented the integration with scraper python server, tenders summarise, and some fixes.

2025-12-27 12:47:08 +03:30
parent e4da3b5bb2
commit f6bdcc6d7c
18 changed files with 1054 additions and 45 deletions
@@ -17,6 +17,7 @@ type Repository interface {
 	GetByID(ctx context.Context, id string) (*Notice, error)
 	GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error)
 	GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error)
+	GetProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error)
 	Delete(ctx context.Context, id string) error
 }

@@ -166,6 +167,28 @@ func (r *noticeRepository) GetUnProcessedNotices(ctx context.Context, limit int,
 	return result.Items, result.TotalCount, nil
 }

+// GetProcessedNotices retrieves notices that have been successfully processed
+func (r *noticeRepository) GetProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) {
+	filter := bson.M{"processing_metadata.processed": true}
+
+	pagination := orm.Pagination{
+		Limit:     limit,
+		Skip:      skip,
+		SortField: "updated_at",
+		SortOrder: -1,
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to get processed notices", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, 0, err
+	}
+
+	return result.Items, result.TotalCount, nil
+}
+
 // Delete deletes a notice by ID
 func (r *noticeRepository) Delete(ctx context.Context, id string) error {
 	err := r.ormRepo.Delete(ctx, id)
@@ -21,8 +21,8 @@ type scraperService struct {
 }

 // NewService creates a new scraper service with stored credentials
-func NewService(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) Service {
-	sdk := scraper.NewClient(baseURL, timeout, logger, username, password, loginURL)
+func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
+	sdk := scraper.NewClient(baseURL, timeout, logger)
 	return &scraperService{
 		sdk:    sdk,
 		logger: logger,
@@ -8,6 +8,28 @@ import (
 	"tm/pkg/mongo"
 )

+// DocumentSummary represents a summary of a tender document
+type DocumentSummary struct {
+	DocumentName    string `bson:"document_name" json:"document_name"`
+	ObjectName      string `bson:"object_name" json:"object_name"`
+	BucketName      string `bson:"bucket_name" json:"bucket_name"`
+	Summary         string `bson:"summary" json:"summary"`
+	SummaryLanguage string `bson:"summary_language" json:"summary_language"`
+	DocumentType    string `bson:"document_type" json:"document_type"`
+	SummarizedAt    int64  `bson:"summarized_at" json:"summarized_at"`
+	SummaryModel    string `bson:"summary_model" json:"summary_model"`
+	Error           string `bson:"error,omitempty" json:"error,omitempty"`
+}
+
+// ScrapedDocument represents information about a document that was scraped and uploaded to MinIO
+type ScrapedDocument struct {
+	Filename   string `bson:"filename" json:"filename"`
+	ObjectName string `bson:"object_name" json:"object_name"`
+	BucketName string `bson:"bucket_name" json:"bucket_name"`
+	Size       int64  `bson:"size" json:"size"`
+	ScrapedAt  int64  `bson:"scraped_at" json:"scraped_at"`
+}
+
 // Tender represents a tender/contract notice entity
 type Tender struct {
 	mongo.Model               `bson:",inline"`
@@ -66,6 +88,8 @@ type Tender struct {
 	SuspensionDate     int64                `bson:"suspension_date,omitempty" json:"suspension_date,omitempty"` // Unix milliseconds
 	Modifications      []TenderModification `bson:"modifications,omitempty" json:"modifications,omitempty"`
 	AwardedEntities    []Awarded            `bson:"awarded,omitempty" json:"awarded,omitempty"`
+	ScrapedDocuments   []ScrapedDocument    `bson:"scraped_documents,omitempty" json:"scraped_documents,omitempty"`
+	DocumentSummaries  []DocumentSummary    `bson:"document_summaries,omitempty" json:"document_summaries,omitempty"`
 }

 // Organization represents organization information
@@ -102,12 +126,19 @@ type SelectionCriterion struct {

 // ProcessingMetadata contains metadata about how the tender was processed
 type ProcessingMetadata struct {
-	ScrapedAt         int64             `bson:"scraped_at" json:"scraped_at"`     // Unix milliseconds
-	ProcessedAt       int64             `bson:"processed_at" json:"processed_at"` // Unix milliseconds
-	ProcessingVersion string            `bson:"processing_version" json:"processing_version"`
-	ParsingErrors     []string          `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
-	ValidationErrors  []string          `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
-	EnrichmentData    map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
+	ScrapedAt             int64             `bson:"scraped_at" json:"scraped_at"`     // Unix milliseconds
+	ProcessedAt           int64             `bson:"processed_at" json:"processed_at"` // Unix milliseconds
+	ProcessingVersion     string            `bson:"processing_version" json:"processing_version"`
+	ParsingErrors         []string          `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
+	ValidationErrors      []string          `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
+	EnrichmentData        map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
+	TranslatedData        map[string]string `bson:"translated_data,omitempty" json:"translated_data,omitempty"`
+	TranslatedAt          int64             `bson:"translated_at,omitempty" json:"translated_at,omitempty"`
+	Processed             bool              `bson:"processed" json:"processed"`
+	DocumentsScraped      bool              `bson:"documents_scraped" json:"documents_scraped"`             // Whether documents have been scraped
+	DocumentsScrapedAt    int64             `bson:"documents_scraped_at" json:"documents_scraped_at"`       // When documents were scraped
+	DocumentsSummarized   bool              `bson:"documents_summarized" json:"documents_summarized"`       // Whether documents have been summarized
+	DocumentsSummarizedAt int64             `bson:"documents_summarized_at" json:"documents_summarized_at"` // When documents were summarized
 }

 // TenderModification represents a modification made to a tender
@@ -312,3 +312,32 @@ func (h *TenderHandler) GetPublicTenderDetails(c echo.Context) error {

 	return response.Success(c, tender, "Tender details retrieved successfully")
 }
+
+// GetDocumentSummaries retrieves document summaries for a tender
+// @Summary Get document summaries for a tender
+// @Description Retrieve AI-generated summaries of documents for a specific tender
+// @Tags Tenders
+// @Produce json
+// @Param id path string true "Tender ID"
+// @Success 200 {object} response.APIResponse{data=[]DocumentSummary}
+// @Failure 400 {object} response.APIResponse
+// @Failure 404 {object} response.APIResponse
+// @Failure 500 {object} response.APIResponse
+// @Router /api/v1/tenders/{id}/document-summaries [get]
+func (h *TenderHandler) GetDocumentSummaries(c echo.Context) error {
+	id := c.Param("id")
+	if id == "" {
+		return response.BadRequest(c, "Tender ID is required", "ID parameter cannot be empty")
+	}
+
+	summaries, err := h.service.GetDocumentSummaries(c.Request().Context(), id)
+	if err != nil {
+		h.logger.Error("Failed to get document summaries", map[string]interface{}{
+			"tender_id": id,
+			"error":     err.Error(),
+		})
+		return response.NotFound(c, "Document summaries not found")
+	}
+
+	return response.Success(c, summaries, "Document summaries retrieved successfully")
+}
@@ -23,6 +23,8 @@ type TenderRepository interface {
 	GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
 	Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) ([]Tender, int64, error)
 	GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
+	GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
+	GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
 	Update(ctx context.Context, tender *Tender) error
 	Delete(ctx context.Context, id string) error
 	GetStatistics(ctx context.Context) (*TenderStatistics, error)
@@ -275,6 +277,66 @@ func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int
 	return result.Items, nil
 }

+// GetUnScrapedTenders retrieves tenders that haven't been processed for document scraping
+func (r *tenderRepository) GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
+	filter := bson.M{
+		"$or": []bson.M{
+			{"processing_metadata.documents_scraped": bson.M{"$exists": false}},
+			{"processing_metadata.documents_scraped": false},
+			{"processing_metadata.documents_scraped": nil},
+		},
+	}
+
+	pagination := orm.Pagination{
+		Limit:     limit,
+		Skip:      skip,
+		SortField: "created_at",
+		SortOrder: 1, // Oldest first
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to get un-scraped tenders", map[string]interface{}{
+			"limit": limit,
+			"skip":  skip,
+			"error": err.Error(),
+		})
+		return nil, 0, err
+	}
+
+	return result.Items, result.TotalCount, nil
+}
+
+// GetUnSummarizedTenders retrieves tenders that have documents scraped but not summarized
+func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
+	filter := bson.M{
+		"processing_metadata.documents_scraped": true,
+		"$or": []bson.M{
+			{"processing_metadata.documents_summarized": bson.M{"$exists": false}},
+			{"processing_metadata.documents_summarized": false},
+		},
+	}
+
+	pagination := orm.Pagination{
+		Limit:     limit,
+		Skip:      skip,
+		SortField: "processing_metadata.documents_scraped_at",
+		SortOrder: 1, // Oldest first (process oldest scraped documents first)
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
+			"limit": limit,
+			"skip":  skip,
+			"error": err.Error(),
+		})
+		return nil, 0, err
+	}
+
+	return result.Items, result.TotalCount, nil
+}
+
 // GetStatistics calculates tender statistics using aggregation
 func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
 	stats := &TenderStatistics{
@@ -17,6 +17,7 @@ type Service interface {
 	Delete(ctx context.Context, id string) error
 	Search(ctx context.Context, req *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
 	Recommend(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
+	GetDocumentSummaries(ctx context.Context, id string) ([]DocumentSummary, error)

 	// Maintenance operations
 	UpdateExpiredTenders(ctx context.Context) error
@@ -129,6 +130,39 @@ func (s *tenderService) Delete(ctx context.Context, id string) error {
 	return nil
 }

+// GetDocumentSummaries retrieves document summaries for a specific tender
+func (s *tenderService) GetDocumentSummaries(ctx context.Context, id string) ([]DocumentSummary, error) {
+	if id == "" {
+		return nil, fmt.Errorf("tender ID cannot be empty")
+	}
+
+	// Get tender from repository
+	tender, err := s.repository.GetByID(ctx, id)
+	if err != nil {
+		s.logger.Error("Failed to get tender for document summaries", map[string]interface{}{
+			"tender_id": id,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to get tender: %w", err)
+	}
+
+	if tender == nil {
+		return nil, fmt.Errorf("tender not found")
+	}
+
+	// Return document summaries (empty slice if none exist)
+	if tender.DocumentSummaries == nil {
+		return []DocumentSummary{}, nil
+	}
+
+	s.logger.Info("Retrieved document summaries for tender", map[string]interface{}{
+		"tender_id":            id,
+		"summaries_count":      len(tender.DocumentSummaries),
+	})
+
+	return tender.DocumentSummaries, nil
+}
+
 // ListTenders retrieves tenders with pagination and filtering
 func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) {
 	s.logger.Info("Listing tenders", map[string]interface{}{