From f7763cf99792da8e7bd84e579aa16f9ad0fe33a4 Mon Sep 17 00:00:00 2001 From: Mazyar Date: Tue, 19 May 2026 19:23:18 +0330 Subject: [PATCH] get all tenders with documeents scraped API --- cmd/web/router/routes.go | 1 + internal/tender/entity.go | 4 +- internal/tender/form.go | 31 ++++++++ internal/tender/handler.go | 26 +++++++ internal/tender/service.go | 137 ++++++++++++++++++++++++++++++++++ pkg/ai_summarizer/entities.go | 7 ++ pkg/ai_summarizer/storage.go | 102 +++++++++++++++++++++++++ 7 files changed, 306 insertions(+), 2 deletions(-) diff --git a/cmd/web/router/routes.go b/cmd/web/router/routes.go index 3adf509..74df0c0 100644 --- a/cmd/web/router/routes.go +++ b/cmd/web/router/routes.go @@ -107,6 +107,7 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler tendersGP.Use(userHandler.AuthMiddleware()) tendersGP.GET("", tenderHandler.Search) tendersGP.GET("/ai-summaries", tenderHandler.GetAllAISummaries) + tendersGP.GET("/scraped-documents", tenderHandler.ListTendersWithScrapedDocuments) tendersGP.GET("/:id", tenderHandler.Get) tendersGP.PUT("/:id", tenderHandler.Update) tendersGP.DELETE("/:id", tenderHandler.Delete) diff --git a/internal/tender/entity.go b/internal/tender/entity.go index 10a259a..3bb9d45 100644 --- a/internal/tender/entity.go +++ b/internal/tender/entity.go @@ -175,9 +175,9 @@ type ProcessingMetadata struct { TranslatedAt int64 `bson:"translated_at,omitempty" json:"translated_at,omitempty"` Processed bool `bson:"processed" json:"processed"` DocumentsScraped bool `bson:"documents_scraped" json:"documents_scraped"` // Whether documents have been scraped - DocumentsScrapedAt int64 `bson:"documents_scraped_at" json:"documents_scraped_at"` // When documents were scraped + DocumentsScrapedAt int64 `bson:"documents_scraped_at" json:"documents_scraped_at"` // Unix timestamp (seconds) when documents were scraped DocumentsSummarized bool `bson:"documents_summarized" json:"documents_summarized"` // Whether documents have been summarized - DocumentsSummarizedAt int64 `bson:"documents_summarized_at" json:"documents_summarized_at"` // When documents were summarized + DocumentsSummarizedAt int64 `bson:"documents_summarized_at" json:"documents_summarized_at"` // Unix timestamp (seconds) when documents were summarized } // TenderModification represents a modification made to a tender diff --git a/internal/tender/form.go b/internal/tender/form.go index edb4c89..c9fe943 100644 --- a/internal/tender/form.go +++ b/internal/tender/form.go @@ -69,6 +69,37 @@ type SearchResponse struct { Metadata *response.Meta `json:"-"` } +// TenderScrapedDocumentsItem is a tender that has scraped documents stored in the database. +type TenderScrapedDocumentsItem struct { + TenderID string `json:"tender_id"` + NoticePublicationID string `json:"notice_publication_id"` + ContractFolderID string `json:"contract_folder_id"` + Title string `json:"title"` + DocumentsScrapedAt int64 `json:"documents_scraped_at,omitempty"` // Unix timestamp (seconds) + DocumentCount int `json:"document_count"` +} + +// TendersWithScrapedDocumentsResponse is the paginated list of tenders with scraped documents. +type TendersWithScrapedDocumentsResponse struct { + Tenders []TenderScrapedDocumentsItem `json:"tenders"` + Metadata *response.Meta `json:"-"` +} + +// ToTenderScrapedDocumentsItem maps a tender entity to a list item. +func (t *Tender) ToTenderScrapedDocumentsItem() TenderScrapedDocumentsItem { + if t == nil { + return TenderScrapedDocumentsItem{} + } + return TenderScrapedDocumentsItem{ + TenderID: t.ID.Hex(), + NoticePublicationID: strings.TrimSpace(t.NoticePublicationID), + ContractFolderID: strings.TrimSpace(t.ContractFolderID), + Title: strings.TrimSpace(t.Title), + DocumentsScrapedAt: t.ProcessingMetadata.DocumentsScrapedAt, + DocumentCount: len(t.ScrapedDocuments), + } +} + // AddressResponse is postal / NUTS address data for API consumers. type AddressResponse struct { StreetName string `json:"street_name,omitempty"` diff --git a/internal/tender/handler.go b/internal/tender/handler.go index 29e849a..90262e7 100644 --- a/internal/tender/handler.go +++ b/internal/tender/handler.go @@ -657,6 +657,32 @@ func (h *TenderHandler) TriggerPublicAITranslate(c echo.Context) error { return response.Success(c, result, "AI translation completed successfully") } +// ListTendersWithScrapedDocuments lists tenders that have scraped documents in the database +// @Summary List tenders with scraped documents +// @Description Lists database tenders whose contract_folder_id has documents in MinIO (PROC_/…) and syncs scraped document metadata +// @Tags Admin-Tenders +// @Produce json +// @Param limit query int false "Number of items per page (default: 10, max: 100)" +// @Param offset query int false "Number of items to skip (default: 0)" +// @Success 200 {object} response.APIResponse{data=TendersWithScrapedDocumentsResponse} +// @Failure 400 {object} response.APIResponse +// @Failure 500 {object} response.APIResponse +// @Router /admin/v1/tenders/scraped-documents [get] +// @Security BearerAuth +func (h *TenderHandler) ListTendersWithScrapedDocuments(c echo.Context) error { + pagination, err := response.NewPagination(c) + if err != nil { + return response.PaginationBadRequest(c, err) + } + + result, err := h.service.ListTendersWithScrapedDocuments(c.Request().Context(), pagination) + if err != nil { + return response.InternalServerError(c, "Failed to list tenders with scraped documents") + } + + return response.SuccessWithMeta(c, result, result.Metadata, "Tenders with scraped documents retrieved successfully") +} + // GetAllAISummaries retrieves all AI-generated summaries from storage // @Summary Get all AI-generated tender summaries // @Description Retrieve all AI-generated summaries from the AI pipeline storage, including pending and completed ones diff --git a/internal/tender/service.go b/internal/tender/service.go index 60c4e0e..a70d5dd 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -59,6 +59,7 @@ type Service interface { TriggerAISummarize(ctx context.Context, id string) (*AISummarizeResponse, error) TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error) GetAllAISummaries(ctx context.Context) ([]ai_summarizer.TenderSummaryItem, error) + ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination) (*TendersWithScrapedDocumentsResponse, error) // Maintenance operations UpdateExpiredTenders(ctx context.Context) error @@ -1069,6 +1070,142 @@ func (s *tenderService) GetAllAISummaries(ctx context.Context) ([]ai_summarizer. return results, nil } +// ListTendersWithScrapedDocuments lists database tenders whose contract_folder_id has +// documents in MinIO, and syncs scraped document metadata onto each tender. +func (s *tenderService) ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination) (*TendersWithScrapedDocumentsResponse, error) { + if s.aiSummarizerStorage == nil { + return nil, fmt.Errorf("document storage is not configured") + } + + type procedureLister interface { + ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) + } + storage, ok := s.aiSummarizerStorage.(procedureLister) + if !ok { + return nil, fmt.Errorf("document storage does not support procedure listing") + } + + s.logger.Info("Listing tenders with scraped documents from MinIO", map[string]interface{}{ + "limit": pagination.Limit, + "offset": pagination.Offset, + }) + + procedures, err := storage.ListProceduresWithDocuments(ctx) + if err != nil { + fields := map[string]interface{}{"error": err.Error()} + if errors.Is(err, ai_summarizer.ErrMinIOUnavailable) { + s.logger.Error("Failed to list scraped documents: MinIO unavailable", fields) + return nil, fmt.Errorf("failed to list tenders with scraped documents: MinIO connection unavailable: %w", err) + } + s.logger.Error("Failed to list procedure documents from MinIO", fields) + return nil, fmt.Errorf("failed to list tenders with scraped documents: %w", err) + } + + matched := make([]TenderScrapedDocumentsItem, 0, len(procedures)) + for _, proc := range procedures { + tender, err := s.repository.GetByContractFolderID(ctx, proc.ContractFolderID) + if err != nil || tender == nil { + continue + } + + docs, listErr := s.listDocumentsForTender(ctx, tender) + if listErr != nil { + if errors.Is(listErr, ai_summarizer.ErrMinIOUnavailable) { + return nil, fmt.Errorf("failed to list tenders with scraped documents: MinIO connection unavailable: %w", listErr) + } + s.logger.Debug("Skipping tender: could not list documents from MinIO", map[string]interface{}{ + "tender_id": tender.ID.Hex(), + "contract_folder_id": proc.ContractFolderID, + "error": listErr.Error(), + }) + continue + } + if len(docs) == 0 { + continue + } + + s.persistScrapedDocuments(ctx, tender, docs) + + item := tender.ToTenderScrapedDocumentsItem() + item.DocumentCount = len(docs) + matched = append(matched, item) + } + + total := int64(len(matched)) + start := pagination.Offset + if start > len(matched) { + start = len(matched) + } + end := start + pagination.Limit + if end > len(matched) { + end = len(matched) + } + page := matched[start:end] + hasMore := int64(end) < total + + s.logger.Info("Listed tenders with scraped documents", map[string]interface{}{ + "count": len(page), + "total_count": total, + }) + + return &TendersWithScrapedDocumentsResponse{ + Tenders: page, + Metadata: pagination.ListMeta(total, "", hasMore, pagination.Offset), + }, nil +} + +// persistScrapedDocuments updates the tender when MinIO has scraped documents for its procedure. +func (s *tenderService) persistScrapedDocuments(ctx context.Context, t *Tender, docs []ai_summarizer.StoredDocument) { + if t == nil || len(docs) == 0 { + return + } + + bucket := "" + if named, ok := s.aiSummarizerStorage.(interface{ BucketName() string }); ok { + bucket = named.BucketName() + } + + scraped := make([]ScrapedDocument, len(docs)) + var latestModified int64 + for i, doc := range docs { + scraped[i] = ScrapedDocument{ + Filename: doc.DocumentName, + ObjectName: doc.ObjectName, + BucketName: bucket, + Size: doc.Size, + ScrapedAt: doc.LastModified, + } + if doc.LastModified > latestModified { + latestModified = doc.LastModified + } + } + + unchanged := t.ProcessingMetadata.DocumentsScraped && + len(t.ScrapedDocuments) == len(scraped) + if unchanged { + return + } + + t.ProcessingMetadata.DocumentsScraped = true + if t.ProcessingMetadata.DocumentsScrapedAt == 0 { + if latestModified > 0 { + t.ProcessingMetadata.DocumentsScrapedAt = latestModified + } else { + t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix() + } + } + t.ScrapedDocuments = scraped + t.UpdatedAt = time.Now().Unix() + + if err := s.repository.Update(ctx, t); err != nil { + s.logger.Warn("Failed to persist scraped documents on tender", map[string]interface{}{ + "tender_id": t.ID.Hex(), + "contract_folder_id": t.ContractFolderID, + "error": err.Error(), + }) + } +} + // TriggerAITranslate triggers on-demand AI translation for a tender and stores // translated result per language in MongoDB for future reuse. func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error) { diff --git a/pkg/ai_summarizer/entities.go b/pkg/ai_summarizer/entities.go index 1ee3212..56cd7f3 100644 --- a/pkg/ai_summarizer/entities.go +++ b/pkg/ai_summarizer/entities.go @@ -85,6 +85,13 @@ type DocumentSummary struct { Error string `json:"error"` // empty string when no error } +// ProcedureDocumentsSummary reports scraped documents stored under one procedure folder in MinIO. +type ProcedureDocumentsSummary struct { + ContractFolderID string `json:"contract_folder_id"` + DocumentCount int `json:"document_count"` + LatestModified int64 `json:"latest_modified"` // Unix seconds +} + // StoredDocument represents a document object stored for a tender in MinIO. type StoredDocument struct { ObjectName string `json:"-"` diff --git a/pkg/ai_summarizer/storage.go b/pkg/ai_summarizer/storage.go index 38f2ffc..c0bfa5f 100644 --- a/pkg/ai_summarizer/storage.go +++ b/pkg/ai_summarizer/storage.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "path/filepath" + "sort" "strings" "time" @@ -180,6 +181,14 @@ func NewStorageClient(config *Config, log logger.Logger) (*StorageClient, error) return storage, nil } +// BucketName returns the configured MinIO bucket used for AI pipeline artefacts. +func (s *StorageClient) BucketName() string { + if s == nil || s.config == nil { + return "" + } + return s.config.MinioBucket +} + // verifyConnection checks that MinIO is reachable and the configured bucket exists. func (s *StorageClient) verifyConnection(ctx context.Context) error { exists, err := s.client.BucketExists(ctx, s.config.MinioBucket) @@ -522,6 +531,99 @@ func (s *StorageClient) ListDocuments(ctx context.Context, contractFolderID, not return documents, nil } +// ListProceduresWithDocuments scans MinIO for PROC_/…/documents/* keys +// and returns one entry per contract folder id that has at least one document file. +func (s *StorageClient) ListProceduresWithDocuments(ctx context.Context) ([]ProcedureDocumentsSummary, error) { + s.logger.Debug("Listing procedure folders with documents from MinIO", map[string]interface{}{ + "bucket": s.config.MinioBucket, + "prefix": procedurePrefix, + }) + + byFolder := make(map[string]*ProcedureDocumentsSummary) + var fatalErr error + + appendFromPrefix := func(prefix string) bool { + objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) + for object := range objectCh { + if object.Err != nil { + fatalErr = s.logMinIOFailure("list_procedure_documents", object.Err, map[string]interface{}{ + "prefix": prefix, + }) + return false + } + contractFolderID, noticePrefix, ok := contractFolderIDFromDocumentKey(object.Key) + if !ok { + continue + } + if !isTenderDocumentObject(noticePrefix, object.Key) { + continue + } + entry := byFolder[contractFolderID] + if entry == nil { + entry = &ProcedureDocumentsSummary{ContractFolderID: contractFolderID} + byFolder[contractFolderID] = entry + } + entry.DocumentCount++ + modified := object.LastModified.Unix() + if modified > entry.LatestModified { + entry.LatestModified = modified + } + } + return true + } + + if !appendFromPrefix(procedurePrefix) { + return nil, fatalErr + } + if len(byFolder) == 0 { + if !appendFromPrefix("") { + return nil, fatalErr + } + } + if fatalErr != nil { + return nil, fatalErr + } + + results := make([]ProcedureDocumentsSummary, 0, len(byFolder)) + for _, entry := range byFolder { + results = append(results, *entry) + } + sort.Slice(results, func(i, j int) bool { + if results[i].LatestModified == results[j].LatestModified { + return results[i].ContractFolderID < results[j].ContractFolderID + } + return results[i].LatestModified > results[j].LatestModified + }) + + s.logger.Info("Listed procedure folders with documents from storage", map[string]interface{}{ + "bucket": s.config.MinioBucket, + "procedure_count": len(results), + }) + + return results, nil +} + +// contractFolderIDFromDocumentKey parses PROC_//documents/. +func contractFolderIDFromDocumentKey(key string) (contractFolderID, noticePrefix string, ok bool) { + key = strings.TrimPrefix(strings.TrimSpace(key), "/") + parts := strings.Split(key, "/") + if len(parts) < 4 { + return "", "", false + } + if !strings.EqualFold(parts[2], "documents") { + return "", "", false + } + contractFolderID = stripLeadingProcPrefix(parts[0]) + if contractFolderID == "" { + return "", "", false + } + noticePrefix = strings.Join(parts[:3], "/") + "/" + return contractFolderID, noticePrefix, true +} + // isTenderDocumentObject reports whether an object key under the given prefix // represents a downloadable tender document (PDF/DOCX/...) rather than an // internal artefact such as tender.json or a cache file.