List and download tender documents - worker auto translation to english job

2026-05-10 02:14:53 +03:30
parent 567e20b19c
commit e136f0eaa7
11 changed files with 652 additions and 2 deletions
@@ -70,6 +70,20 @@ type TenderResponse struct {
 	Language            string                `json:"language,omitempty"`
 }

+// TenderDocumentResponse represents a scraped tender document available for download.
+type TenderDocumentResponse struct {
+	Filename     string `json:"filename"`
+	Size         int64  `json:"size"`
+	LastModified int64  `json:"last_modified"`
+	DocumentType string `json:"document_type,omitempty"`
+}
+
+// TenderDocumentsResponse represents scraped documents for a tender.
+type TenderDocumentsResponse struct {
+	TenderID  string                   `json:"tender_id"`
+	Documents []TenderDocumentResponse `json:"documents"`
+}
+
 type OrganizationResponse struct {
 	Name string `json:"name"`
 }
@@ -1,6 +1,10 @@
 package tender

 import (
+	"errors"
+	"mime"
+	"net/http"
+	"strconv"
 	"time"
 	"tm/pkg/logger"
 	"tm/pkg/response"
@@ -361,6 +365,71 @@ func (h *TenderHandler) GetDocumentSummaries(c echo.Context) error {
 	return response.Success(c, summaries, "Document summaries retrieved successfully")
 }

+// GetDocuments retrieves scraped documents for a tender
+// @Summary Get scraped documents for a tender
+// @Description Retrieve scraped document metadata for a specific tender
+// @Tags Tenders
+// @Produce json
+// @Param id path string true "Tender ID"
+// @Success 200 {object} response.APIResponse{data=TenderDocumentsResponse}
+// @Failure 400 {object} response.APIResponse
+// @Failure 404 {object} response.APIResponse
+// @Failure 500 {object} response.APIResponse
+// @Router /api/v1/tenders/{id}/documents [get]
+func (h *TenderHandler) GetDocuments(c echo.Context) error {
+	id := c.Param("id")
+	if id == "" {
+		return response.BadRequest(c, "Tender ID is required", "ID parameter cannot be empty")
+	}
+
+	documents, err := h.service.GetDocuments(c.Request().Context(), id)
+	if err != nil {
+		return response.NotFound(c, "Tender documents not found")
+	}
+
+	return response.Success(c, documents, "Tender documents retrieved successfully")
+}
+
+// DownloadDocuments downloads all scraped documents for a tender
+// @Summary Download scraped documents for a tender
+// @Description Stream all scraped tender documents from storage as a ZIP archive
+// @Tags Tenders
+// @Produce application/zip
+// @Param id path string true "Tender ID"
+// @Success 200 {file} file
+// @Failure 400 {object} response.APIResponse
+// @Failure 404 {object} response.APIResponse
+// @Failure 500 {object} response.APIResponse
+// @Router /api/v1/tenders/{id}/documents/download [get]
+func (h *TenderHandler) DownloadDocuments(c echo.Context) error {
+	id := c.Param("id")
+	if id == "" {
+		return response.BadRequest(c, "Tender ID is required", "ID parameter cannot be empty")
+	}
+
+	download, err := h.service.DownloadDocuments(c.Request().Context(), id)
+	if err != nil {
+		if errors.Is(err, ErrTenderDocumentNotFound) {
+			return response.NotFound(c, "Tender documents not found")
+		}
+		if errors.Is(err, ErrTenderDocumentStorageUnavailable) {
+			return response.InternalServerError(c, "Tender document storage is unavailable")
+		}
+		return response.InternalServerError(c, "Failed to download tender documents")
+	}
+	defer download.Reader.Close()
+
+	headers := c.Response().Header()
+	headers.Set(echo.HeaderContentDisposition, mime.FormatMediaType("attachment", map[string]string{
+		"filename": download.Filename,
+	}))
+	if download.Size > 0 {
+		headers.Set(echo.HeaderContentLength, strconv.FormatInt(download.Size, 10))
+	}
+
+	return c.Stream(http.StatusOK, download.ContentType, download.Reader)
+}
+
 // GetAISummary retrieves the AI-generated overall summary for a tender
 // @Summary Get AI-generated tender summary
 // @Description Retrieve the AI-generated overall summary for a tender from the AI pipeline storage
@@ -25,6 +25,7 @@ type TenderRepository interface {
 	GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
 	GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
 	GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
+	GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error)
 	Update(ctx context.Context, tender *Tender) error
 	Delete(ctx context.Context, id string) error
 	GetStatistics(ctx context.Context) (*TenderStatistics, error)
@@ -337,6 +338,52 @@ func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, sk
 	return result.Items, result.TotalCount, nil
 }

+// GetUnTranslatedTenders retrieves tenders that don't have a translation for the given language yet.
+func (r *tenderRepository) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) {
+	translationKey := "translations." + language
+	metadataKey := "processing_metadata.translated_data." + language
+
+	filter := bson.M{
+		"notice_publication_id": bson.M{"$ne": ""},
+		"title":                 bson.M{"$ne": ""},
+		"description":           bson.M{"$ne": ""},
+		"$and": []bson.M{
+			{
+				"$or": []bson.M{
+					{translationKey: bson.M{"$exists": false}},
+					{translationKey: nil},
+				},
+			},
+			{
+				"$or": []bson.M{
+					{metadataKey: bson.M{"$exists": false}},
+					{metadataKey: bson.M{"$ne": "done"}},
+				},
+			},
+		},
+	}
+
+	pagination := orm.Pagination{
+		Limit:     limit,
+		Skip:      skip,
+		SortField: "created_at",
+		SortOrder: 1, // Oldest first
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to get untranslated tenders", map[string]interface{}{
+			"language": language,
+			"limit":    limit,
+			"skip":     skip,
+			"error":    err.Error(),
+		})
+		return nil, 0, err
+	}
+
+	return result.Items, result.TotalCount, nil
+}
+
 // GetStatistics calculates tender statistics using aggregation
 func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
 	stats := &TenderStatistics{
@@ -1,9 +1,13 @@
 package tender

 import (
+	"archive/zip"
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"path"
+	"path/filepath"
 	"strings"
 	"time"
 	"tm/internal/company"
@@ -22,6 +26,16 @@ type AISummarizerClient interface {
 // AISummarizerStorage defines the interface for fetching summaries from MinIO storage
 type AISummarizerStorage interface {
 	GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error)
+	ListDocuments(ctx context.Context, noticeID string) ([]ai_summarizer.StoredDocument, error)
+	DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error)
+}
+
+// TenderDocumentDownload contains an authorized document stream.
+type TenderDocumentDownload struct {
+	Reader      io.ReadCloser
+	Filename    string
+	ContentType string
+	Size        int64
 }

 // TenderService interface defines tender business logic operations
@@ -33,6 +47,8 @@ type Service interface {
 	Search(ctx context.Context, req *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
 	Recommend(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
 	GetDocumentSummaries(ctx context.Context, id string) ([]DocumentSummary, error)
+	GetDocuments(ctx context.Context, id string) (*TenderDocumentsResponse, error)
+	DownloadDocuments(ctx context.Context, id string) (*TenderDocumentDownload, error)

 	// AI summarization operations
 	GetAISummary(ctx context.Context, id string) (*AISummaryResponse, error)
@@ -58,6 +74,11 @@ type tenderService struct {

 const aiSummaryEnrichmentTimeout = 2 * time.Second

+var (
+	ErrTenderDocumentStorageUnavailable = errors.New("tender document storage is unavailable")
+	ErrTenderDocumentNotFound           = errors.New("tender document not found")
+)
+
 // NewService creates a new tender service
 func NewService(repository TenderRepository, companyService company.Service, customerService customer.Service, logger logger.Logger, aiClient AISummarizerClient, aiStorage AISummarizerStorage, defaultLanguage string) Service {
 	if strings.TrimSpace(defaultLanguage) == "" {
@@ -241,6 +262,178 @@ func (s *tenderService) GetDocumentSummaries(ctx context.Context, id string) ([]
 	return tender.DocumentSummaries, nil
 }

+// GetDocuments retrieves scraped document metadata for a tender.
+func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDocumentsResponse, error) {
+	if id == "" {
+		return nil, fmt.Errorf("tender ID cannot be empty")
+	}
+	if s.aiSummarizerStorage == nil {
+		return nil, ErrTenderDocumentStorageUnavailable
+	}
+
+	tender, err := s.repository.GetByID(ctx, id)
+	if err != nil {
+		s.logger.Error("Failed to get tender for documents", map[string]interface{}{
+			"tender_id": id,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to get tender: %w", err)
+	}
+	if tender == nil {
+		return nil, fmt.Errorf("tender not found")
+	}
+	if strings.TrimSpace(tender.NoticePublicationID) == "" {
+		return nil, fmt.Errorf("tender notice publication ID is empty")
+	}
+
+	storedDocuments, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
+	if err != nil {
+		s.logger.Error("Failed to list tender documents from storage", map[string]interface{}{
+			"tender_id": id,
+			"notice_id": tender.NoticePublicationID,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to list tender documents: %w", err)
+	}
+
+	documents := make([]TenderDocumentResponse, 0, len(storedDocuments))
+	for _, doc := range storedDocuments {
+		documents = append(documents, TenderDocumentResponse{
+			Filename:     doc.DocumentName,
+			Size:         doc.Size,
+			LastModified: doc.LastModified,
+			DocumentType: doc.DocumentType,
+		})
+	}
+
+	s.logger.Info("Retrieved scraped documents for tender", map[string]interface{}{
+		"tender_id":       id,
+		"notice_id":       tender.NoticePublicationID,
+		"documents_count": len(documents),
+	})
+
+	return &TenderDocumentsResponse{
+		TenderID:  id,
+		Documents: documents,
+	}, nil
+}
+
+// DownloadDocuments returns a ZIP stream containing all documents for a tender.
+func (s *tenderService) DownloadDocuments(ctx context.Context, id string) (*TenderDocumentDownload, error) {
+	if id == "" {
+		return nil, fmt.Errorf("tender ID cannot be empty")
+	}
+	if s.aiSummarizerStorage == nil {
+		return nil, ErrTenderDocumentStorageUnavailable
+	}
+
+	tender, err := s.repository.GetByID(ctx, id)
+	if err != nil {
+		s.logger.Error("Failed to get tender for document download", map[string]interface{}{
+			"tender_id": id,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to get tender: %w", err)
+	}
+	if tender == nil {
+		return nil, fmt.Errorf("tender not found")
+	}
+	if strings.TrimSpace(tender.NoticePublicationID) == "" {
+		return nil, ErrTenderDocumentNotFound
+	}
+
+	documents, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
+	if err != nil {
+		s.logger.Error("Failed to list tender documents for archive", map[string]interface{}{
+			"tender_id": id,
+			"notice_id": tender.NoticePublicationID,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to list tender documents: %w", err)
+	}
+	if len(documents) == 0 {
+		return nil, ErrTenderDocumentNotFound
+	}
+
+	reader, writer := io.Pipe()
+	go s.writeDocumentsArchive(ctx, writer, documents)
+
+	filename := fmt.Sprintf("%s-documents.zip", tender.NoticePublicationID)
+	s.logger.Info("Tender documents archive download prepared", map[string]interface{}{
+		"tender_id":       id,
+		"notice_id":       tender.NoticePublicationID,
+		"documents_count": len(documents),
+	})
+
+	return &TenderDocumentDownload{
+		Reader:      reader,
+		Filename:    filename,
+		ContentType: "application/zip",
+	}, nil
+}
+
+func (s *tenderService) writeDocumentsArchive(ctx context.Context, writer *io.PipeWriter, documents []ai_summarizer.StoredDocument) {
+	zipWriter := zip.NewWriter(writer)
+	usedNames := make(map[string]int, len(documents))
+
+	for _, doc := range documents {
+		select {
+		case <-ctx.Done():
+			zipWriter.Close()
+			writer.CloseWithError(ctx.Err())
+			return
+		default:
+		}
+
+		if err := s.addDocumentToArchive(ctx, zipWriter, usedNames, doc); err != nil {
+			zipWriter.Close()
+			writer.CloseWithError(err)
+			return
+		}
+	}
+
+	if err := zipWriter.Close(); err != nil {
+		writer.CloseWithError(err)
+		return
+	}
+	writer.Close()
+}
+
+func (s *tenderService) addDocumentToArchive(ctx context.Context, zipWriter *zip.Writer, usedNames map[string]int, doc ai_summarizer.StoredDocument) error {
+	reader, err := s.aiSummarizerStorage.DownloadDocument(ctx, doc.ObjectName)
+	if err != nil {
+		return fmt.Errorf("failed to open document %s: %w", doc.DocumentName, err)
+	}
+	defer reader.Close()
+
+	entry, err := zipWriter.Create(uniqueArchiveName(usedNames, doc.DocumentName))
+	if err != nil {
+		return fmt.Errorf("failed to create archive entry for %s: %w", doc.DocumentName, err)
+	}
+
+	if _, err := io.Copy(entry, reader); err != nil {
+		return fmt.Errorf("failed to write document %s to archive: %w", doc.DocumentName, err)
+	}
+	return nil
+}
+
+func uniqueArchiveName(usedNames map[string]int, filename string) string {
+	cleanName := path.Base(filename)
+	if cleanName == "." || cleanName == "/" || cleanName == "" {
+		cleanName = "document"
+	}
+
+	count := usedNames[cleanName]
+	usedNames[cleanName] = count + 1
+	if count == 0 {
+		return cleanName
+	}
+
+	extension := filepath.Ext(cleanName)
+	base := strings.TrimSuffix(cleanName, extension)
+	return fmt.Sprintf("%s-%d%s", base, count+1, extension)
+}
+
 // ListTenders retrieves tenders with pagination and filtering
 func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) {
 	s.logger.Info("Listing tenders", map[string]interface{}{