mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented the integration with scraper python server, tenders summarise, and some fixes.

Browse Source
This commit is contained in:
Mazyar
2025-12-27 12:47:08 +03:30
parent e4da3b5bb2
commit f6bdcc6d7c
18 changed files with 1054 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tender/entity.go
+37 -6
View File
@@ -8,6 +8,28 @@ import (
"tm/pkg/mongo"
)
// DocumentSummary represents a summary of a tender document
type DocumentSummary struct {
DocumentName string `bson:"document_name" json:"document_name"`
ObjectName string `bson:"object_name" json:"object_name"`
BucketName string `bson:"bucket_name" json:"bucket_name"`
Summary string `bson:"summary" json:"summary"`
SummaryLanguage string `bson:"summary_language" json:"summary_language"`
DocumentType string `bson:"document_type" json:"document_type"`
SummarizedAt int64 `bson:"summarized_at" json:"summarized_at"`
SummaryModel string `bson:"summary_model" json:"summary_model"`
Error string `bson:"error,omitempty" json:"error,omitempty"`
}
// ScrapedDocument represents information about a document that was scraped and uploaded to MinIO
type ScrapedDocument struct {
Filename string `bson:"filename" json:"filename"`
ObjectName string `bson:"object_name" json:"object_name"`
BucketName string `bson:"bucket_name" json:"bucket_name"`
Size int64 `bson:"size" json:"size"`
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"`
}
// Tender represents a tender/contract notice entity
type Tender struct {
mongo.Model `bson:",inline"`
@@ -66,6 +88,8 @@ type Tender struct {
SuspensionDate int64 `bson:"suspension_date,omitempty" json:"suspension_date,omitempty"` // Unix milliseconds
Modifications []TenderModification `bson:"modifications,omitempty" json:"modifications,omitempty"`
AwardedEntities []Awarded `bson:"awarded,omitempty" json:"awarded,omitempty"`
ScrapedDocuments []ScrapedDocument `bson:"scraped_documents,omitempty" json:"scraped_documents,omitempty"`
DocumentSummaries []DocumentSummary `bson:"document_summaries,omitempty" json:"document_summaries,omitempty"`
}
// Organization represents organization information
@@ -102,12 +126,19 @@ type SelectionCriterion struct {
// ProcessingMetadata contains metadata about how the tender was processed
type ProcessingMetadata struct {
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds
ProcessedAt int64 `bson:"processed_at" json:"processed_at"` // Unix milliseconds
ProcessingVersion string `bson:"processing_version" json:"processing_version"`
ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds
ProcessedAt int64 `bson:"processed_at" json:"processed_at"` // Unix milliseconds
ProcessingVersion string `bson:"processing_version" json:"processing_version"`
ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
TranslatedData map[string]string `bson:"translated_data,omitempty" json:"translated_data,omitempty"`
TranslatedAt int64 `bson:"translated_at,omitempty" json:"translated_at,omitempty"`
Processed bool `bson:"processed" json:"processed"`
DocumentsScraped bool `bson:"documents_scraped" json:"documents_scraped"` // Whether documents have been scraped
DocumentsScrapedAt int64 `bson:"documents_scraped_at" json:"documents_scraped_at"` // When documents were scraped
DocumentsSummarized bool `bson:"documents_summarized" json:"documents_summarized"` // Whether documents have been summarized
DocumentsSummarizedAt int64 `bson:"documents_summarized_at" json:"documents_summarized_at"` // When documents were summarized
}
// TenderModification represents a modification made to a tender
internal/tender/handler.go
+29
View File
@@ -312,3 +312,32 @@ func (h *TenderHandler) GetPublicTenderDetails(c echo.Context) error {
return response.Success(c, tender, "Tender details retrieved successfully")
}
// GetDocumentSummaries retrieves document summaries for a tender
// @Summary Get document summaries for a tender
// @Description Retrieve AI-generated summaries of documents for a specific tender
// @Tags Tenders
// @Produce json
// @Param id path string true "Tender ID"
// @Success 200 {object} response.APIResponse{data=[]DocumentSummary}
// @Failure 400 {object} response.APIResponse
// @Failure 404 {object} response.APIResponse
// @Failure 500 {object} response.APIResponse
// @Router /api/v1/tenders/{id}/document-summaries [get]
func (h *TenderHandler) GetDocumentSummaries(c echo.Context) error {
id := c.Param("id")
if id == "" {
return response.BadRequest(c, "Tender ID is required", "ID parameter cannot be empty")
}
summaries, err := h.service.GetDocumentSummaries(c.Request().Context(), id)
if err != nil {
h.logger.Error("Failed to get document summaries", map[string]interface{}{
"tender_id": id,
"error": err.Error(),
})
return response.NotFound(c, "Document summaries not found")
}
return response.Success(c, summaries, "Document summaries retrieved successfully")
}
internal/tender/repository.go
+62
View File
@@ -23,6 +23,8 @@ type TenderRepository interface {
GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) ([]Tender, int64, error)
GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
Update(ctx context.Context, tender *Tender) error
Delete(ctx context.Context, id string) error
GetStatistics(ctx context.Context) (*TenderStatistics, error)
@@ -275,6 +277,66 @@ func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int
return result.Items, nil
}
// GetUnScrapedTenders retrieves tenders that haven't been processed for document scraping
func (r *tenderRepository) GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
filter := bson.M{
"$or": []bson.M{
{"processing_metadata.documents_scraped": bson.M{"$exists": false}},
{"processing_metadata.documents_scraped": false},
{"processing_metadata.documents_scraped": nil},
},
}
pagination := orm.Pagination{
Limit: limit,
Skip: skip,
SortField: "created_at",
SortOrder: 1, // Oldest first
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to get un-scraped tenders", map[string]interface{}{
"limit": limit,
"skip": skip,
"error": err.Error(),
})
return nil, 0, err
}
return result.Items, result.TotalCount, nil
}
// GetUnSummarizedTenders retrieves tenders that have documents scraped but not summarized
func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
filter := bson.M{
"processing_metadata.documents_scraped": true,
"$or": []bson.M{
{"processing_metadata.documents_summarized": bson.M{"$exists": false}},
{"processing_metadata.documents_summarized": false},
},
}
pagination := orm.Pagination{
Limit: limit,
Skip: skip,
SortField: "processing_metadata.documents_scraped_at",
SortOrder: 1, // Oldest first (process oldest scraped documents first)
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
"limit": limit,
"skip": skip,
"error": err.Error(),
})
return nil, 0, err
}
return result.Items, result.TotalCount, nil
}
// GetStatistics calculates tender statistics using aggregation
func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
stats := &TenderStatistics{
internal/tender/service.go
+34
View File
@@ -17,6 +17,7 @@ type Service interface {
Delete(ctx context.Context, id string) error
Search(ctx context.Context, req *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
Recommend(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
GetDocumentSummaries(ctx context.Context, id string) ([]DocumentSummary, error)
// Maintenance operations
UpdateExpiredTenders(ctx context.Context) error
@@ -129,6 +130,39 @@ func (s *tenderService) Delete(ctx context.Context, id string) error {
return nil
}
// GetDocumentSummaries retrieves document summaries for a specific tender
func (s *tenderService) GetDocumentSummaries(ctx context.Context, id string) ([]DocumentSummary, error) {
if id == "" {
return nil, fmt.Errorf("tender ID cannot be empty")
}
// Get tender from repository
tender, err := s.repository.GetByID(ctx, id)
if err != nil {
s.logger.Error("Failed to get tender for document summaries", map[string]interface{}{
"tender_id": id,
"error": err.Error(),
})
return nil, fmt.Errorf("failed to get tender: %w", err)
}
if tender == nil {
return nil, fmt.Errorf("tender not found")
}
// Return document summaries (empty slice if none exist)
if tender.DocumentSummaries == nil {
return []DocumentSummary{}, nil
}
s.logger.Info("Retrieved document summaries for tender", map[string]interface{}{
"tender_id": id,
"summaries_count": len(tender.DocumentSummaries),
})
return tender.DocumentSummaries, nil
}
// ListTenders retrieves tenders with pagination and filtering
func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) {
s.logger.Info("Listing tenders", map[string]interface{}{