mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented the integration with scraper python server, tenders summarise, and some fixes.

Browse Source
This commit is contained in:
Mazyar
2025-12-27 12:47:08 +03:30
parent e4da3b5bb2
commit f6bdcc6d7c
18 changed files with 1054 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tender/entity.go
+37 -6
View File
@@ -8,6 +8,28 @@ import (
"tm/pkg/mongo"
)
// DocumentSummary represents a summary of a tender document
type DocumentSummary struct {
DocumentName string `bson:"document_name" json:"document_name"`
ObjectName string `bson:"object_name" json:"object_name"`
BucketName string `bson:"bucket_name" json:"bucket_name"`
Summary string `bson:"summary" json:"summary"`
SummaryLanguage string `bson:"summary_language" json:"summary_language"`
DocumentType string `bson:"document_type" json:"document_type"`
SummarizedAt int64 `bson:"summarized_at" json:"summarized_at"`
SummaryModel string `bson:"summary_model" json:"summary_model"`
Error string `bson:"error,omitempty" json:"error,omitempty"`
}
// ScrapedDocument represents information about a document that was scraped and uploaded to MinIO
type ScrapedDocument struct {
Filename string `bson:"filename" json:"filename"`
ObjectName string `bson:"object_name" json:"object_name"`
BucketName string `bson:"bucket_name" json:"bucket_name"`
Size int64 `bson:"size" json:"size"`
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"`
}
// Tender represents a tender/contract notice entity
type Tender struct {
mongo.Model `bson:",inline"`
@@ -66,6 +88,8 @@ type Tender struct {
SuspensionDate int64 `bson:"suspension_date,omitempty" json:"suspension_date,omitempty"` // Unix milliseconds
Modifications []TenderModification `bson:"modifications,omitempty" json:"modifications,omitempty"`
AwardedEntities []Awarded `bson:"awarded,omitempty" json:"awarded,omitempty"`
ScrapedDocuments []ScrapedDocument `bson:"scraped_documents,omitempty" json:"scraped_documents,omitempty"`
DocumentSummaries []DocumentSummary `bson:"document_summaries,omitempty" json:"document_summaries,omitempty"`
}
// Organization represents organization information
@@ -102,12 +126,19 @@ type SelectionCriterion struct {
// ProcessingMetadata contains metadata about how the tender was processed
type ProcessingMetadata struct {
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds
ProcessedAt int64 `bson:"processed_at" json:"processed_at"` // Unix milliseconds
ProcessingVersion string `bson:"processing_version" json:"processing_version"`
ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds
ProcessedAt int64 `bson:"processed_at" json:"processed_at"` // Unix milliseconds
ProcessingVersion string `bson:"processing_version" json:"processing_version"`
ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
TranslatedData map[string]string `bson:"translated_data,omitempty" json:"translated_data,omitempty"`
TranslatedAt int64 `bson:"translated_at,omitempty" json:"translated_at,omitempty"`
Processed bool `bson:"processed" json:"processed"`
DocumentsScraped bool `bson:"documents_scraped" json:"documents_scraped"` // Whether documents have been scraped
DocumentsScrapedAt int64 `bson:"documents_scraped_at" json:"documents_scraped_at"` // When documents were scraped
DocumentsSummarized bool `bson:"documents_summarized" json:"documents_summarized"` // Whether documents have been summarized
DocumentsSummarizedAt int64 `bson:"documents_summarized_at" json:"documents_summarized_at"` // When documents were summarized
}
// TenderModification represents a modification made to a tender