mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added missing tender data to response

Browse Source
This commit is contained in:
Mazyar
2026-05-13 01:07:43 +03:30
parent c518cd3afb
commit 83c8ece91f
11 changed files with 1383 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/document_scraper/form.go
+64 -5
View File
@@ -11,6 +11,46 @@ type DocumentScraperGetRequest struct {
NoticeID string `param:"notice_id" valid:"required~Notice ID is required"`
}
// DocumentScraperBuyerResponse is buyer/contact and location data exposed to the document scraper.
type DocumentScraperBuyerResponse struct {
BuyerName string `json:"buyer_name"`
BuyerEmail string `json:"buyer_email"`
BuyerCountry string `json:"buyer_country"`
BuyerRegionNuts string `json:"buyer_region_nuts"`
BuyerCity string `json:"buyer_city"`
BuyerProfileURL string `json:"buyer_profile_url"`
}
// DocumentScraperLotResponse is one procurement lot for the scraper task payload.
type DocumentScraperLotResponse struct {
LotID string `json:"lot_id,omitempty"`
LotTitle string `json:"lot_title"`
LotDescription string `json:"lot_description"`
LotMainNature string `json:"lot_main_nature"`
LotMainClassificationCode string `json:"lot_main_classification_code"`
LotMainClassificationDescription string `json:"lot_main_classification_description"`
LotMainClassificationDisplay string `json:"lot_main_classification_display,omitempty"`
AdditionalClassifications []string `json:"additional_classifications,omitempty"`
LotDuration string `json:"lot_duration"`
LotEstimatedValueExclVAT float64 `json:"lot_estimated_value_excl_vat"`
LotEstimatedValueCurrency string `json:"lot_estimated_value_currency,omitempty"`
}
// DocumentScraperWinnerResponse is a contract winner / winning tenderer for result notices.
type DocumentScraperWinnerResponse struct {
OfficialName string `json:"official_name"`
RegistrationNumber string `json:"registration_number"`
PostalAddress string `json:"postal_address"`
Town string `json:"town"`
Postcode string `json:"postcode"`
CountrySubdivisionNuts string `json:"country_subdivision_nuts"`
Country string `json:"country"`
ContactPoint string `json:"contact_point"`
Email string `json:"email"`
Telephone string `json:"telephone"`
LotID string `json:"lot_id,omitempty"`
}
// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service.
//
// ContractFolderID is the TED procedure identifier (BT-04 / ContractFolderID),
@@ -18,11 +58,30 @@ type DocumentScraperGetRequest struct {
//
// PROC_<contract_folder_id>/<notice_publication_id>/<file>
type DocumentScraperTenderResponse struct {
ContractFolderID string `json:"contract_folder_id"`
NoticePublicationID string `json:"notice_publication_id"`
DocumentURL string `json:"document_url"`
Title string `json:"title"`
Description string `json:"description"`
ContractFolderID string `json:"contract_folder_id"`
NoticePublicationID string `json:"notice_publication_id"`
DocumentURL string `json:"document_url"`
Title string `json:"title"`
Description string `json:"description"`
NoticeIdentifier string `json:"notice_identifier"`
NoticeVersion string `json:"notice_version"`
FormType string `json:"form_type"`
NoticeType string `json:"notice_type"`
NoticeSubtype string `json:"notice_subtype"`
PublicationDate int64 `json:"publication_date"`
NoticeDispatchDateWithTimezone string `json:"notice_dispatch_date_with_timezone"`
ProcurementTypeCode string `json:"procurement_type_code"`
MainClassification string `json:"main_classification"`
MainClassificationDescription string `json:"main_classification_description"`
MainClassificationDisplay string `json:"main_classification_display,omitempty"`
AdditionalClassifications []string `json:"additional_classifications,omitempty"`
EstimatedValue float64 `json:"estimated_value"`
EstimatedValueCurrency string `json:"estimated_value_currency,omitempty"`
EstimatedValueVATBasis string `json:"estimated_value_vat_basis"`
Status string `json:"status"`
Buyer DocumentScraperBuyerResponse `json:"buyer"`
Lots []DocumentScraperLotResponse `json:"lots"`
Winners []DocumentScraperWinnerResponse `json:"winners"`
}
// DocumentScraperListResponse represents the paginated list response
internal/document_scraper/handler.go
+2 -2
View File
@@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler {
// ListPendingTenders retrieves tenders that need document scraping
// @Summary List pending tenders for document scraping
// @Description Retrieve tenders that have not yet been scraped for documents, including notice ID, title, description
// @Description Retrieve tenders that have not yet been scraped for documents. Each item includes notice identifiers, dispatch time (RFC3339), procurement lots, buyer, winners (when present), CPV fields, and value metadata.
// @Tags Document-Scraper
// @Produce json
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
@@ -62,7 +62,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
// @Summary Get tender by notice ID for document scraping
// @Description Retrieve a specific tender by its notice publication ID, including notice ID, title, description
// @Description Retrieve a specific tender by its notice publication ID with full notice, lot, buyer, and winner fields for the document scraper pipeline.
// @Tags Document-Scraper
// @Produce json
// @Param notice_id path string true "Notice Publication ID"
internal/document_scraper/service.go
+2 -21
View File
@@ -7,7 +7,6 @@ import (
"tm/internal/tender"
"tm/pkg/logger"
"tm/pkg/response"
"tm/ted"
)
// Service defines business logic for the document scraper API
@@ -20,7 +19,6 @@ type Service interface {
type service struct {
tenderRepo tender.TenderRepository
tedParser *ted.TEDParser
logger logger.Logger
}
@@ -28,27 +26,10 @@ type service struct {
func NewService(tenderRepo tender.TenderRepository, logger logger.Logger) Service {
return &service{
tenderRepo: tenderRepo,
tedParser: ted.NewTEDParser(),
logger: logger,
}
}
// buildTenderResponse creates a DocumentScraperTenderResponse from a tender
func (s *service) buildTenderResponse(_ context.Context, t *tender.Tender) *DocumentScraperTenderResponse {
documentURL := t.DocumentURI
if documentURL == "" {
documentURL = t.TenderURL
}
return &DocumentScraperTenderResponse{
ContractFolderID: t.ContractFolderID,
NoticePublicationID: t.NoticePublicationID,
DocumentURL: documentURL,
Title: t.Title,
Description: t.Description,
}
}
// ListPendingTenders retrieves tenders that have not yet been scraped for documents.
// Only returns tenders from Sweden with active deadlines (deadline not yet reached).
func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) {
@@ -107,7 +88,7 @@ func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*D
}
for i := range filteredTenders {
result.Tenders[i] = *s.buildTenderResponse(ctx, &filteredTenders[i])
result.Tenders[i] = *BuildDocumentScraperTenderResponse(&filteredTenders[i])
}
// Build pagination metadata
@@ -160,7 +141,7 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do
return nil, fmt.Errorf("tender not found")
}
result := s.buildTenderResponse(ctx, t)
result := BuildDocumentScraperTenderResponse(t)
s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{
"notice_id": noticeID,
internal/document_scraper/tender_response.go
+233
View File
@@ -0,0 +1,233 @@
package document_scraper
import (
"strings"
"time"
"tm/internal/tender"
)
const (
enrichmentMainCPVDescription = "main_classification_description"
vatBasisUnspecified = "unspecified"
)
// BuildDocumentScraperTenderResponse maps a stored tender to the scraper API contract.
func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTenderResponse {
if t == nil {
return &DocumentScraperTenderResponse{
Lots: []DocumentScraperLotResponse{},
Winners: []DocumentScraperWinnerResponse{},
}
}
documentURL := strings.TrimSpace(t.DocumentURI)
if documentURL == "" {
documentURL = strings.TrimSpace(t.TenderURL)
}
mainDesc := firstNonEmpty(
strings.TrimSpace(t.ProcessingMetadata.EnrichmentData[enrichmentMainCPVDescription]),
strings.TrimSpace(t.MainClassificationDescription),
)
resp := &DocumentScraperTenderResponse{
ContractFolderID: t.ContractFolderID,
NoticePublicationID: t.NoticePublicationID,
DocumentURL: documentURL,
Title: t.Title,
Description: t.Description,
NoticeIdentifier: strings.TrimSpace(t.NoticeIdentifier),
NoticeVersion: strings.TrimSpace(t.NoticeVersion),
FormType: strings.TrimSpace(t.FormType),
NoticeType: strings.TrimSpace(t.NoticeTypeCode),
NoticeSubtype: strings.TrimSpace(t.NoticeSubTypeCode),
PublicationDate: t.PublicationDate,
NoticeDispatchDateWithTimezone: formatNoticeDispatchRFC3339(t),
ProcurementTypeCode: strings.TrimSpace(t.ProcurementTypeCode),
MainClassification: strings.TrimSpace(t.MainClassification),
MainClassificationDescription: mainDesc,
MainClassificationDisplay: classificationDisplay(strings.TrimSpace(t.MainClassification), mainDesc),
AdditionalClassifications: append([]string(nil), t.AdditionalClassifications...),
EstimatedValue: t.EstimatedValue,
EstimatedValueCurrency: strings.TrimSpace(t.Currency),
EstimatedValueVATBasis: vatBasisUnspecified,
Status: string(t.Status),
Buyer: buildBuyer(t),
Lots: buildLots(t, mainDesc),
Winners: buildWinners(t),
}
return resp
}
func formatNoticeDispatchRFC3339(t *tender.Tender) string {
sec := t.NoticeDispatchAt
if sec == 0 {
sec = t.ESenderDispatchAt
}
if sec == 0 {
return ""
}
return time.Unix(sec, 0).UTC().Format(time.RFC3339)
}
func buildBuyer(t *tender.Tender) DocumentScraperBuyerResponse {
if t.BuyerOrganization == nil {
return DocumentScraperBuyerResponse{}
}
b := t.BuyerOrganization
return DocumentScraperBuyerResponse{
BuyerName: strings.TrimSpace(b.Name),
BuyerEmail: strings.TrimSpace(b.ContactEmail),
BuyerCountry: strings.TrimSpace(b.Address.CountryCode),
BuyerRegionNuts: strings.TrimSpace(b.Address.CountrySubentityCode),
BuyerCity: strings.TrimSpace(b.Address.CityName),
BuyerProfileURL: strings.TrimSpace(firstNonEmpty(t.BuyerProfileURL, b.WebsiteURI)),
}
}
func firstNonEmpty(values ...string) string {
for _, v := range values {
if s := strings.TrimSpace(v); s != "" {
return s
}
}
return ""
}
func classificationDisplay(code, description string) string {
code = strings.TrimSpace(code)
description = strings.TrimSpace(description)
if code != "" && description != "" {
return code + " - " + description
}
if code != "" {
return code
}
return description
}
func buildLots(t *tender.Tender, globalMainDesc string) []DocumentScraperLotResponse {
if len(t.ProcurementLots) == 0 {
return []DocumentScraperLotResponse{}
}
out := make([]DocumentScraperLotResponse, 0, len(t.ProcurementLots))
for _, lot := range t.ProcurementLots {
lotCode := strings.TrimSpace(lot.MainClassification)
perLotDesc := strings.TrimSpace(lot.MainClassificationDescription)
if perLotDesc == "" && lotCode != "" && lotCode == strings.TrimSpace(t.MainClassification) {
perLotDesc = globalMainDesc
}
out = append(out, DocumentScraperLotResponse{
LotID: strings.TrimSpace(lot.LotID),
LotTitle: strings.TrimSpace(lot.Title),
LotDescription: strings.TrimSpace(lot.Description),
LotMainNature: strings.TrimSpace(lot.MainNatureOfContract),
LotMainClassificationCode: lotCode,
LotMainClassificationDescription: perLotDesc,
LotMainClassificationDisplay: classificationDisplay(lotCode, perLotDesc),
AdditionalClassifications: append([]string(nil), lot.AdditionalClassifications...),
LotDuration: strings.TrimSpace(lot.Duration),
LotEstimatedValueExclVAT: lot.EstimatedValue,
LotEstimatedValueCurrency: strings.TrimSpace(lot.Currency),
})
}
return out
}
func buildWinners(t *tender.Tender) []DocumentScraperWinnerResponse {
if len(t.AwardedEntities) > 0 {
out := make([]DocumentScraperWinnerResponse, 0, len(t.AwardedEntities))
for _, a := range t.AwardedEntities {
w := DocumentScraperWinnerResponse{
OfficialName: strings.TrimSpace(a.Name),
RegistrationNumber: strings.TrimSpace(a.CompanyID),
PostalAddress: strings.TrimSpace(a.Address),
Country: strings.TrimSpace(a.Country),
LotID: strings.TrimSpace(a.LotID),
Town: "",
Postcode: "",
CountrySubdivisionNuts: "",
ContactPoint: "",
Email: "",
Telephone: "",
}
if org := orgByID(t, a.OrganizationID); org != nil {
mergeWinnerFromOrganization(&w, org)
}
out = append(out, w)
}
return out
}
if t.WinningTenderer == nil {
return []DocumentScraperWinnerResponse{}
}
w := t.WinningTenderer
if strings.TrimSpace(w.Name) == "" {
return []DocumentScraperWinnerResponse{}
}
addr := w.Address
return []DocumentScraperWinnerResponse{{
OfficialName: strings.TrimSpace(w.Name),
RegistrationNumber: strings.TrimSpace(w.CompanyID),
PostalAddress: strings.TrimSpace(addr.StreetName),
Town: strings.TrimSpace(addr.CityName),
Postcode: strings.TrimSpace(addr.PostalZone),
CountrySubdivisionNuts: strings.TrimSpace(addr.CountrySubentityCode),
Country: strings.TrimSpace(addr.CountryCode),
ContactPoint: strings.TrimSpace(w.ContactName),
Email: strings.TrimSpace(w.ContactEmail),
Telephone: strings.TrimSpace(w.ContactTelephone),
}}
}
func orgByID(t *tender.Tender, id string) *tender.Organization {
id = strings.TrimSpace(id)
if id == "" || t == nil {
return nil
}
for i := range t.Organizations {
if strings.TrimSpace(t.Organizations[i].ID) == id {
return &t.Organizations[i]
}
}
return nil
}
func mergeWinnerFromOrganization(w *DocumentScraperWinnerResponse, org *tender.Organization) {
if w == nil || org == nil {
return
}
if w.OfficialName == "" {
w.OfficialName = strings.TrimSpace(org.Name)
}
if w.RegistrationNumber == "" {
w.RegistrationNumber = strings.TrimSpace(org.CompanyID)
}
if w.PostalAddress == "" {
w.PostalAddress = strings.TrimSpace(org.Address.StreetName)
}
if w.Town == "" {
w.Town = strings.TrimSpace(org.Address.CityName)
}
if w.Postcode == "" {
w.Postcode = strings.TrimSpace(org.Address.PostalZone)
}
if w.CountrySubdivisionNuts == "" {
w.CountrySubdivisionNuts = strings.TrimSpace(org.Address.CountrySubentityCode)
}
if w.Country == "" {
w.Country = strings.TrimSpace(org.Address.CountryCode)
}
if w.ContactPoint == "" {
w.ContactPoint = strings.TrimSpace(org.ContactName)
}
if w.Email == "" {
w.Email = strings.TrimSpace(org.ContactEmail)
}
if w.Telephone == "" {
w.Telephone = strings.TrimSpace(org.ContactTelephone)
}
}