mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Tender Management to Introduce Notice Entity and Repository

Browse Source 
- Replaced the tender repository with a new notice repository, encapsulating notice-related data access methods.
- Introduced the Notice entity to represent tender/contract notices, including relevant fields and methods for managing notice data.
- Updated the TED scraper to utilize the new notice repository for creating and managing notices, enhancing the integration with the tender management system.
- Implemented Ollama SDK initialization in the web bootstrap process, allowing for improved AI interactions.
- Enhanced the tender service to include Ollama SDK for additional functionality, ensuring a more robust service layer.
This commit is contained in:
n.nakhostin
2025-10-04 15:22:49 +03:30
parent 154610e2a0
commit cc3d6163ed
15 changed files with 1088 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ted/mapper.go
+21 -21
View File
@@ -4,11 +4,11 @@ import (
"strconv"
"strings"
"time"
"tm/internal/tender"
"tm/internal/notice"
)
// mapToTenderFromParsedDoc maps a parsed TED document to tender entity
func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *tender.Tender {
func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice {
switch parsedDoc.Type {
case DocumentTypeContractNotice:
if parsedDoc.ContractNotice == nil {
@@ -45,7 +45,7 @@ func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceF
}
// mapGenericNoticeToTender maps any TED document type to tender entity using the common interface
func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *tender.Tender {
func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *notice.Notice {
if doc == nil {
s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{})
return nil
@@ -53,7 +53,7 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so
now := time.Now().Unix()
t := &tender.Tender{
t := &notice.Notice{
ContractNoticeID: doc.GetID(),
ContractFolderID: "", // Not available in generic interface
NoticeTypeCode: doc.GetNoticeType(),
@@ -68,11 +68,11 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so
// Basic tender information
Title: "Generic Notice " + doc.GetNoticeType(),
Description: "Processed from " + doc.GetNoticeType() + " document",
Status: tender.TenderStatusActive,
Source: tender.TenderSourceTEDScraper,
Status: notice.TenderStatusActive,
Source: notice.TenderSourceTEDScraper,
SourceFileURL: sourceFileURL,
SourceFileName: sourceFileName,
ProcessingMetadata: tender.ProcessingMetadata{
ProcessingMetadata: notice.ProcessingMetadata{
ScrapedAt: now,
ProcessedAt: now,
ProcessingVersion: "1.0",
@@ -86,10 +86,10 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so
}
// mapToTender maps a TED ContractNotice to tender entity
func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *tender.Tender {
func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *notice.Notice {
now := time.Now().Unix()
t := &tender.Tender{
t := &notice.Notice{
ContractNoticeID: cn.ID,
ContractFolderID: cn.ContractFolderID,
NoticeTypeCode: cn.GetNoticeSubType(),
@@ -105,11 +105,11 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
MainClassification: cn.GetMainClassification(),
PlaceOfPerformance: cn.GetPlaceOfPerformance(),
DocumentURI: cn.GetDocumentURI(),
Status: tender.TenderStatusActive,
Source: tender.TenderSourceTEDScraper,
Status: notice.TenderStatusActive,
Source: notice.TenderSourceTEDScraper,
SourceFileURL: sourceFileURL,
SourceFileName: sourceFileName,
ProcessingMetadata: tender.ProcessingMetadata{
ProcessingMetadata: notice.ProcessingMetadata{
ScrapedAt: now,
ProcessedAt: now,
ProcessingVersion: "1.0",
@@ -215,7 +215,7 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
// Set selection criteria
if criteria := cn.GetSelectionCriteria(); criteria != nil {
for _, criterion := range criteria {
t.SelectionCriteria = append(t.SelectionCriteria, tender.SelectionCriterion{
t.SelectionCriteria = append(t.SelectionCriteria, notice.SelectionCriterion{
TypeCode: criterion.CriterionTypeCode,
Description: criterion.Description,
LanguageID: criterion.LanguageID,
@@ -235,10 +235,10 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
}
// mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity
func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *tender.Tender {
func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *notice.Notice {
now := time.Now().Unix()
t := &tender.Tender{
t := &notice.Notice{
ContractNoticeID: can.ID,
ContractFolderID: can.ContractFolderID,
NoticeTypeCode: can.NoticeTypeCode,
@@ -252,11 +252,11 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
ProcurementTypeCode: can.GetContractNature(),
ProcedureCode: can.GetProcedureCode(),
MainClassification: can.GetMainClassification(),
Status: tender.TenderStatusAwarded, // Award notices are for completed tenders
Source: tender.TenderSourceTEDScraper,
Status: notice.TenderStatusAwarded, // Award notices are for completed tenders
Source: notice.TenderSourceTEDScraper,
SourceFileURL: sourceFileURL,
SourceFileName: sourceFileName,
ProcessingMetadata: tender.ProcessingMetadata{
ProcessingMetadata: notice.ProcessingMetadata{
ScrapedAt: now,
ProcessedAt: now,
ProcessingVersion: "1.0",
@@ -315,13 +315,13 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
}
// mapOrganization maps a TED Organization to tender Organization
func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender.Organization {
func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.Organization {
if tedOrg == nil || tedOrg.Company == nil {
return nil
}
company := tedOrg.Company
org := &tender.Organization{
org := &notice.Organization{
Role: role,
}
@@ -351,7 +351,7 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender.
// Set address
if company.PostalAddress != nil {
org.Address = tender.Address{
org.Address = notice.Address{
StreetName: company.PostalAddress.StreetName,
CityName: company.PostalAddress.CityName,
PostalZone: company.PostalAddress.PostalZone,
ted/scraper.go
+52 -52
View File
@@ -10,7 +10,7 @@ import (
"path/filepath"
"strings"
"time"
"tm/internal/tender"
"tm/internal/notice"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
@@ -37,7 +37,7 @@ type TEDScraper struct {
httpClient *http.Client
xmlParser *TEDParser
mongoManager *mongo.ConnectionManager
tenderRepo tender.TenderRepository
tenderRepo notice.Repository
}
// NewTEDScraper creates a new TED scraper instance
@@ -45,7 +45,7 @@ func NewTEDScraper(
config *Config,
logger logger.Logger,
mongoManager *mongo.ConnectionManager,
tenderRepo tender.TenderRepository,
tenderRepo notice.Repository,
notify notification.SDK,
) *TEDScraper {
httpClient := &http.Client{
@@ -275,7 +275,7 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
return result, nil
}
func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*tender.Tender, error) {
func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) {
existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID)
if err != nil {
if err.Error() == "document not found" {
@@ -308,60 +308,60 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
// Map to tender entity
t := s.mapToTenderFromParsedDoc(notice, "", fileName)
existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID)
// existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID)
// if err != nil {
// s.logger.Warn("Error checking for existing tender", map[string]interface{}{
// "contract_notice_id": t.ContractNoticeID,
// "error": err.Error(),
// })
// }
// if err == nil && existingTender != nil {
// // Update existing tender
// s.logger.Info("Found existing tender, updating", map[string]interface{}{
// "existing_tender_id": existingTender.ID.Hex(),
// "contract_notice_id": t.ContractNoticeID,
// })
// t.ID = existingTender.ID
// t.UpdatedAt = time.Now().Unix()
// err = s.tenderRepo.Update(ctx, t)
// if err != nil {
// s.logger.Error("Failed to update existing tender", map[string]interface{}{
// "tender_id": t.ID.Hex(),
// "error": err.Error(),
// })
// return fmt.Errorf("failed to update existing tender: %w", err)
// }
// s.logger.Info("Successfully updated tender", map[string]interface{}{
// "tender_id": t.ID.Hex(),
// })
// } else {
// // Create new tender
s.logger.Info("Creating new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"tender_id": t.TenderID,
})
t.CreatedAt = time.Now().Unix()
t.UpdatedAt = t.CreatedAt
err = s.tenderRepo.Import(ctx, t)
if err != nil {
s.logger.Warn("Error checking for existing tender", map[string]interface{}{
s.logger.Error("Failed to create new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"tender_id": t.TenderID,
"error": err.Error(),
})
return fmt.Errorf("failed to create tender: %w", err)
}
if err == nil && existingTender != nil {
// Update existing tender
s.logger.Info("Found existing tender, updating", map[string]interface{}{
"existing_tender_id": existingTender.ID.Hex(),
"contract_notice_id": t.ContractNoticeID,
})
t.ID = existingTender.ID
t.UpdatedAt = time.Now().Unix()
err = s.tenderRepo.Update(ctx, t)
if err != nil {
s.logger.Error("Failed to update existing tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"error": err.Error(),
})
return fmt.Errorf("failed to update existing tender: %w", err)
}
s.logger.Info("Successfully updated tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
})
} else {
// Create new tender
s.logger.Info("Creating new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"tender_id": t.TenderID,
})
t.CreatedAt = time.Now().Unix()
t.UpdatedAt = t.CreatedAt
err = s.tenderRepo.Create(ctx, t)
if err != nil {
s.logger.Error("Failed to create new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"tender_id": t.TenderID,
"error": err.Error(),
})
return fmt.Errorf("failed to create tender: %w", err)
}
s.logger.Info("Successfully created new tender", map[string]interface{}{
"tender_id": t.TenderID,
"contract_notice_id": t.ContractNoticeID,
})
}
s.logger.Info("Successfully created new tender", map[string]interface{}{
"tender_id": t.TenderID,
"contract_notice_id": t.ContractNoticeID,
})
// }
// Log detailed parsed information based on notice type
s.logger.Info("Successfully parsed notice", map[string]interface{}{
ted/service.go
+2 -2
View File
@@ -5,7 +5,7 @@ import (
"strconv"
"strings"
"time"
"tm/internal/tender"
"tm/internal/notice"
)
// parseDate parses various date formats used in TED XML
@@ -146,7 +146,7 @@ func GenerateTenderURL(noticePublicationID string) string {
// generateTenderID generates a unique tender ID using the format:
// {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
func GenerateTenderID(t *tender.Tender) string {
func GenerateTenderID(t *notice.Notice) string {
var parts []string
// Source (TED)