mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge tenders which have the same procedure id - worker performance fix

Browse Source
This commit is contained in:
Mazyar
2026-05-11 15:10:01 +03:30
parent 8ba667e4f4
commit b4efb97f47
22 changed files with 661 additions and 2234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/customer/handler.go
+1 -1
View File
@@ -403,7 +403,7 @@ func (h *Handler) GetProfile(c echo.Context) error {
// UpdateProfileKeywords updates customer profile keywords (Mobile)
// @Summary Update customer profile keywords
// @Description Update profile keywords manually and trigger asynchronous GLM enhancement based on customer company information (including country).
// @Description Update profile keywords manually; optional async keyword refresh is reserved for the AI service.
// @Tags Authorization
// @Accept json
// @Produce json
internal/customer/service.go
+20 -119
View File
@@ -11,7 +11,6 @@ import (
"time"
"tm/internal/company"
"tm/pkg/glm"
"tm/pkg/logger"
"tm/pkg/notification"
"tm/pkg/redis"
@@ -63,11 +62,10 @@ type customerService struct {
redisClient redis.Client
notification notification.SDK
validator ValidationService
glmSDK *glm.SDK
}
// New creates a new customer service
func NewService(repository Repository, logger logger.Logger, authService authorization.AuthorizationService, companyService company.Service, redisClient redis.Client, notificationSDK notification.SDK, validator ValidationService, glmSDK *glm.SDK) Service {
func NewService(repository Repository, logger logger.Logger, authService authorization.AuthorizationService, companyService company.Service, redisClient redis.Client, notificationSDK notification.SDK, validator ValidationService) Service {
return &customerService{
repository: repository,
logger: logger,
@@ -76,7 +74,6 @@ func NewService(repository Repository, logger logger.Logger, authService authori
redisClient: redisClient,
notification: notificationSDK,
validator: validator,
glmSDK: glmSDK,
}
}
@@ -134,7 +131,7 @@ func (s *customerService) Register(ctx context.Context, form *CreateCustomerForm
Phone: form.Phone,
}
// Generate keywords using GLM AI
// Keyword auto-generation is handled by the AI service when integrated; backend leaves keywords empty on register.
customer.KeywordsStatus = KeywordsStatusInProgress
keywords, err := s.generateKeywords(ctx, customer)
if err != nil {
@@ -1349,99 +1346,11 @@ func (s *customerService) AssignRole(ctx context.Context, customerID string, for
}, nil
}
// generateKeywords generates keywords for a customer using GLM AI based on their information
// generateKeywords is reserved for future integration with the AI service; the backend does not generate keywords.
func (s *customerService) generateKeywords(ctx context.Context, customer *Customer) ([]string, error) {
// If GLM SDK is not available, return empty keywords
if s.glmSDK == nil {
s.logger.Warn("GLM SDK not available, skipping keyword generation", map[string]interface{}{
"customer_id": customer.ID,
})
return []string{}, nil
}
// Build customer information string for AI
var customerInfo strings.Builder
if customer.FullName != nil {
customerInfo.WriteString(fmt.Sprintf("Name: %s\n", *customer.FullName))
}
customerInfo.WriteString(fmt.Sprintf("Type: %s\n", customer.Type))
customerInfo.WriteString(fmt.Sprintf("Role: %s\n", customer.Role))
if customer.Phone != nil {
customerInfo.WriteString(fmt.Sprintf("Phone: %s\n", *customer.Phone))
}
// Get company information if available
if len(customer.Companies) > 0 {
customerInfo.WriteString("Companies: ")
for i, companyID := range customer.Companies {
comp, err := s.companyService.GetByID(ctx, companyID)
if err == nil {
if i > 0 {
customerInfo.WriteString(", ")
}
customerInfo.WriteString(comp.Name)
if comp.Industry != "" {
customerInfo.WriteString(fmt.Sprintf(" (Industry: %s)", comp.Industry))
}
if comp.Address != nil && comp.Address.Country != "" {
customerInfo.WriteString(fmt.Sprintf(" (Country: %s)", comp.Address.Country))
}
if comp.Tags != nil && len(comp.Tags.Keywords) > 0 {
customerInfo.WriteString(fmt.Sprintf(" (Company Keywords: %s)", strings.Join(comp.Tags.Keywords, ", ")))
}
}
}
customerInfo.WriteString("\n")
}
systemPrompt := `You are a keyword extraction assistant for a tender management system.
Analyze the customer information and generate 5-10 relevant keywords that would help match them with appropriate tenders.
Keywords should be:
- Relevant to tender/procurement categories
- Based on customer type, role, and company information
- Professional and specific
- Suitable for matching with tender descriptions
Return ONLY a comma-separated list of keywords, no explanations or additional text.`
userMessage := fmt.Sprintf("Generate keywords for this customer:\n\n%s", customerInfo.String())
s.logger.Info("Generating keywords using GLM AI", map[string]interface{}{
"customer_id": customer.ID,
})
response, err := s.glmSDK.QuickChatWithSystem(ctx, systemPrompt, userMessage)
if err != nil {
s.logger.Error("Failed to generate keywords using GLM AI", map[string]interface{}{
"error": err.Error(),
"customer_id": customer.ID,
})
return []string{}, fmt.Errorf("failed to generate keywords: %w", err)
}
// Parse comma-separated keywords
keywordsStr := strings.TrimSpace(response)
if keywordsStr == "" {
return []string{}, nil
}
// Split by comma and clean up
keywordList := strings.Split(keywordsStr, ",")
keywords := make([]string, 0, len(keywordList))
for _, kw := range keywordList {
kw = strings.TrimSpace(kw)
if kw != "" {
keywords = append(keywords, kw)
}
}
s.logger.Info("Keywords generated successfully", map[string]interface{}{
"customer_id": customer.ID,
"keywords": keywords,
"count": len(keywords),
})
return keywords, nil
_ = ctx
_ = customer
return []string{}, nil
}
func (s *customerService) processKeywordsAsync(customerID string) {
@@ -1456,34 +1365,26 @@ func (s *customerService) processKeywordsAsync(customerID string) {
return
}
if s.glmSDK == nil {
s.logger.Info("GLM SDK not available, keeping manual keywords", map[string]interface{}{
keywords, err := s.generateKeywords(ctx, cust)
if err != nil {
s.logger.Warn("Failed to process keywords after manual update", map[string]interface{}{
"error": err.Error(),
"customer_id": customerID,
})
cust.KeywordsStatus = KeywordsStatusFailed
} else if len(keywords) > 0 {
cust.Keywords = keywords
cust.KeywordsStatus = KeywordsStatusReady
} else {
s.logger.Debug("No auto-generated keywords from backend, keeping existing keywords", map[string]interface{}{
"customer_id": customerID,
})
cust.KeywordsStatus = KeywordsStatusReady
} else {
keywords, err := s.generateKeywords(ctx, cust)
if err != nil {
s.logger.Warn("Failed to process keywords after manual update", map[string]interface{}{
"error": err.Error(),
"customer_id": customerID,
})
cust.KeywordsStatus = KeywordsStatusFailed
} else if len(keywords) > 0 {
cust.Keywords = keywords
cust.KeywordsStatus = KeywordsStatusReady
} else {
s.logger.Warn("GLM returned empty keywords, keeping manual keywords", map[string]interface{}{
"customer_id": customerID,
})
cust.KeywordsStatus = KeywordsStatusReady
}
}
err = s.repository.Update(ctx, cust)
if err != nil {
if updateErr := s.repository.Update(ctx, cust); updateErr != nil {
s.logger.Error("Failed to update customer keywords status", map[string]interface{}{
"error": err.Error(),
"error": updateErr.Error(),
"customer_id": customerID,
})
} else {
internal/tender/entity.go
+53 -50
View File
@@ -32,56 +32,59 @@ type ScrapedDocument struct {
// Tender represents a tender/contract notice entity
type Tender struct {
mongo.Model `bson:",inline"`
ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"`
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"`
IssueTime int64 `bson:"issue_time" json:"issue_time"`
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"`
PublicationDate int64 `bson:"publication_date" json:"publication_date"`
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"`
ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"`
GazetteID string `bson:"gazette_id" json:"gazette_id"`
Title string `bson:"title" json:"title"`
Description string `bson:"description" json:"description"`
ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"`
ProcedureCode string `bson:"procedure_code" json:"procedure_code"`
MainClassification string `bson:"main_classification" json:"main_classification"`
AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"`
EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"`
Currency string `bson:"currency" json:"currency"`
Duration string `bson:"duration" json:"duration"`
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
CountryCode string `bson:"country_code" json:"country_code"`
RegionCode string `bson:"region_code" json:"region_code"`
CityName string `bson:"city_name" json:"city_name"`
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
Organizations []Organization `bson:"organizations" json:"organizations"`
SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"`
OfficialLanguages []string `bson:"official_languages" json:"official_languages"`
Status TenderStatus `bson:"status" json:"status"`
Source TenderSource `bson:"source" json:"source"`
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"`
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
mongo.Model `bson:",inline"`
ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"`
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
// RelatedNoticePublicationIDs lists every TED notice publication id that has contributed to this tender
// (same procedure / contract_folder_id). Ordered oldest-first; the latest is also reflected in NoticePublicationID.
RelatedNoticePublicationIDs []string `bson:"related_notice_publication_ids,omitempty" json:"related_notice_publication_ids,omitempty"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"`
IssueTime int64 `bson:"issue_time" json:"issue_time"`
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"`
PublicationDate int64 `bson:"publication_date" json:"publication_date"`
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"`
ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"`
GazetteID string `bson:"gazette_id" json:"gazette_id"`
Title string `bson:"title" json:"title"`
Description string `bson:"description" json:"description"`
ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"`
ProcedureCode string `bson:"procedure_code" json:"procedure_code"`
MainClassification string `bson:"main_classification" json:"main_classification"`
AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"`
EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"`
Currency string `bson:"currency" json:"currency"`
Duration string `bson:"duration" json:"duration"`
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
CountryCode string `bson:"country_code" json:"country_code"`
RegionCode string `bson:"region_code" json:"region_code"`
CityName string `bson:"city_name" json:"city_name"`
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
Organizations []Organization `bson:"organizations" json:"organizations"`
SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"`
OfficialLanguages []string `bson:"official_languages" json:"official_languages"`
Status TenderStatus `bson:"status" json:"status"`
Source TenderSource `bson:"source" json:"source"`
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"`
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
// AIOverallSummary optional denormalized overall summary for search/list (e.g. synced from AI pipeline storage).
AIOverallSummary string `bson:"ai_overall_summary,omitempty" json:"ai_overall_summary,omitempty"`
TenderID string `bson:"tender_id" json:"tender_id"` // PBL Project Number (SCDYYNNN format)
internal/tender/repository.go
+39 -2
View File
@@ -20,6 +20,7 @@ type TenderRepository interface {
GetByID(ctx context.Context, id string) (*Tender, error)
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error)
GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error)
GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error)
FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error)
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
GetTenderCountByType(ctx context.Context) (map[string]int64, error)
@@ -52,6 +53,7 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
*orm.CreateUniqueIndex("tender_id_idx", bson.D{{Key: "tender_id", Value: 1}}),
*orm.NewIndex("project_name_idx", bson.D{{Key: "project_name", Value: 1}}),
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
*orm.NewIndex("contract_folder_id_idx", bson.D{{Key: "contract_folder_id", Value: 1}}),
*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
*orm.NewIndex("source_idx", bson.D{{Key: "source", Value: 1}}),
*orm.NewIndex("country_code_idx", bson.D{{Key: "country_code", Value: 1}}),
@@ -158,9 +160,16 @@ func (r *tenderRepository) GetByContractNoticeID(ctx context.Context, contractNo
return &result.Items[0], nil
}
// GetByNoticePublicationID retrieves a tender by notice publication ID
// GetByNoticePublicationID retrieves a tender by notice publication ID.
// Matches both the canonical NoticePublicationID and any id retained in RelatedNoticePublicationIDs
// so historical TED detail links keep resolving after a tender update from a follow-up notice.
func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error) {
filter := bson.M{"notice_publication_id": noticePublicationID}
filter := bson.M{
"$or": []bson.M{
{"notice_publication_id": noticePublicationID},
{"related_notice_publication_ids": noticePublicationID},
},
}
pagination := orm.Pagination{Limit: 1}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
@@ -179,6 +188,34 @@ func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticeP
return &result.Items[0], nil
}
// GetByContractFolderID retrieves a tender by its procedure-level identifier (ContractFolderID).
// TED groups multiple notice publications (e.g. competition notice + result notice) under the same
// ContractFolderID, so this lookup is the authoritative way to find the existing tender for a procedure
// when a follow-up notice arrives. Returns the most recently updated match if there are several.
func (r *tenderRepository) GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error) {
filter := bson.M{"contract_folder_id": contractFolderID}
pagination := orm.Pagination{
Limit: 1,
SortField: "updated_at",
SortOrder: -1,
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to get tender by contract folder ID", map[string]interface{}{
"contract_folder_id": contractFolderID,
"error": err.Error(),
})
return nil, err
}
if len(result.Items) == 0 {
return nil, orm.ErrDocumentNotFound
}
return &result.Items[0], nil
}
// FindTendersWithContentXML returns tenders that still have TED XML (for backfill after notices were deleted).
func (r *tenderRepository) FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) {
filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}