mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Worker Initialization to Integrate GLM SDK

Browse Source 
- Updated the worker initialization process to include the new GLM SDK, enhancing the worker's capabilities for translation tasks.
- Modified the InitWorker function and NewNoticeWorker constructor to accept the GLM service, ensuring a cohesive integration.
- Implemented the GLM service initialization and logging for successful setup, improving maintainability and usability.
- Updated the NoticeWorker to utilize the GLM SDK for translating notice titles and descriptions, enhancing functionality and user experience.
This commit is contained in:
n.nakhostin
2025-11-04 16:51:45 +03:30
parent a5d3a94c97
commit fafccd0d74
10 changed files with 379 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ted/mapper.go
-9
View File
@@ -79,9 +79,6 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so
},
}
// Generate the tender ID after creating the tender
t.TenderID = GenerateTenderID(t)
return t
}
@@ -228,9 +225,6 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
t.OfficialLanguages = languages
}
// Generate unique tender ID
t.TenderID = GenerateTenderID(t)
return t
}
@@ -308,9 +302,6 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
}
}
// Generate unique tender ID
t.TenderID = GenerateTenderID(t)
return t
}
ted/scraper.go
+92 -12
View File
@@ -345,7 +345,6 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
// // Create new tender
s.logger.Info("Creating new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"tender_id": t.TenderID,
})
t.CreatedAt = time.Now().Unix()
@@ -355,14 +354,12 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
if err != nil {
s.logger.Error("Failed to create new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"tender_id": t.TenderID,
"error": err.Error(),
})
return fmt.Errorf("failed to create tender: %w", err)
}
s.logger.Info("Successfully created new tender", map[string]interface{}{
"tender_id": t.TenderID,
"contract_notice_id": t.ContractNoticeID,
})
// }
@@ -372,7 +369,6 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
"notice_id": id,
"notice_type": noticeType,
"file_name": fileName,
"tender_id": t.TenderID,
"tender_url": t.TenderURL,
})
@@ -380,34 +376,118 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
return nil
}
func (s *TEDScraper) Run(ctx context.Context) error {
now := time.Now().Local()
s.logger.Info("Running periodic scraping", map[string]interface{}{
"time": now.Format(time.RFC3339),
func (s *TEDScraper) Run(ctx context.Context, from, to *time.Time) error {
// If no date range is provided, use current date
if from == nil && to == nil {
now := time.Now().Local()
s.logger.Info("Running periodic scraping for current date", map[string]interface{}{
"date": now.Format("02/01/2006"),
})
return s.processSingleDate(ctx, now)
}
// Validate date range
if from == nil || to == nil {
return fmt.Errorf("both from and to dates must be provided for date range scraping")
}
if from.After(*to) {
return fmt.Errorf("from date cannot be after to date")
}
s.logger.Info("Running date range scraping", map[string]interface{}{
"from_date": from.Format("02/01/2006"),
"to_date": to.Format("02/01/2006"),
})
// Process each date in the range
current := *from
totalDays := 0
successfulDays := 0
var errors []string
for !current.After(*to) {
totalDays++
s.logger.Info("Processing date", map[string]interface{}{
"date": current.Format("02/01/2006"),
"progress": fmt.Sprintf("%d/%d", totalDays, int(to.Sub(*from).Hours()/24)+1),
})
err := s.processSingleDate(ctx, current)
if err != nil {
errorMsg := fmt.Sprintf("failed to process date %s: %v", current.Format("02/01/2006"), err)
errors = append(errors, errorMsg)
s.logger.Error("Failed to process date", map[string]interface{}{
"date": current.Format("02/01/2006"),
"error": err.Error(),
})
} else {
successfulDays++
}
// Check for context cancellation
select {
case <-ctx.Done():
s.logger.Warn("Date range scraping cancelled", map[string]interface{}{
"processed_days": totalDays,
"successful_days": successfulDays,
"cancelled_at": current.Format("02/01/2006"),
})
return ctx.Err()
default:
}
// Move to next day
current = current.AddDate(0, 0, 1)
}
s.logger.Info("Date range scraping completed", map[string]interface{}{
"total_days": totalDays,
"successful_days": successfulDays,
"failed_days": len(errors),
"errors": errors,
})
// Return error if no days were processed successfully
if successfulDays == 0 && len(errors) > 0 {
return fmt.Errorf("failed to process any dates in range: %v", errors)
}
return nil
}
// processSingleDate processes data for a single date
func (s *TEDScraper) processSingleDate(ctx context.Context, date time.Time) error {
s.logger.Info("Processing single date", map[string]interface{}{
"date": date.Format("02/01/2006"),
})
// get ojs
ojs, err := GetOJS(s.config.BaseURL, now.Year(), now.Format("02/01/2006"))
ojs, err := GetOJS(s.config.BaseURL, date.Year(), date.Format("02/01/2006"))
if err != nil {
s.logger.Error("Failed to get OJS", map[string]interface{}{
"date": date.Format("02/01/2006"),
"error": err.Error(),
})
return nil
return fmt.Errorf("failed to get OJS for date %s: %w", date.Format("02/01/2006"), err)
}
// download daily package
result, err := s.DownloadDailyPackage(ctx, ojs)
if err != nil {
s.logger.Error("Failed to download daily package", map[string]interface{}{
"date": date.Format("02/01/2006"),
"ojs": ojs,
"error": err.Error(),
})
return err
return fmt.Errorf("failed to download daily package for date %s: %w", date.Format("02/01/2006"), err)
}
// process daily package
s.logger.Info("Processed daily package successfully", map[string]interface{}{
"result": result,
"time": now.Format(time.RFC3339),
"date": date.Format("02/01/2006"),
"ojs": ojs,
})
ted/service.go
-59
View File
@@ -5,7 +5,6 @@ import (
"strconv"
"strings"
"time"
"tm/internal/notice"
)
// parseDate parses various date formats used in TED XML
@@ -69,15 +68,6 @@ func ParseTimeToUnix(timeStr string) int64 {
return 0
}
// unixToDateString converts Unix to date string for TenderID generation
func UnixToDateString(unix int64) string {
if unix == 0 {
return "00000000"
}
return time.Unix(unix, 0).Format("20060102")
}
func CalculateSubmissionDeadline(publicationDate time.Time) time.Time {
deadline := publicationDate
workingHoursAdded := 0
@@ -108,20 +98,6 @@ func CalculateApplicationDeadline(tenderDeadline time.Time) time.Time {
return applicationDeadline
}
// padOrTruncate pads a string with zeros or truncates it to the specified length
func PadOrTruncate(str string, length int) string {
// Remove any non-alphanumeric characters and convert to uppercase
clean := strings.ToUpper(strings.ReplaceAll(str, "-", ""))
clean = strings.ReplaceAll(clean, " ", "")
if len(clean) >= length {
return clean[:length]
}
// Pad with zeros
return clean + strings.Repeat("0", length-len(clean))
}
// generateTenderURL generates the TED tender URL from NoticePublicationID
func GenerateTenderURL(noticePublicationID string) string {
if noticePublicationID == "" {
@@ -143,38 +119,3 @@ func GenerateTenderURL(noticePublicationID string) string {
return ""
}
// generateTenderID generates a unique tender ID using the format:
// {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
func GenerateTenderID(t *notice.Notice) string {
var parts []string
// Source (TED)
parts = append(parts, "1")
// TED code (Notice Publication ID, first 8 chars)
tedCode := "00000000"
if t.NoticePublicationID != "" {
tedCode = PadOrTruncate(t.NoticePublicationID, 8)
}
parts = append(parts, tedCode)
// Buyer code (first 8 chars of buyer organization ID, or 8 zeros if not available)
// buyerCode := "00000000"
// if t.BuyerOrganization != nil && t.BuyerOrganization.ID != "" {
// buyerCode = padOrTruncate(t.BuyerOrganization.ID, 8)
// }
// parts = append(parts, buyerCode)
// Date (YYYYMMDD format from issue date)
parts = append(parts, PadOrTruncate(UnixToDateString(t.IssueDate), 4))
// Location code (country code + region code, padded to 6 chars)
locationCode := "EU"
if t.CountryCode != "" {
locationCode = t.CountryCode
}
parts = append(parts, locationCode)
return strings.Join(parts, "")
}