From 4beb57f42ecb6db95450fef7509d980c1707901b Mon Sep 17 00:00:00 2001 From: Mazyar Date: Mon, 1 Jun 2026 11:14:18 +0330 Subject: [PATCH] Refactor document scraper service to support multiple country codes for pending tenders retrieval. --- internal/document_scraper/service.go | 21 +++++++++++++++------ internal/tender/repository.go | 14 +++++++++----- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/internal/document_scraper/service.go b/internal/document_scraper/service.go index 4f13216..3a819d6 100644 --- a/internal/document_scraper/service.go +++ b/internal/document_scraper/service.go @@ -9,7 +9,16 @@ import ( "tm/pkg/response" ) -const documentScraperCountryCode = "SWE" +var documentScraperCountryCodes = []string{"SWE", "FIN", "NOR"} + +func isDocumentScraperCountry(countryCode string) bool { + for _, code := range documentScraperCountryCodes { + if countryCode == code { + return true + } + } + return false +} // Service defines business logic for the document scraper API type Service interface { @@ -33,7 +42,7 @@ func NewService(tenderRepo tender.TenderRepository, logger logger.Logger) Servic } // ListPendingTenders retrieves tenders that have not yet been scraped for documents. -// Only returns tenders from Sweden with active deadlines (deadline not yet reached). +// Only returns tenders from Sweden, Finland, and Norway with active deadlines (deadline not yet reached). func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) { s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{ "limit": limit, @@ -42,7 +51,7 @@ func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*D now := time.Now().Unix() - tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, documentScraperCountryCode, now, limit, offset) + tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, documentScraperCountryCodes, now, limit, offset) if err != nil { s.logger.Error("Failed to list pending tenders", map[string]interface{}{ "error": err.Error(), @@ -77,7 +86,7 @@ func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*D } // GetTenderByNoticeID retrieves a specific tender by its notice publication ID. -// Only returns tenders from Sweden with active deadlines (deadline not yet reached). +// Only returns tenders from Sweden, Finland, and Norway with active deadlines (deadline not yet reached). func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) { s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{ "notice_id": noticeID, @@ -97,8 +106,8 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do } now := time.Now().Unix() - if t.CountryCode != documentScraperCountryCode || t.TenderDeadline <= now { - s.logger.Info("Tender filtered out: not Sweden or deadline passed", map[string]interface{}{ + if !isDocumentScraperCountry(t.CountryCode) || t.TenderDeadline <= now { + s.logger.Info("Tender filtered out: unsupported country or deadline passed", map[string]interface{}{ "notice_id": noticeID, "country_code": t.CountryCode, "tender_deadline": t.TenderDeadline, diff --git a/internal/tender/repository.go b/internal/tender/repository.go index 1bcad44..b8a12a3 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -32,7 +32,7 @@ type TenderRepository interface { Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error) GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error) GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) - GetPendingDocumentScrapeTenders(ctx context.Context, countryCode string, minDeadline int64, limit, skip int) ([]Tender, bool, error) + GetPendingDocumentScrapeTenders(ctx context.Context, countryCodes []string, minDeadline int64, limit, skip int) ([]Tender, bool, error) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) Update(ctx context.Context, tender *Tender) error @@ -579,10 +579,14 @@ func (r *tenderRepository) GetUnScrapedTenders(ctx context.Context, limit, skip return result.Items, result.TotalCount, nil } -// GetPendingDocumentScrapeTenders returns unscraped tenders for one country with an active deadline. -func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, countryCode string, minDeadline int64, limit, skip int) ([]Tender, bool, error) { +// GetPendingDocumentScrapeTenders returns unscraped tenders for the given countries with an active deadline. +func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, countryCodes []string, minDeadline int64, limit, skip int) ([]Tender, bool, error) { + if len(countryCodes) == 0 { + return nil, false, nil + } + filter := bson.M{ - "country_code": countryCode, + "country_code": bson.M{"$in": countryCodes}, "tender_deadline": bson.M{ "$gt": minDeadline, }, @@ -601,7 +605,7 @@ func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, result, err := r.ormRepo.FindAll(ctx, filter, pagination) if err != nil { r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{ - "country_code": countryCode, + "country_codes": countryCodes, "min_deadline": minDeadline, "limit": limit, "skip": skip,