mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removed old document scraper service

Browse Source
This commit is contained in:
m.nazemi
2026-05-03 13:46:38 +03:30
parent 0bd5e41b70
commit 446c11dfbf
11 changed files with 9 additions and 1371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/workers/notice.go
+1 -63
View File
@@ -11,7 +11,6 @@ import (
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/scraper"
"go.mongodb.org/mongo-driver/v2/bson"
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
@@ -24,11 +23,10 @@ type NoticeWorker struct {
NoticeRepo notice.Repository
TenderRepo tender.TenderRepository
GLM *glm.SDK
Scraper scraper.SDK
ProcessingLimit int
}
func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, scraper scraper.SDK, processingLimit int) *NoticeWorker {
func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, processingLimit int) *NoticeWorker {
if processingLimit < 0 {
processingLimit = 5 // Default limit
}
@@ -39,7 +37,6 @@ func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notif
NoticeRepo: noticeRepo,
TenderRepo: tenderRepo,
GLM: glmService,
Scraper: scraper,
ProcessingLimit: processingLimit,
}
}
@@ -204,65 +201,6 @@ func (w *NoticeWorker) Run() {
w.cleanupProcessedNotices()
}
// scrapeDocumentsForNotice calls the scraper SDK to scrape documents for a processed notice
func (w *NoticeWorker) scrapeDocumentsForNotice(notice *notice.Notice) {
// Skip if no NoticePublicationID
if notice.NoticePublicationID == "" {
w.Logger.Warn("Skipping document scraping - no NoticePublicationID", map[string]interface{}{
"notice_id": notice.ID.Hex(),
})
return
}
// Get the tender URL
noticeURL := notice.GetTenderURL()
if noticeURL == "" {
w.Logger.Warn("Skipping document scraping - no tender URL available", map[string]interface{}{
"notice_id": notice.ID.Hex(),
"notice_publication_id": notice.NoticePublicationID,
})
return
}
// Call scraper SDK
scrapeReq := &scraper.ScrapeRequest{
NoticeID: notice.NoticePublicationID, // Use NoticePublicationID as notice_id
NoticeURL: noticeURL,
}
w.Logger.Info("Starting document scraping for processed notice", map[string]interface{}{
"notice_id": notice.ID.Hex(),
"notice_publication_id": notice.NoticePublicationID,
"notice_url": noticeURL,
})
response, err := w.Scraper.ScrapeDocuments(context.Background(), scrapeReq)
if err != nil {
w.Logger.Error("Failed to scrape documents", map[string]interface{}{
"notice_id": notice.ID.Hex(),
"notice_publication_id": notice.NoticePublicationID,
"error": err.Error(),
})
return
}
if !response.Success {
w.Logger.Warn("Document scraping failed", map[string]interface{}{
"notice_id": notice.ID.Hex(),
"notice_publication_id": notice.NoticePublicationID,
"errors": response.Errors,
})
return
}
w.Logger.Info("Document scraping completed successfully", map[string]interface{}{
"notice_id": notice.ID.Hex(),
"notice_publication_id": notice.NoticePublicationID,
"uploaded_count": response.UploadedCount,
"bucket": response.Buckets,
})
}
func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
t, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), n.NoticePublicationID)
if err != nil {