mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removed old document scraper service

Browse Source
This commit is contained in:
m.nazemi
2026-05-03 13:46:38 +03:30
parent 0bd5e41b70
commit 446c11dfbf
11 changed files with 9 additions and 1371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/web/main.go
+2 -57
View File
@@ -104,9 +104,7 @@ import (
"tm/internal/feedback"
"tm/internal/inquiry"
"tm/internal/kanban"
"tm/internal/notice"
"tm/internal/notification"
"tm/internal/scraper"
"tm/internal/tender"
"tm/internal/tender_approval"
"tm/internal/user"
@@ -118,56 +116,6 @@ import (
"tm/cmd/web/router"
)
func buildTEDConfig(c *bootstrap.TEDConfig) *scraper.TEDConfig {
if c == nil {
return &scraper.TEDConfig{
BaseURL: "https://ted.europa.eu",
Timeout: 30 * time.Second,
MaxRetries: 3,
RetryDelay: 5 * time.Second,
UserAgent: "TM-TED-Scraper/1.0",
MaxConcurrency: 5,
DownloadDir: "./downloads",
CleanupAfter: 24 * time.Hour,
ScrapingInterval: "* * 10 * * *",
AlertMail: "alerts@opplens.com",
}
}
baseURL := c.BaseURL
if baseURL == "" {
baseURL = "https://ted.europa.eu"
}
userAgent := c.UserAgent
if userAgent == "" {
userAgent = "TM-TED-Scraper/1.0"
}
downloadDir := c.DownloadDir
if downloadDir == "" {
downloadDir = "./downloads"
}
scrapingInterval := c.ScrapingInterval
if scrapingInterval == "" {
scrapingInterval = "* * 10 * * *"
}
alertMail := c.AlertMail
if alertMail == "" {
alertMail = "alerts@opplens.com"
}
return &scraper.TEDConfig{
BaseURL: baseURL,
Timeout: c.Timeout,
MaxRetries: c.MaxRetries,
RetryDelay: c.RetryDelay,
UserAgent: userAgent,
MaxConcurrency: c.MaxConcurrency,
DownloadDir: downloadDir,
CleanupAfter: c.CleanupAfter,
ScrapingInterval: scrapingInterval,
MaxNoticesPerDay: c.MaxNoticesPerDay,
AlertMail: alertMail,
}
}
func main() {
conf := bootstrap.InitConfig()
@@ -229,7 +177,7 @@ func main() {
inquiryRepository := inquiry.NewRepository(mongoManager, logger)
contactRepository := contact.NewContactRepository(mongoManager, logger)
cmsRepository := cms.NewRepository(mongoManager, logger)
noticeRepository := notice.NewRepository(mongoManager, logger)
// noticeRepository := notice.NewRepository(mongoManager, logger)
boardRepository := kanban.NewBoardRepository(mongoManager, logger)
columnRepository := kanban.NewColumnRepository(mongoManager, logger)
cardRepository := kanban.NewCardRepository(mongoManager, logger)
@@ -256,8 +204,6 @@ func main() {
notificationService := notification.NewService(notificationSDK, notificationRepository, userService, customerService, logger)
contactService := contact.NewService(contactRepository, logger, notificationSDK)
cmsService := cms.NewService(cmsRepository, logger)
tedConfig := buildTEDConfig(&conf.TED)
scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig)
kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger)
documentScraperService := document_scraper.NewService(tenderRepository, logger)
logger.Info("Services initialized successfully", map[string]interface{}{
@@ -277,7 +223,6 @@ func main() {
notificationHandler := notification.NewHandler(notificationService)
contactHandler := contact.NewHandler(contactService, hcaptchaVerifier)
cmsHandler := cms.NewHandler(cmsService)
scraperHandler := scraper.NewHandler(scraperService)
kanbanHandler := kanban.NewHandler(kanbanService)
fileStoreHandler := filestore.NewHandler(fileStoreService, logger)
documentScraperHandler := document_scraper.NewHandler(documentScraperService, logger)
@@ -289,7 +234,7 @@ func main() {
e := bootstrap.InitHTTPServer(conf, logger)
// Register routes
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler, kanbanHandler, fileStoreHandler, xssPolicy)
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, xssPolicy)
router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, fileStoreHandler, xssPolicy)
router.RegisterDocumentScraperRoutes(e, documentScraperHandler, conf.DocumentScraper.APIKey)
cmd/web/router/routes.go
+1 -10
View File
@@ -12,7 +12,6 @@ import (
"tm/internal/inquiry"
"tm/internal/kanban"
"tm/internal/notification"
"tm/internal/scraper"
"tm/internal/tender"
"tm/internal/tender_approval"
"tm/internal/user"
@@ -22,7 +21,7 @@ import (
"github.com/labstack/echo/v4"
)
func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, xssPolicy *security.XSSPolicy) {
func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, xssPolicy *security.XSSPolicy) {
adminV1 := e.Group("/admin/v1")
// Add CSP middleware to admin routes (stricter policy)
@@ -180,14 +179,6 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
cmsGP.DELETE("/:id", cmsHandler.Delete)
}
// Scraper Routes
scraperGP := adminV1.Group("/scraper")
{
scraperGP.Use(userHandler.AuthMiddleware(), userHandler.AdminMiddleware())
scraperGP.POST("/download", scraperHandler.ScrapeDocuments)
scraperGP.POST("/ted/one-time", scraperHandler.TriggerTEDOneTimeScraping)
}
// Kanban Routes
kanbanGP := adminV1.Group("/kanban")
{
cmd/worker/bootstrap/bootstrap.go
+3 -43
View File
@@ -15,7 +15,6 @@ import (
"tm/pkg/ollama"
"tm/pkg/queue"
"tm/pkg/schedule"
"tm/pkg/scraper"
)
// Init Application Configuration
@@ -78,7 +77,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
}
// Init Worker
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, scraperService scraper.SDK, minioService *minio.Service) {
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, minioService *minio.Service) {
// Debug: Log worker config
appLogger.Info("Worker configuration", map[string]interface{}{
"worker_interval": config.Worker.Interval,
@@ -93,7 +92,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
appLogger.Info("Starting notice worker", map[string]interface{}{
"processing_limit": config.Worker.NoticeProcessingLimit,
})
noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, scraperService, config.Worker.NoticeProcessingLimit)
noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit)
noticeWorker.Run()
// Initialize Queue-based scraping if enabled
@@ -124,24 +123,6 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
}
}
// Start Queue Consumer Worker (24/7 continuous processing)
if config.Queue.Enabled && (config.Queue.Mode == "consumer" || config.Queue.Mode == "both") {
appLogger.Info("Starting queue consumer worker", map[string]interface{}{
"concurrency": config.Queue.Concurrency,
})
consumerWorker := workers.NewDocumentScraperConsumerWorker(
mongoManager,
appLogger,
tenderRepo,
queueService,
scraperService,
config.Queue.Concurrency,
)
go consumerWorker.Run()
} else if !config.Queue.Enabled {
appLogger.Info("Queue consumer worker disabled, using scheduled scraping instead", map[string]interface{}{})
}
// Start Queue Producer Worker (enqueues unscraped tenders periodically)
if config.Queue.Enabled && (config.Queue.Mode == "producer" || config.Queue.Mode == "both") {
appLogger.Info("Scheduling queue producer worker", map[string]interface{}{
@@ -165,19 +146,6 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
})
}
// Fallback: Keep the old scheduled scraper for backward compatibility
if !config.Queue.Enabled {
appLogger.Info("Scheduling fallback document scraper worker", map[string]interface{}{})
schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
Name: "Document Scraper Worker Job (Fallback)",
Func: func() {
worker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService)
worker.Run()
},
Expr: "0 10 * * * *", // Run at 10 AM every day
})
}
// Initialize Document Summarization Worker (only if MinIO is available)
if minioService != nil {
appLogger.Info("Starting document summarization worker", map[string]interface{}{})
@@ -209,7 +177,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
Name: "Notice Worker Job",
Func: func() {
worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, scraperService, config.Worker.NoticeProcessingLimit)
worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit)
worker.Run()
},
Expr: workerInterval,
@@ -367,11 +335,3 @@ func InitMinIOService(conf config.MinIOConfig, log logger.Logger) *minio.Service
return service
}
func InitScraperService(config Config, logger logger.Logger) scraper.SDK {
return scraper.NewClient(
config.Scraper.BaseURL,
config.Scraper.Timeout,
logger,
)
}
cmd/worker/main.go
+2 -2
View File
@@ -46,7 +46,7 @@ func main() {
}
// Initialize scraper service
scraperService := bootstrap.InitScraperService(*config, appLogger)
// scraperService := bootstrap.InitScraperService(*config, appLogger)
// Initialize MinIO service
minioService := bootstrap.InitMinIOService(config.MinIO, appLogger)
@@ -55,7 +55,7 @@ func main() {
}
// Initialize Worker
bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, glmService, scraperService, minioService)
bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, glmService, minioService)
// Set up signal handling for graceful shutdown
signalChan := make(chan os.Signal, 1)
cmd/worker/workers/notice.go
+1 -63
View File
@@ -11,7 +11,6 @@ import (
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/scraper"
"go.mongodb.org/mongo-driver/v2/bson"
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
@@ -24,11 +23,10 @@ type NoticeWorker struct {
NoticeRepo notice.Repository
TenderRepo tender.TenderRepository
GLM *glm.SDK
Scraper scraper.SDK
ProcessingLimit int
}
func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, scraper scraper.SDK, processingLimit int) *NoticeWorker {
func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, processingLimit int) *NoticeWorker {
if processingLimit < 0 {
processingLimit = 5 // Default limit
}
@@ -39,7 +37,6 @@ func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notif
NoticeRepo: noticeRepo,
TenderRepo: tenderRepo,
GLM: glmService,
Scraper: scraper,
ProcessingLimit: processingLimit,
}
}
@@ -204,65 +201,6 @@ func (w *NoticeWorker) Run() {
w.cleanupProcessedNotices()
}
// scrapeDocumentsForNotice calls the scraper SDK to scrape documents for a processed notice
func (w *NoticeWorker) scrapeDocumentsForNotice(notice *notice.Notice) {
// Skip if no NoticePublicationID
if notice.NoticePublicationID == "" {
w.Logger.Warn("Skipping document scraping - no NoticePublicationID", map[string]interface{}{
"notice_id": notice.ID.Hex(),
})
return
}
// Get the tender URL
noticeURL := notice.GetTenderURL()
if noticeURL == "" {
w.Logger.Warn("Skipping document scraping - no tender URL available", map[string]interface{}{
"notice_id": notice.ID.Hex(),
"notice_publication_id": notice.NoticePublicationID,
})
return
}
// Call scraper SDK
scrapeReq := &scraper.ScrapeRequest{
NoticeID: notice.NoticePublicationID, // Use NoticePublicationID as notice_id
NoticeURL: noticeURL,
}
w.Logger.Info("Starting document scraping for processed notice", map[string]interface{}{
"notice_id": notice.ID.Hex(),
"notice_publication_id": notice.NoticePublicationID,
"notice_url": noticeURL,
})
response, err := w.Scraper.ScrapeDocuments(context.Background(), scrapeReq)
if err != nil {
w.Logger.Error("Failed to scrape documents", map[string]interface{}{
"notice_id": notice.ID.Hex(),
"notice_publication_id": notice.NoticePublicationID,
"error": err.Error(),
})
return
}
if !response.Success {
w.Logger.Warn("Document scraping failed", map[string]interface{}{
"notice_id": notice.ID.Hex(),
"notice_publication_id": notice.NoticePublicationID,
"errors": response.Errors,
})
return
}
w.Logger.Info("Document scraping completed successfully", map[string]interface{}{
"notice_id": notice.ID.Hex(),
"notice_publication_id": notice.NoticePublicationID,
"uploaded_count": response.UploadedCount,
"bucket": response.Buckets,
})
}
func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
t, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), n.NoticePublicationID)
if err != nil {
cmd/worker/workers/scraper.go
-200
View File
@@ -1,200 +0,0 @@
package workers
import (
"context"
"time"
"tm/internal/tender"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/scraper"
)
// DocumentScraperWorker handles document scraping for tenders
type DocumentScraperWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
TenderRepo tender.TenderRepository
Scraper scraper.SDK
}
// NewDocumentScraperWorker creates a new document scraper worker
func NewDocumentScraperWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, scraper scraper.SDK) *DocumentScraperWorker {
logger.Info("Creating new document scraper worker", map[string]interface{}{})
return &DocumentScraperWorker{
Mongo: mongo,
Logger: logger,
TenderRepo: tenderRepo,
Scraper: scraper,
}
}
// Run starts the document scraping process
func (w *DocumentScraperWorker) Run() {
w.Logger.Info("Document scraper worker started", map[string]interface{}{})
limit := 10
skip := 0
for {
w.Logger.Info("Document scraper worker checking for tenders", map[string]interface{}{
"skip": skip,
"limit": limit,
})
tenders, totalCount, err := w.TenderRepo.GetUnScrapedTenders(context.Background(), limit, skip)
if err != nil {
w.Logger.Error("Failed to get un-scraped tenders", map[string]interface{}{
"error": err.Error(),
})
continue
}
w.Logger.Info("Found tenders for document scraping", map[string]interface{}{
"count": len(tenders),
"total_count": totalCount,
"skip": skip,
})
if len(tenders) == 0 {
w.Logger.Info("No more tenders to process for document scraping", map[string]interface{}{})
break
}
w.Logger.Info("Processing tenders for document scraping", map[string]interface{}{
"tender_count": len(tenders),
})
for _, t := range tenders {
w.Logger.Info("Processing tender for document scraping", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_publication_id": t.NoticePublicationID,
})
w.scrapeDocumentsForTender(&t)
}
skip += limit
}
w.Logger.Info("Document scraper worker completed", map[string]interface{}{})
}
// scrapeDocumentsForTender scrapes documents for a single tender
func (w *DocumentScraperWorker) scrapeDocumentsForTender(t *tender.Tender) {
w.Logger.Info("Starting document scraping for tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_publication_id": t.NoticePublicationID,
"document_uri": t.DocumentURI,
})
// Skip if no NoticePublicationID
if t.NoticePublicationID == "" {
w.Logger.Warn("Skipping document scraping - no NoticePublicationID", map[string]interface{}{
"tender_id": t.ID.Hex(),
})
return
}
// Use DocumentURI as NoticeURL for the scraper
noticeURL := t.DocumentURI
if noticeURL == "" {
w.Logger.Warn("Skipping document scraping - no DocumentURI available", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_publication_id": t.NoticePublicationID,
})
return
}
// Call scraper SDK
scrapeReq := &scraper.ScrapeRequest{
NoticeID: t.NoticePublicationID, // Use NoticePublicationID as notice_id
NoticeURL: noticeURL, // Use DocumentURI as notice_url
}
w.Logger.Info("Starting document scraping for tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_publication_id": t.NoticePublicationID,
"document_uri": noticeURL,
})
response, err := w.Scraper.ScrapeDocuments(context.Background(), scrapeReq)
if err != nil {
w.Logger.Error("Failed to scrape documents for tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_publication_id": t.NoticePublicationID,
"error": err.Error(),
})
// Mark as processed with error (don't retry indefinitely)
t.ProcessingMetadata.DocumentsScraped = false
t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix()
w.updateTenderScrapingStatus(t)
return
}
if !response.Success {
w.Logger.Warn("Document scraping failed for tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_publication_id": t.NoticePublicationID,
"errors": response.Errors,
})
// Mark as processed with failure
t.ProcessingMetadata.DocumentsScraped = false
t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix()
w.updateTenderScrapingStatus(t)
return
}
// Store scraped documents information
scrapedAt := time.Now().Unix()
t.ScrapedDocuments = make([]tender.ScrapedDocument, len(response.UploadedFiles))
for i, uploadedFile := range response.UploadedFiles {
t.ScrapedDocuments[i] = tender.ScrapedDocument{
Filename: uploadedFile.Filename,
ObjectName: uploadedFile.ObjectName,
BucketName: "opplens",
Size: uploadedFile.Size,
ScrapedAt: scrapedAt,
}
}
// Mark as successfully scraped
t.ProcessingMetadata.DocumentsScraped = true
t.ProcessingMetadata.DocumentsScrapedAt = scrapedAt
err = w.updateTenderScrapingStatus(t)
if err != nil {
w.Logger.Error("Failed to update tender scraping status", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_publication_id": t.NoticePublicationID,
"error": err.Error(),
})
return
}
w.Logger.Info("Document scraping completed successfully for tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_publication_id": t.NoticePublicationID,
"uploaded_count": response.UploadedCount,
"bucket_count": len(response.Buckets),
})
}
// updateTenderScrapingStatus updates the tender's document scraping metadata
func (w *DocumentScraperWorker) updateTenderScrapingStatus(t *tender.Tender) error {
err := w.TenderRepo.Update(context.Background(), t)
if err != nil {
w.Logger.Error("Failed to update tender scraping status", map[string]interface{}{
"tender_id": t.ID.Hex(),
"error": err.Error(),
})
return err
}
w.Logger.Debug("Updated tender scraping status", map[string]interface{}{
"tender_id": t.ID.Hex(),
"documents_scraped": t.ProcessingMetadata.DocumentsScraped,
"documents_scraped_at": t.ProcessingMetadata.DocumentsScrapedAt,
})
return nil
}
cmd/worker/workers/scraper_consumer.go
-422
View File
@@ -1,422 +0,0 @@
package workers
import (
"context"
"fmt"
"time"
tenderPkg "tm/internal/tender"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/queue"
"tm/pkg/scraper"
)
// DocumentScraperConsumerWorker processes scraping jobs from the queue continuously
type DocumentScraperConsumerWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
TenderRepo tenderPkg.TenderRepository
Queue queue.Queue
Scraper scraper.SDK
Concurrency int
StopChan chan struct{}
}
// NewDocumentScraperConsumerWorker creates a new consumer worker
func NewDocumentScraperConsumerWorker(
mongo *mongo.ConnectionManager,
logger logger.Logger,
tenderRepo tenderPkg.TenderRepository,
queue queue.Queue,
scraperService scraper.SDK,
concurrency int,
) *DocumentScraperConsumerWorker {
if concurrency <= 0 {
concurrency = 5 // Default concurrency
}
logger.Info("Creating new document scraper consumer worker", map[string]interface{}{
"concurrency": concurrency,
})
return &DocumentScraperConsumerWorker{
Mongo: mongo,
Logger: logger,
TenderRepo: tenderRepo,
Queue: queue,
Scraper: scraperService,
Concurrency: concurrency,
StopChan: make(chan struct{}),
}
}
// Run starts consuming jobs from the queue with specified concurrency
func (w *DocumentScraperConsumerWorker) Run() {
w.Logger.Info("Document scraper consumer worker started", map[string]interface{}{
"concurrency": w.Concurrency,
})
// Create worker pool channels
jobsChan := make(chan *queue.DocumentScraperJob, w.Concurrency)
resultsChan := make(chan *ProcessingResult, w.Concurrency)
// Start worker goroutines
for i := 0; i < w.Concurrency; i++ {
go w.processJobs(i, jobsChan, resultsChan)
}
// Start job fetcher
go w.fetchJobs(jobsChan)
// Start result handler
go w.handleResults(resultsChan)
// Wait for stop signal
<-w.StopChan
w.Logger.Info("Document scraper consumer worker stopping", map[string]interface{}{})
close(jobsChan)
close(resultsChan)
}
// Stop signals the worker to stop
func (w *DocumentScraperConsumerWorker) Stop() {
w.Logger.Info("Stop signal sent to consumer worker", map[string]interface{}{})
close(w.StopChan)
}
// fetchJobs continuously fetches jobs from the queue and sends them to workers
func (w *DocumentScraperConsumerWorker) fetchJobs(jobsChan chan<- *queue.DocumentScraperJob) {
defer func() {
w.Logger.Info("Job fetcher stopped", map[string]interface{}{})
}()
for {
select {
case <-w.StopChan:
return
default:
// Try to dequeue a job
ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
job, err := w.Queue.Dequeue(ctx)
cancel()
if err != nil {
w.Logger.Error("Failed to dequeue job", map[string]interface{}{
"error": err.Error(),
})
time.Sleep(5 * time.Second) // Back off on error
continue
}
if job == nil {
// Queue is empty, brief wait before retry
time.Sleep(2 * time.Second)
continue
}
// Send to processing channel
select {
case jobsChan <- job:
// Job sent successfully
case <-w.StopChan:
return
}
}
}
}
// processJobs processes jobs from the jobs channel
func (w *DocumentScraperConsumerWorker) processJobs(
workerID int,
jobsChan <-chan *queue.DocumentScraperJob,
resultsChan chan<- *ProcessingResult,
) {
w.Logger.Info("Worker started", map[string]interface{}{
"worker_id": workerID,
})
defer func() {
w.Logger.Info("Worker stopped", map[string]interface{}{
"worker_id": workerID,
})
}()
for {
select {
case job, ok := <-jobsChan:
if !ok {
return // Channel closed
}
if job == nil {
continue
}
// Process the job
result := w.processJob(job, workerID)
select {
case resultsChan <- result:
// Result sent successfully
case <-w.StopChan:
return
}
case <-w.StopChan:
return
}
}
}
// processJob executes the actual scraping task
func (w *DocumentScraperConsumerWorker) processJob(
job *queue.DocumentScraperJob,
workerID int,
) *ProcessingResult {
startTime := time.Now()
w.Logger.Info("Processing job", map[string]interface{}{
"worker_id": workerID,
"job_id": job.ID,
"notice_id": job.NoticeID,
"attempt": job.Attempts,
"max_retries": job.MaxRetries,
})
result := &ProcessingResult{
JobID: job.ID,
WorkerID: workerID,
StartedAt: startTime,
Job: job,
}
// Skip if missing required fields
if job.NoticeID == "" || job.NoticeURL == "" {
result.Success = false
result.Error = "missing required fields: notice_id or notice_url"
result.CompletedAt = time.Now()
return result
}
// Call scraper SDK with timeout
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
scrapeReq := &scraper.ScrapeRequest{
NoticeID: job.NoticeID,
NoticeURL: job.NoticeURL,
}
w.Logger.Info("Calling scraper service", map[string]interface{}{
"job_id": job.ID,
"notice_id": job.NoticeID,
"notice_url": job.NoticeURL,
"worker_id": workerID,
})
response, err := w.Scraper.ScrapeDocuments(ctx, scrapeReq)
if err != nil {
result.Success = false
result.Error = err.Error()
result.CompletedAt = time.Now()
result.ShouldRetry = job.Attempts < job.MaxRetries
w.Logger.Error("Scraping failed", map[string]interface{}{
"job_id": job.ID,
"worker_id": workerID,
"attempt": job.Attempts,
"error": err.Error(),
"should_retry": result.ShouldRetry,
})
return result
}
if !response.Success {
result.Success = false
result.Error = "scraper returned failure"
if len(response.Errors) > 0 {
result.Error += ": " + response.Errors[0].Error
}
result.CompletedAt = time.Now()
result.ShouldRetry = job.Attempts < job.MaxRetries
w.Logger.Warn("Scraping reported failure", map[string]interface{}{
"job_id": job.ID,
"worker_id": workerID,
"error_count": len(response.Errors),
"should_retry": result.ShouldRetry,
})
return result
}
// Update tender with scraped documents
scrapedAt := time.Now().Unix()
tender, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), job.NoticeID)
if err != nil {
result.Success = false
result.Error = "failed to fetch tender: " + err.Error()
result.CompletedAt = time.Now()
result.ShouldRetry = job.Attempts < job.MaxRetries
w.Logger.Error("Failed to fetch tender for update", map[string]interface{}{
"job_id": job.ID,
"notice_id": job.NoticeID,
"error": err.Error(),
"worker_id": workerID,
})
return result
}
if tender == nil {
result.Success = false
result.Error = "tender not found"
result.CompletedAt = time.Now()
w.Logger.Warn("Tender not found for scraping result", map[string]interface{}{
"job_id": job.ID,
"notice_id": job.NoticeID,
"worker_id": workerID,
})
return result
}
// Update tender with scraped documents
tender.ScrapedDocuments = make([]tenderPkg.ScrapedDocument, len(response.UploadedFiles))
for i, uploadedFile := range response.UploadedFiles {
tender.ScrapedDocuments[i] = tenderPkg.ScrapedDocument{
Filename: uploadedFile.Filename,
ObjectName: uploadedFile.ObjectName,
BucketName: "opplens", // Default bucket
Size: uploadedFile.Size,
ScrapedAt: scrapedAt,
}
}
// Update metadata
tender.ProcessingMetadata.DocumentsScraped = true
tender.ProcessingMetadata.DocumentsScrapedAt = scrapedAt
// Save to database
if err := w.TenderRepo.Update(context.Background(), tender); err != nil {
result.Success = false
result.Error = "failed to update tender: " + err.Error()
result.CompletedAt = time.Now()
result.ShouldRetry = job.Attempts < job.MaxRetries
w.Logger.Error("Failed to update tender with scraped documents", map[string]interface{}{
"job_id": job.ID,
"tender_id": tender.ID.Hex(),
"error": err.Error(),
"worker_id": workerID,
"should_retry": result.ShouldRetry,
})
return result
}
result.Success = true
result.UploadedCount = response.UploadedCount
result.CompletedAt = time.Now()
w.Logger.Info("Scraping job completed successfully", map[string]interface{}{
"job_id": job.ID,
"tender_id": tender.ID.Hex(),
"uploaded_count": response.UploadedCount,
"worker_id": workerID,
"duration_seconds": time.Since(startTime).Seconds(),
})
return result
}
// handleResults processes the results from worker jobs
func (w *DocumentScraperConsumerWorker) handleResults(resultsChan <-chan *ProcessingResult) {
defer func() {
w.Logger.Info("Result handler stopped", map[string]interface{}{})
}()
for {
select {
case result, ok := <-resultsChan:
if !ok {
return // Channel closed
}
if result == nil {
continue
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
if result.Success {
// Acknowledge successful job
if err := w.Queue.AckJob(ctx, result.JobID); err != nil {
w.Logger.Error("Failed to acknowledge job", map[string]interface{}{
"job_id": result.JobID,
"error": err.Error(),
})
}
} else {
// Handle failure - potentially retry
if result.ShouldRetry {
w.Logger.Info("Job will be retried", map[string]interface{}{
"job_id": result.JobID,
"attempt": result.Job.Attempts,
"error": result.Error,
"retry_in": "30 seconds",
})
// Re-enqueue the job for retry
job := result.Job
job.NextRetryTime = time.Now().Add(30 * time.Second).Unix()
job.Status = string(queue.StatusRetrying)
if err := w.Queue.Enqueue(ctx, job); err != nil {
w.Logger.Error("Failed to re-enqueue job for retry", map[string]interface{}{
"job_id": job.ID,
"error": err.Error(),
})
// Still nack it if re-enqueue fails
w.Queue.NackJob(ctx, result.JobID, fmt.Errorf(result.Error))
}
} else {
// Max retries exceeded, send to DLQ
w.Logger.Error("Job exceeded max retries, sending to DLQ", map[string]interface{}{
"job_id": result.JobID,
"max_retry": result.Job.MaxRetries,
"error": result.Error,
})
if err := w.Queue.NackJob(ctx, result.JobID, fmt.Errorf(result.Error)); err != nil {
w.Logger.Error("Failed to nack job", map[string]interface{}{
"job_id": result.JobID,
"error": err.Error(),
})
}
}
}
cancel()
case <-w.StopChan:
return
}
}
}
// ProcessingResult represents the result of processing a job
type ProcessingResult struct {
JobID string
WorkerID int
Success bool
Error string
ShouldRetry bool
UploadedCount int
StartedAt time.Time
CompletedAt time.Time
Job *queue.DocumentScraperJob
}
internal/scraper/form.go
-43
View File
@@ -1,43 +0,0 @@
package scraper
// ScrapeRequestForm represents the request form for scraping documents
type ScrapeRequestForm struct {
NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
}
// ScrapeResponse represents the response from scraping operation
type ScrapeResponse struct {
Success bool `json:"success"`
NoticeID string `json:"notice_id"`
UploadedCount int `json:"uploaded_count"`
UploadedFiles []UploadedFileInfo `json:"uploaded_files"`
Errors []FileError `json:"errors,omitempty"`
Buckets []BucketInfo `json:"buckets,omitempty"`
}
// UploadedFileInfo represents information about an uploaded file
type UploadedFileInfo struct {
Filename string `json:"filename"`
ObjectName string `json:"object_name"`
Size int64 `json:"size"`
}
// FileError represents an error for a specific file
type FileError struct {
Filename string `json:"filename"`
Error string `json:"error"`
}
// BucketInfo represents information about a bucket
type BucketInfo struct {
Name string `json:"name"`
NoticeID string `json:"notice_id"`
Count int `json:"count"`
}
// TEDOneTimeScrapeForm represents the request form for triggering TED one-time scraping
type TEDOneTimeScrapeForm struct {
FromDate string `json:"from_date" valid:"required" example:"01/01/2026"`
ToDate string `json:"to_date" valid:"required" example:"19/02/2026"`
}
internal/scraper/handler.go
-72
View File
@@ -1,72 +0,0 @@
package scraper
import (
"tm/pkg/response"
"github.com/labstack/echo/v4"
)
// Handler handles HTTP requests for scraper operations
type Handler struct {
service Service
}
// NewHandler creates a new scraper handler
func NewHandler(service Service) *Handler {
return &Handler{
service: service,
}
}
// ScrapeDocuments handles document scraping requests
// @Summary Scrape documents from external source
// @Description Scrapes documents from TendSign and uploads them to MinIO storage
// @Tags Admin-Scraper
// @Accept json
// @Produce json
// @Param request body ScrapeRequestForm true "Scraping request information"
// @Success 200 {object} response.Response{data=ScrapeResponse}
// @Failure 400 {object} response.Response
// @Failure 500 {object} response.Response
// @Router /admin/v1/scraper/download [post]
func (h *Handler) ScrapeDocuments(c echo.Context) error {
form, err := response.Parse[ScrapeRequestForm](c)
if err != nil {
return response.ValidationError(c, "Invalid request data", err.Error())
}
result, err := h.service.ScrapeDocuments(c.Request().Context(), form)
if err != nil {
return response.InternalServerError(c, "Failed to scrape documents")
}
return response.Success(c, result, "Documents scraped and uploaded successfully")
}
// TriggerTEDOneTimeScraping handles TED one-time scraping requests
// @Summary Trigger TED one-time scraping
// @Description Starts TED one-time scraping for the specified date range. The operation runs asynchronously; check logs for progress.
// @Tags Admin-Scraper
// @Accept json
// @Produce json
// @Param request body TEDOneTimeScrapeForm true "Date range for scraping (DD/MM/YYYY)"
// @Success 202 {object} response.APIResponse "TED scraping started"
// @Failure 400 {object} response.APIResponse "Invalid date format or range"
// @Failure 500 {object} response.APIResponse "TED scraping not configured"
// @Security BearerAuth
// @Router /admin/v1/scraper/ted/one-time [post]
func (h *Handler) TriggerTEDOneTimeScraping(c echo.Context) error {
form, err := response.Parse[TEDOneTimeScrapeForm](c)
if err != nil {
return response.ValidationError(c, "Invalid request data", err.Error())
}
err = h.service.TriggerTEDOneTimeScraping(c.Request().Context(), form)
if err != nil {
return response.BadRequest(c, err.Error(), "")
}
return response.Accepted(c, map[string]string{
"message": "TED one-time scraping started. Check logs for progress.",
}, "TED one-time scraping started")
}
internal/scraper/service.go
-213
View File
@@ -1,213 +0,0 @@
package scraper
import (
"context"
"fmt"
"time"
"tm/internal/notice"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/scraper"
"tm/ted"
)
// TEDConfig holds configuration for TED one-time scraping (used when creating service)
type TEDConfig struct {
BaseURL string
Timeout time.Duration
MaxRetries int
RetryDelay time.Duration
UserAgent string
MaxConcurrency int
DownloadDir string
CleanupAfter time.Duration
ScrapingInterval string
MaxNoticesPerDay int
AlertMail string
}
// Service defines the interface for scraper business operations
type Service interface {
// ScrapeDocuments scrapes documents and uploads them to MinIO
ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range (runs in background)
TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error
}
// scraperService implements Service interface
type scraperService struct {
sdk scraper.SDK
logger logger.Logger
mongoManager *mongo.ConnectionManager
noticeRepo notice.Repository
notificationSDK notification.SDK
tedConfig *TEDConfig
}
// NewService creates a new scraper service with stored credentials
func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
sdk := scraper.NewClient(baseURL, timeout, logger)
return &scraperService{
sdk: sdk,
logger: logger,
}
}
// NewServiceWithTED creates a new scraper service with both TendSign SDK and TED scraping support
func NewServiceWithTED(baseURL string, timeout time.Duration, logger logger.Logger, mongoManager *mongo.ConnectionManager, noticeRepo notice.Repository, notificationSDK notification.SDK, tedConfig *TEDConfig) Service {
sdk := scraper.NewClient(baseURL, timeout, logger)
return &scraperService{
sdk: sdk,
logger: logger,
mongoManager: mongoManager,
noticeRepo: noticeRepo,
notificationSDK: notificationSDK,
tedConfig: tedConfig,
}
}
// ScrapeDocuments scrapes documents and uploads them to MinIO
func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
s.logger.Info("Starting document scraping", map[string]interface{}{
"notice_id": form.NoticeID,
})
// Convert form to SDK request (only dynamic fields)
req := &scraper.ScrapeRequest{
NoticeID: form.NoticeID,
NoticeURL: form.NoticeURL,
}
// Call SDK (credentials are already configured)
response, err := s.sdk.ScrapeDocuments(ctx, req)
if err != nil {
s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{
"error": err.Error(),
"notice_id": form.NoticeID,
})
return nil, err
}
// Convert SDK response to service response
serviceResp := &ScrapeResponse{
Success: response.Success,
NoticeID: response.NoticeID,
UploadedCount: response.UploadedCount,
UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)),
Errors: make([]FileError, len(response.Errors)),
Buckets: make([]BucketInfo, len(response.Buckets)),
}
// Convert uploaded files
for i, file := range response.UploadedFiles {
serviceResp.UploadedFiles[i] = UploadedFileInfo{
Filename: file.Filename,
ObjectName: file.ObjectName,
Size: file.Size,
}
}
// Convert errors
for i, err := range response.Errors {
serviceResp.Errors[i] = FileError{
Filename: err.Filename,
Error: err.Error,
}
}
// Convert buckets
for i, bucket := range response.Buckets {
serviceResp.Buckets[i] = BucketInfo{
Name: bucket.Name,
NoticeID: bucket.NoticeID,
Count: bucket.Count,
}
}
s.logger.Info("Document scraping completed", map[string]interface{}{
"notice_id": serviceResp.NoticeID,
"uploaded_count": serviceResp.UploadedCount,
"success": serviceResp.Success,
})
return serviceResp, nil
}
// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range.
// The scraping runs in a background goroutine; the handler returns 202 Accepted immediately.
func (s *scraperService) TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error {
if s.tedConfig == nil || s.mongoManager == nil || s.noticeRepo == nil {
return fmt.Errorf("TED scraping is not configured")
}
from, err := time.Parse("02/01/2006", form.FromDate)
if err != nil {
return fmt.Errorf("invalid from_date format: expected DD/MM/YYYY (e.g. 01/01/2026)")
}
to, err := time.Parse("02/01/2006", form.ToDate)
if err != nil {
return fmt.Errorf("invalid to_date format: expected DD/MM/YYYY (e.g. 19/02/2026)")
}
if from.After(to) {
return fmt.Errorf("from_date cannot be after to_date")
}
tedConfig := s.tedConfig
mongoManager := s.mongoManager
noticeRepo := s.noticeRepo
notificationSDK := s.notificationSDK
log := s.logger
go func() {
tedScraperConfig := &ted.Config{
BaseURL: tedConfig.BaseURL,
UserAgent: tedConfig.UserAgent,
DownloadDir: tedConfig.DownloadDir,
MaxRetries: tedConfig.MaxRetries,
MaxConcurrency: tedConfig.MaxConcurrency,
Timeout: tedConfig.Timeout,
RetryDelay: tedConfig.RetryDelay,
CleanupAfter: tedConfig.CleanupAfter,
ScrapingInterval: tedConfig.ScrapingInterval,
AlertMail: tedConfig.AlertMail,
MaxNoticesPerDay: tedConfig.MaxNoticesPerDay,
}
tedScraper := ted.NewTEDScraper(
tedScraperConfig,
log,
mongoManager,
noticeRepo,
notificationSDK,
)
runCtx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
defer cancel()
log.Info("Starting TED one-time scraping (triggered via API)", map[string]interface{}{
"from_date": from.Format("02/01/2006"),
"to_date": to.Format("02/01/2006"),
})
err := tedScraper.Run(runCtx, &from, &to)
if err != nil {
log.Error("TED one-time scraping failed", map[string]interface{}{
"error": err.Error(),
"from_date": from.Format("02/01/2006"),
"to_date": to.Format("02/01/2006"),
})
return
}
log.Info("TED one-time scraping completed successfully", map[string]interface{}{
"from_date": from.Format("02/01/2006"),
"to_date": to.Format("02/01/2006"),
})
}()
return nil
}
pkg/scraper/sdk.go
-246
View File
@@ -1,246 +0,0 @@
package scraper
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"tm/pkg/logger"
)
// SDK represents the scraper SDK interface
type SDK interface {
// ScrapeDocuments scrapes documents for a given notice
ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error)
// ListBuckets lists all buckets with notice IDs
ListBuckets(ctx context.Context) (*ListBucketsResponse, error)
// HealthCheck checks if the scraper service is healthy
HealthCheck(ctx context.Context) error
}
// ScrapeRequest represents a request to scrape documents (only dynamic fields)
type ScrapeRequest struct {
NoticeID string `json:"notice_id"`
NoticeURL string `json:"notice_url"`
}
// ScrapeResponse represents the response from scraping operation
type ScrapeResponse struct {
Success bool `json:"success"`
NoticeID string `json:"notice_id"`
UploadedCount int `json:"uploaded_count"`
UploadedFiles []UploadedFile `json:"uploaded_files"`
Errors []FileError `json:"errors,omitempty"`
Buckets []BucketInfo `json:"buckets,omitempty"`
}
// UploadedFile represents information about an uploaded file
type UploadedFile struct {
Filename string `json:"filename"`
ObjectName string `json:"object_name"`
Size int64 `json:"size"`
}
// FileError represents an error for a specific file
type FileError struct {
Filename string `json:"filename"`
Error string `json:"error"`
}
// BucketInfo represents information about a bucket
type BucketInfo struct {
Name string `json:"name"`
NoticeID string `json:"notice_id"`
Count int `json:"count"`
}
// ListBucketsResponse represents the response from listing buckets
type ListBucketsResponse struct {
Buckets []BucketInfo `json:"buckets"`
Total int `json:"total"`
}
// Client implements the SDK interface
type Client struct {
httpClient *http.Client
baseURL string
logger logger.Logger
}
// NewClient creates a new scraper SDK client with stored credentials
func NewClient(baseURL string, timeout time.Duration, logger logger.Logger) SDK {
return &Client{
httpClient: &http.Client{
Timeout: timeout,
},
baseURL: baseURL,
logger: logger,
}
}
// ScrapeDocuments scrapes documents and uploads them to MinIO
func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) {
c.logger.Info("Starting document scraping via SDK", map[string]interface{}{
"notice_id": req.NoticeID,
"base_url": c.baseURL,
})
// Prepare request body with stored credentials + dynamic fields
requestBody := map[string]interface{}{
"notice_id": req.NoticeID,
"notice_url": req.NoticeURL,
}
// Use unified handler API
var response ScrapeResponse
err := c.makeRequest(ctx, "POST", "/download", requestBody, &response)
if err != nil {
return nil, err
}
// Get buckets list after successful scraping
bucketsResp, err := c.ListBuckets(ctx)
if err != nil {
c.logger.Warn("Failed to list buckets after scraping", map[string]interface{}{
"error": err.Error(),
})
} else {
response.Buckets = bucketsResp.Buckets
}
c.logger.Info("Document scraping completed via SDK", map[string]interface{}{
"notice_id": response.NoticeID,
"uploaded_count": response.UploadedCount,
"success": response.Success,
})
return &response, nil
}
// ListBuckets lists all buckets with notice IDs
func (c *Client) ListBuckets(ctx context.Context) (*ListBucketsResponse, error) {
var response ListBucketsResponse
err := c.makeRequest(ctx, "GET", "/buckets", nil, &response)
return &response, err
}
// HealthCheck checks if the scraper service is healthy
func (c *Client) HealthCheck(ctx context.Context) error {
var response map[string]interface{}
err := c.makeRequest(ctx, "GET", "/health", nil, &response)
if err != nil {
return err
}
// Health check doesn't need to return data, just check if request succeeded
return nil
}
// Unified HTTP Handler API
// makeRequest handles all HTTP requests with automatic JSON marshaling/unmarshaling
func (c *Client) makeRequest(ctx context.Context, method, endpoint string, requestBody interface{}, responseBody interface{}) error {
// Build full URL
url := fmt.Sprintf("%s%s", c.baseURL, endpoint)
// Marshal request body to JSON if provided
var bodyReader io.Reader
if requestBody != nil {
jsonData, err := json.Marshal(requestBody)
if err != nil {
c.logger.Error("Failed to marshal request body", map[string]interface{}{
"error": err.Error(),
"endpoint": endpoint,
})
return fmt.Errorf("failed to marshal request: %w", err)
}
bodyReader = bytes.NewReader(jsonData)
}
// Create HTTP request
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
if err != nil {
c.logger.Error("Failed to create HTTP request", map[string]interface{}{
"error": err.Error(),
"method": method,
"url": url,
})
return fmt.Errorf("failed to create request: %w", err)
}
// Set headers
if requestBody != nil {
req.Header.Set("Content-Type", "application/json")
}
req.Header.Set("Accept", "application/json")
// Send request
resp, err := c.httpClient.Do(req)
if err != nil {
c.logger.Error("Failed to send HTTP request", map[string]interface{}{
"error": err.Error(),
"method": method,
"url": url,
})
return fmt.Errorf("failed to send request: %w", err)
}
defer resp.Body.Close()
// Read response body
respBody, err := io.ReadAll(resp.Body)
if err != nil {
c.logger.Error("Failed to read response body", map[string]interface{}{
"error": err.Error(),
"url": url,
})
return fmt.Errorf("failed to read response: %w", err)
}
// Handle error responses
if resp.StatusCode >= 400 {
// Try to parse error response
var errorResp map[string]interface{}
if err := json.Unmarshal(respBody, &errorResp); err == nil {
if errorMsg, ok := errorResp["error"].(string); ok {
c.logger.Error("Request failed with error response", map[string]interface{}{
"status_code": resp.StatusCode,
"error": errorMsg,
"url": url,
})
return fmt.Errorf("request failed: %s", errorMsg)
}
}
c.logger.Error("Request failed with status code", map[string]interface{}{
"status_code": resp.StatusCode,
"url": url,
"body": string(respBody),
})
return fmt.Errorf("request failed with status %d", resp.StatusCode)
}
// Handle successful responses
if responseBody != nil {
if err := json.Unmarshal(respBody, responseBody); err != nil {
c.logger.Error("Failed to unmarshal response", map[string]interface{}{
"error": err.Error(),
"url": url,
"body": string(respBody),
})
return fmt.Errorf("failed to unmarshal response: %w", err)
}
}
c.logger.Debug("Request completed successfully", map[string]interface{}{
"method": method,
"url": url,
"status_code": resp.StatusCode,
})
return nil
}