mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removed old document scraper service

Browse Source
This commit is contained in:
m.nazemi
2026-05-03 13:46:38 +03:30
parent 0bd5e41b70
commit 446c11dfbf
11 changed files with 9 additions and 1371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/scraper/form.go
-43
View File
@@ -1,43 +0,0 @@
package scraper
// ScrapeRequestForm represents the request form for scraping documents
type ScrapeRequestForm struct {
NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
}
// ScrapeResponse represents the response from scraping operation
type ScrapeResponse struct {
Success bool `json:"success"`
NoticeID string `json:"notice_id"`
UploadedCount int `json:"uploaded_count"`
UploadedFiles []UploadedFileInfo `json:"uploaded_files"`
Errors []FileError `json:"errors,omitempty"`
Buckets []BucketInfo `json:"buckets,omitempty"`
}
// UploadedFileInfo represents information about an uploaded file
type UploadedFileInfo struct {
Filename string `json:"filename"`
ObjectName string `json:"object_name"`
Size int64 `json:"size"`
}
// FileError represents an error for a specific file
type FileError struct {
Filename string `json:"filename"`
Error string `json:"error"`
}
// BucketInfo represents information about a bucket
type BucketInfo struct {
Name string `json:"name"`
NoticeID string `json:"notice_id"`
Count int `json:"count"`
}
// TEDOneTimeScrapeForm represents the request form for triggering TED one-time scraping
type TEDOneTimeScrapeForm struct {
FromDate string `json:"from_date" valid:"required" example:"01/01/2026"`
ToDate string `json:"to_date" valid:"required" example:"19/02/2026"`
}
internal/scraper/handler.go
-72
View File
@@ -1,72 +0,0 @@
package scraper
import (
"tm/pkg/response"
"github.com/labstack/echo/v4"
)
// Handler handles HTTP requests for scraper operations
type Handler struct {
service Service
}
// NewHandler creates a new scraper handler
func NewHandler(service Service) *Handler {
return &Handler{
service: service,
}
}
// ScrapeDocuments handles document scraping requests
// @Summary Scrape documents from external source
// @Description Scrapes documents from TendSign and uploads them to MinIO storage
// @Tags Admin-Scraper
// @Accept json
// @Produce json
// @Param request body ScrapeRequestForm true "Scraping request information"
// @Success 200 {object} response.Response{data=ScrapeResponse}
// @Failure 400 {object} response.Response
// @Failure 500 {object} response.Response
// @Router /admin/v1/scraper/download [post]
func (h *Handler) ScrapeDocuments(c echo.Context) error {
form, err := response.Parse[ScrapeRequestForm](c)
if err != nil {
return response.ValidationError(c, "Invalid request data", err.Error())
}
result, err := h.service.ScrapeDocuments(c.Request().Context(), form)
if err != nil {
return response.InternalServerError(c, "Failed to scrape documents")
}
return response.Success(c, result, "Documents scraped and uploaded successfully")
}
// TriggerTEDOneTimeScraping handles TED one-time scraping requests
// @Summary Trigger TED one-time scraping
// @Description Starts TED one-time scraping for the specified date range. The operation runs asynchronously; check logs for progress.
// @Tags Admin-Scraper
// @Accept json
// @Produce json
// @Param request body TEDOneTimeScrapeForm true "Date range for scraping (DD/MM/YYYY)"
// @Success 202 {object} response.APIResponse "TED scraping started"
// @Failure 400 {object} response.APIResponse "Invalid date format or range"
// @Failure 500 {object} response.APIResponse "TED scraping not configured"
// @Security BearerAuth
// @Router /admin/v1/scraper/ted/one-time [post]
func (h *Handler) TriggerTEDOneTimeScraping(c echo.Context) error {
form, err := response.Parse[TEDOneTimeScrapeForm](c)
if err != nil {
return response.ValidationError(c, "Invalid request data", err.Error())
}
err = h.service.TriggerTEDOneTimeScraping(c.Request().Context(), form)
if err != nil {
return response.BadRequest(c, err.Error(), "")
}
return response.Accepted(c, map[string]string{
"message": "TED one-time scraping started. Check logs for progress.",
}, "TED one-time scraping started")
}
internal/scraper/service.go
-213
View File
@@ -1,213 +0,0 @@
package scraper
import (
"context"
"fmt"
"time"
"tm/internal/notice"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/scraper"
"tm/ted"
)
// TEDConfig holds configuration for TED one-time scraping (used when creating service)
type TEDConfig struct {
BaseURL string
Timeout time.Duration
MaxRetries int
RetryDelay time.Duration
UserAgent string
MaxConcurrency int
DownloadDir string
CleanupAfter time.Duration
ScrapingInterval string
MaxNoticesPerDay int
AlertMail string
}
// Service defines the interface for scraper business operations
type Service interface {
// ScrapeDocuments scrapes documents and uploads them to MinIO
ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range (runs in background)
TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error
}
// scraperService implements Service interface
type scraperService struct {
sdk scraper.SDK
logger logger.Logger
mongoManager *mongo.ConnectionManager
noticeRepo notice.Repository
notificationSDK notification.SDK
tedConfig *TEDConfig
}
// NewService creates a new scraper service with stored credentials
func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
sdk := scraper.NewClient(baseURL, timeout, logger)
return &scraperService{
sdk: sdk,
logger: logger,
}
}
// NewServiceWithTED creates a new scraper service with both TendSign SDK and TED scraping support
func NewServiceWithTED(baseURL string, timeout time.Duration, logger logger.Logger, mongoManager *mongo.ConnectionManager, noticeRepo notice.Repository, notificationSDK notification.SDK, tedConfig *TEDConfig) Service {
sdk := scraper.NewClient(baseURL, timeout, logger)
return &scraperService{
sdk: sdk,
logger: logger,
mongoManager: mongoManager,
noticeRepo: noticeRepo,
notificationSDK: notificationSDK,
tedConfig: tedConfig,
}
}
// ScrapeDocuments scrapes documents and uploads them to MinIO
func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
s.logger.Info("Starting document scraping", map[string]interface{}{
"notice_id": form.NoticeID,
})
// Convert form to SDK request (only dynamic fields)
req := &scraper.ScrapeRequest{
NoticeID: form.NoticeID,
NoticeURL: form.NoticeURL,
}
// Call SDK (credentials are already configured)
response, err := s.sdk.ScrapeDocuments(ctx, req)
if err != nil {
s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{
"error": err.Error(),
"notice_id": form.NoticeID,
})
return nil, err
}
// Convert SDK response to service response
serviceResp := &ScrapeResponse{
Success: response.Success,
NoticeID: response.NoticeID,
UploadedCount: response.UploadedCount,
UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)),
Errors: make([]FileError, len(response.Errors)),
Buckets: make([]BucketInfo, len(response.Buckets)),
}
// Convert uploaded files
for i, file := range response.UploadedFiles {
serviceResp.UploadedFiles[i] = UploadedFileInfo{
Filename: file.Filename,
ObjectName: file.ObjectName,
Size: file.Size,
}
}
// Convert errors
for i, err := range response.Errors {
serviceResp.Errors[i] = FileError{
Filename: err.Filename,
Error: err.Error,
}
}
// Convert buckets
for i, bucket := range response.Buckets {
serviceResp.Buckets[i] = BucketInfo{
Name: bucket.Name,
NoticeID: bucket.NoticeID,
Count: bucket.Count,
}
}
s.logger.Info("Document scraping completed", map[string]interface{}{
"notice_id": serviceResp.NoticeID,
"uploaded_count": serviceResp.UploadedCount,
"success": serviceResp.Success,
})
return serviceResp, nil
}
// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range.
// The scraping runs in a background goroutine; the handler returns 202 Accepted immediately.
func (s *scraperService) TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error {
if s.tedConfig == nil || s.mongoManager == nil || s.noticeRepo == nil {
return fmt.Errorf("TED scraping is not configured")
}
from, err := time.Parse("02/01/2006", form.FromDate)
if err != nil {
return fmt.Errorf("invalid from_date format: expected DD/MM/YYYY (e.g. 01/01/2026)")
}
to, err := time.Parse("02/01/2006", form.ToDate)
if err != nil {
return fmt.Errorf("invalid to_date format: expected DD/MM/YYYY (e.g. 19/02/2026)")
}
if from.After(to) {
return fmt.Errorf("from_date cannot be after to_date")
}
tedConfig := s.tedConfig
mongoManager := s.mongoManager
noticeRepo := s.noticeRepo
notificationSDK := s.notificationSDK
log := s.logger
go func() {
tedScraperConfig := &ted.Config{
BaseURL: tedConfig.BaseURL,
UserAgent: tedConfig.UserAgent,
DownloadDir: tedConfig.DownloadDir,
MaxRetries: tedConfig.MaxRetries,
MaxConcurrency: tedConfig.MaxConcurrency,
Timeout: tedConfig.Timeout,
RetryDelay: tedConfig.RetryDelay,
CleanupAfter: tedConfig.CleanupAfter,
ScrapingInterval: tedConfig.ScrapingInterval,
AlertMail: tedConfig.AlertMail,
MaxNoticesPerDay: tedConfig.MaxNoticesPerDay,
}
tedScraper := ted.NewTEDScraper(
tedScraperConfig,
log,
mongoManager,
noticeRepo,
notificationSDK,
)
runCtx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
defer cancel()
log.Info("Starting TED one-time scraping (triggered via API)", map[string]interface{}{
"from_date": from.Format("02/01/2006"),
"to_date": to.Format("02/01/2006"),
})
err := tedScraper.Run(runCtx, &from, &to)
if err != nil {
log.Error("TED one-time scraping failed", map[string]interface{}{
"error": err.Error(),
"from_date": from.Format("02/01/2006"),
"to_date": to.Format("02/01/2006"),
})
return
}
log.Info("TED one-time scraping completed successfully", map[string]interface{}{
"from_date": from.Format("02/01/2006"),
"to_date": to.Format("02/01/2006"),
})
}()
return nil
}