mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'ted one-time api' (#19) from TM-369 into develop

Browse Source 
Reviewed-on: https://repo.ravanertebat.com/TM/tm_back/pulls/19
This commit is contained in:
m.nazemi
2026-02-23 15:41:56 +03:30
parent 80befb51c7 e0e8b9cf04
commit d5a96a7cf7
7 changed files with 234 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/web/bootstrap/config.go
+16
View File
@@ -18,6 +18,7 @@ type Config struct {
HCaptcha config.HCaptchaConfig
GLM GLMConfig
Scraper ScraperConfig
TED TEDConfig
UserAuth AuthConfig
CustomerAuth AuthConfig
}
@@ -56,6 +57,21 @@ type ScraperConfig struct {
Timeout time.Duration `env:"SCRAPER_TIMEOUT"`
}
// TEDConfig holds configuration for TED scraper (one-time scraping from admin panel)
type TEDConfig struct {
BaseURL string `env:"TED_BASE_URL" envDefault:"https://ted.europa.eu"`
Timeout time.Duration `env:"TED_TIMEOUT" envDefault:"30s"`
MaxRetries int `env:"TED_MAX_RETRIES" envDefault:"3"`
RetryDelay time.Duration `env:"TED_RETRY_DELAY" envDefault:"5s"`
UserAgent string `env:"TED_USER_AGENT" envDefault:"TM-TED-Scraper/1.0"`
MaxConcurrency int `env:"TED_MAX_CONCURRENCY" envDefault:"5"`
DownloadDir string `env:"TED_DOWNLOAD_DIR" envDefault:"./downloads"`
CleanupAfter time.Duration `env:"TED_CLEANUP_AFTER" envDefault:"24h"`
ScrapingInterval string `env:"TED_SCRAPING_INTERVAL" envDefault:"* * 10 * * *"`
MaxNoticesPerDay int `env:"TED_MAX_NOTICES_PER_DAY" envDefault:"0"`
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
}
type AuthConfig struct {
JWT JWTConfig
}
cmd/web/main.go
+55 -2
View File
@@ -103,6 +103,7 @@ import (
"tm/internal/feedback"
"tm/internal/inquiry"
"tm/internal/kanban"
"tm/internal/notice"
"tm/internal/notification"
"tm/internal/scraper"
"tm/internal/tender"
@@ -115,6 +116,56 @@ import (
"tm/cmd/web/router"
)
func buildTEDConfig(c *bootstrap.TEDConfig) *scraper.TEDConfig {
if c == nil {
return &scraper.TEDConfig{
BaseURL: "https://ted.europa.eu",
Timeout: 30 * time.Second,
MaxRetries: 3,
RetryDelay: 5 * time.Second,
UserAgent: "TM-TED-Scraper/1.0",
MaxConcurrency: 5,
DownloadDir: "./downloads",
CleanupAfter: 24 * time.Hour,
ScrapingInterval: "* * 10 * * *",
AlertMail: "alerts@opplens.com",
}
}
baseURL := c.BaseURL
if baseURL == "" {
baseURL = "https://ted.europa.eu"
}
userAgent := c.UserAgent
if userAgent == "" {
userAgent = "TM-TED-Scraper/1.0"
}
downloadDir := c.DownloadDir
if downloadDir == "" {
downloadDir = "./downloads"
}
scrapingInterval := c.ScrapingInterval
if scrapingInterval == "" {
scrapingInterval = "* * 10 * * *"
}
alertMail := c.AlertMail
if alertMail == "" {
alertMail = "alerts@opplens.com"
}
return &scraper.TEDConfig{
BaseURL: baseURL,
Timeout: c.Timeout,
MaxRetries: c.MaxRetries,
RetryDelay: c.RetryDelay,
UserAgent: userAgent,
MaxConcurrency: c.MaxConcurrency,
DownloadDir: downloadDir,
CleanupAfter: c.CleanupAfter,
ScrapingInterval: scrapingInterval,
MaxNoticesPerDay: c.MaxNoticesPerDay,
AlertMail: alertMail,
}
}
func main() {
conf := bootstrap.InitConfig()
@@ -168,11 +219,12 @@ func main() {
inquiryRepository := inquiry.NewRepository(mongoManager, logger)
contactRepository := contact.NewContactRepository(mongoManager, logger)
cmsRepository := cms.NewRepository(mongoManager, logger)
noticeRepository := notice.NewRepository(mongoManager, logger)
boardRepository := kanban.NewBoardRepository(mongoManager, logger)
columnRepository := kanban.NewColumnRepository(mongoManager, logger)
cardRepository := kanban.NewCardRepository(mongoManager, logger)
logger.Info("Repositories initialized successfully", map[string]interface{}{
"repositories": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "contact", "cms", "kanban_board", "kanban_column", "kanban_card"},
"repositories": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "contact", "cms", "notice", "kanban_board", "kanban_column", "kanban_card"},
})
// Initialize validation services
@@ -193,7 +245,8 @@ func main() {
notificationService := notification.NewService(notificationSDK, userService, customerService, logger)
contactService := contact.NewService(contactRepository, logger, notificationSDK)
cmsService := cms.NewService(cmsRepository, logger)
scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger)
tedConfig := buildTEDConfig(&conf.TED)
scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig)
kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger)
logger.Info("Services initialized successfully", map[string]interface{}{
"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban"},
cmd/web/router/routes.go
+2 -1
View File
@@ -181,8 +181,9 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
// Scraper Routes
scraperGP := adminV1.Group("/scraper")
{
scraperGP.Use(userHandler.AuthMiddleware())
scraperGP.Use(userHandler.AuthMiddleware(), userHandler.AdminMiddleware())
scraperGP.POST("/download", scraperHandler.ScrapeDocuments)
scraperGP.POST("/ted/one-time", scraperHandler.TriggerTEDOneTimeScraping)
}
// Kanban Routes
internal/scraper/form.go
+6
View File
@@ -35,3 +35,9 @@ type BucketInfo struct {
NoticeID string `json:"notice_id"`
Count int `json:"count"`
}
// TEDOneTimeScrapeForm represents the request form for triggering TED one-time scraping
type TEDOneTimeScrapeForm struct {
FromDate string `json:"from_date" valid:"required" example:"01/01/2026"`
ToDate string `json:"to_date" valid:"required" example:"19/02/2026"`
}
internal/scraper/handler.go
+28
View File
@@ -42,3 +42,31 @@ func (h *Handler) ScrapeDocuments(c echo.Context) error {
return response.Success(c, result, "Documents scraped and uploaded successfully")
}
// TriggerTEDOneTimeScraping handles TED one-time scraping requests
// @Summary Trigger TED one-time scraping
// @Description Starts TED one-time scraping for the specified date range. The operation runs asynchronously; check logs for progress.
// @Tags Admin-Scraper
// @Accept json
// @Produce json
// @Param request body TEDOneTimeScrapeForm true "Date range for scraping (DD/MM/YYYY)"
// @Success 202 {object} response.APIResponse "TED scraping started"
// @Failure 400 {object} response.APIResponse "Invalid date format or range"
// @Failure 500 {object} response.APIResponse "TED scraping not configured"
// @Security BearerAuth
// @Router /admin/v1/scraper/ted/one-time [post]
func (h *Handler) TriggerTEDOneTimeScraping(c echo.Context) error {
form, err := response.Parse[TEDOneTimeScrapeForm](c)
if err != nil {
return response.ValidationError(c, "Invalid request data", err.Error())
}
err = h.service.TriggerTEDOneTimeScraping(c.Request().Context(), form)
if err != nil {
return response.BadRequest(c, err.Error(), "")
}
return response.Accepted(c, map[string]string{
"message": "TED one-time scraping started. Check logs for progress.",
}, "TED one-time scraping started")
}
internal/scraper/service.go
+118 -2
View File
@@ -2,22 +2,48 @@ package scraper
import (
"context"
"fmt"
"time"
"tm/internal/notice"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/scraper"
"tm/ted"
)
// TEDConfig holds configuration for TED one-time scraping (used when creating service)
type TEDConfig struct {
BaseURL string
Timeout time.Duration
MaxRetries int
RetryDelay time.Duration
UserAgent string
MaxConcurrency int
DownloadDir string
CleanupAfter time.Duration
ScrapingInterval string
MaxNoticesPerDay int
AlertMail string
}
// Service defines the interface for scraper business operations
type Service interface {
// ScrapeDocuments scrapes documents and uploads them to MinIO
ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range (runs in background)
TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error
}
// scraperService implements Service interface
type scraperService struct {
sdk scraper.SDK
logger logger.Logger
sdk scraper.SDK
logger logger.Logger
mongoManager *mongo.ConnectionManager
noticeRepo notice.Repository
notificationSDK notification.SDK
tedConfig *TEDConfig
}
// NewService creates a new scraper service with stored credentials
@@ -29,6 +55,19 @@ func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Ser
}
}
// NewServiceWithTED creates a new scraper service with both TendSign SDK and TED scraping support
func NewServiceWithTED(baseURL string, timeout time.Duration, logger logger.Logger, mongoManager *mongo.ConnectionManager, noticeRepo notice.Repository, notificationSDK notification.SDK, tedConfig *TEDConfig) Service {
sdk := scraper.NewClient(baseURL, timeout, logger)
return &scraperService{
sdk: sdk,
logger: logger,
mongoManager: mongoManager,
noticeRepo: noticeRepo,
notificationSDK: notificationSDK,
tedConfig: tedConfig,
}
}
// ScrapeDocuments scrapes documents and uploads them to MinIO
func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
s.logger.Info("Starting document scraping", map[string]interface{}{
@@ -95,3 +134,80 @@ func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeReques
return serviceResp, nil
}
// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range.
// The scraping runs in a background goroutine; the handler returns 202 Accepted immediately.
func (s *scraperService) TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error {
if s.tedConfig == nil || s.mongoManager == nil || s.noticeRepo == nil {
return fmt.Errorf("TED scraping is not configured")
}
from, err := time.Parse("02/01/2006", form.FromDate)
if err != nil {
return fmt.Errorf("invalid from_date format: expected DD/MM/YYYY (e.g. 01/01/2026)")
}
to, err := time.Parse("02/01/2006", form.ToDate)
if err != nil {
return fmt.Errorf("invalid to_date format: expected DD/MM/YYYY (e.g. 19/02/2026)")
}
if from.After(to) {
return fmt.Errorf("from_date cannot be after to_date")
}
tedConfig := s.tedConfig
mongoManager := s.mongoManager
noticeRepo := s.noticeRepo
notificationSDK := s.notificationSDK
log := s.logger
go func() {
tedScraperConfig := &ted.Config{
BaseURL: tedConfig.BaseURL,
UserAgent: tedConfig.UserAgent,
DownloadDir: tedConfig.DownloadDir,
MaxRetries: tedConfig.MaxRetries,
MaxConcurrency: tedConfig.MaxConcurrency,
Timeout: tedConfig.Timeout,
RetryDelay: tedConfig.RetryDelay,
CleanupAfter: tedConfig.CleanupAfter,
ScrapingInterval: tedConfig.ScrapingInterval,
AlertMail: tedConfig.AlertMail,
MaxNoticesPerDay: tedConfig.MaxNoticesPerDay,
}
tedScraper := ted.NewTEDScraper(
tedScraperConfig,
log,
mongoManager,
noticeRepo,
notificationSDK,
)
runCtx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
defer cancel()
log.Info("Starting TED one-time scraping (triggered via API)", map[string]interface{}{
"from_date": from.Format("02/01/2006"),
"to_date": to.Format("02/01/2006"),
})
err := tedScraper.Run(runCtx, &from, &to)
if err != nil {
log.Error("TED one-time scraping failed", map[string]interface{}{
"error": err.Error(),
"from_date": from.Format("02/01/2006"),
"to_date": to.Format("02/01/2006"),
})
return
}
log.Info("TED one-time scraping completed successfully", map[string]interface{}{
"from_date": from.Format("02/01/2006"),
"to_date": to.Format("02/01/2006"),
})
}()
return nil
}
pkg/response/response.go
+9
View File
@@ -68,6 +68,15 @@ func Created(c echo.Context, data interface{}, message string) error {
})
}
// Accepted returns a 202 Accepted response (for async operations)
func Accepted(c echo.Context, data interface{}, message string) error {
return c.JSON(http.StatusAccepted, APIResponse{
Success: true,
Message: message,
Data: data,
})
}
// BadRequest returns a 400 Bad Request response
func BadRequest(c echo.Context, message, details string) error {
return c.JSON(http.StatusBadRequest, APIResponse{