Implement TED Scraper and Configuration Management

- Introduced a new TED scraper in `cmd/scraper` to handle downloading and parsing TED XML files. - Added configuration management for the scraper, including a new `Config` struct and YAML configuration file. - Created necessary handler, service, and repository layers for tender management, adhering to Clean Architecture principles. - Implemented comprehensive logging and error handling throughout the scraper functionality. - Updated API routes to include tender management operations, enhancing the overall system capabilities.
2025-08-13 16:52:24 +03:30
parent b7fa012d13
commit 96c21b8b78
16 changed files with 6970 additions and 7 deletions
@@ -0,0 +1,523 @@
+package tender
+
+import (
+	"context"
+	"fmt"
+	"time"
+	"tm/pkg/logger"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+// TenderService interface defines tender business logic operations
+type TenderService interface {
+	// Tender operations
+	CreateTender(ctx context.Context, req CreateTenderRequest) (*Tender, error)
+	GetTenderByID(ctx context.Context, id string) (*Tender, error)
+	UpdateTender(ctx context.Context, req UpdateTenderRequest) (*Tender, error)
+	DeleteTender(ctx context.Context, id string) error
+	ListTenders(ctx context.Context, req ListTendersRequest) (*ListTendersResponse, error)
+	SearchTenders(ctx context.Context, req SearchTendersRequest) (*SearchTendersResponse, error)
+
+	// Statistics and reporting
+	GetTenderStatistics(ctx context.Context) (*TenderStatistics, error)
+	GetDashboardData(ctx context.Context) (*DashboardData, error)
+
+	// Scraping management
+	StartScrapingJob(ctx context.Context, req StartScrapingJobRequest) (*ScrapingJob, error)
+	GetScrapingJobStatus(ctx context.Context, jobID string) (*ScrapingJob, error)
+	ListScrapingJobs(ctx context.Context, req ListScrapingJobsRequest) (*ListScrapingJobsResponse, error)
+	CancelScrapingJob(ctx context.Context, jobID string) error
+
+	// Maintenance operations
+	UpdateExpiredTenders(ctx context.Context) error
+	CleanupOldData(ctx context.Context, olderThan time.Duration) error
+}
+
+// tenderService implements TenderService interface
+type tenderService struct {
+	repository TenderRepository
+	logger     logger.Logger
+}
+
+// NewTenderService creates a new tender service
+func NewTenderService(repository TenderRepository, logger logger.Logger) TenderService {
+	return &tenderService{
+		repository: repository,
+		logger:     logger,
+	}
+}
+
+// CreateTender creates a new tender
+func (s *tenderService) CreateTender(ctx context.Context, req CreateTenderRequest) (*Tender, error) {
+	s.logger.Info("Creating new tender", map[string]interface{}{
+		"notice_publication_id": req.NoticePublicationID,
+		"title":                 req.Title,
+	})
+
+	// Check if tender with same notice publication ID already exists
+	if req.NoticePublicationID != "" {
+		existing, err := s.repository.GetByNoticePublicationID(ctx, req.NoticePublicationID)
+		if err == nil && existing != nil {
+			s.logger.Warn("Tender with same notice publication ID already exists", map[string]interface{}{
+				"notice_publication_id": req.NoticePublicationID,
+				"existing_id":           existing.ID,
+			})
+			return nil, fmt.Errorf("tender with notice publication ID %s already exists", req.NoticePublicationID)
+		}
+	}
+
+	tender := &Tender{
+		ContractNoticeID:          req.ContractNoticeID,
+		NoticePublicationID:       req.NoticePublicationID,
+		ContractFolderID:          req.ContractFolderID,
+		NoticeTypeCode:            req.NoticeTypeCode,
+		NoticeSubTypeCode:         req.NoticeSubTypeCode,
+		NoticeLanguageCode:        req.NoticeLanguageCode,
+		IssueDate:                 req.IssueDate,
+		IssueTime:                 req.IssueTime,
+		PublicationDate:           req.PublicationDate,
+		GazetteID:                 req.GazetteID,
+		Title:                     req.Title,
+		Description:               req.Description,
+		ProcurementTypeCode:       req.ProcurementTypeCode,
+		ProcedureCode:             req.ProcedureCode,
+		MainClassification:        req.MainClassification,
+		AdditionalClassifications: req.AdditionalClassifications,
+		EstimatedValue:            req.EstimatedValue,
+		Currency:                  req.Currency,
+		Duration:                  req.Duration,
+		DurationUnit:              req.DurationUnit,
+		TenderDeadline:            req.TenderDeadline,
+		PlaceOfPerformance:        req.PlaceOfPerformance,
+		CountryCode:               req.CountryCode,
+		RegionCode:                req.RegionCode,
+		CityName:                  req.CityName,
+		PostalCode:                req.PostalCode,
+		DocumentURI:               req.DocumentURI,
+		TenderURL:                 req.TenderURL,
+		BuyerOrganization:         req.BuyerOrganization,
+		ReviewOrganization:        req.ReviewOrganization,
+		Organizations:             req.Organizations,
+		SelectionCriteria:         req.SelectionCriteria,
+		OfficialLanguages:         req.OfficialLanguages,
+		Status:                    TenderStatusActive,
+		Source:                    req.Source,
+		SourceFileURL:             req.SourceFileURL,
+		SourceFileName:            req.SourceFileName,
+		ProcessingMetadata: ProcessingMetadata{
+			ProcessedAt:       time.Now().Unix(),
+			ProcessingVersion: "1.0",
+		},
+	}
+
+	// Generate tender URL if not provided
+	if tender.TenderURL == "" && tender.NoticePublicationID != "" {
+		tender.TenderURL = generateTenderURL(tender.NoticePublicationID)
+	}
+
+	if err := s.repository.Create(ctx, tender); err != nil {
+		s.logger.Error("Failed to create tender", map[string]interface{}{
+			"error":                 err.Error(),
+			"notice_publication_id": req.NoticePublicationID,
+		})
+		return nil, fmt.Errorf("failed to create tender: %w", err)
+	}
+
+	s.logger.Info("Tender created successfully", map[string]interface{}{
+		"tender_id":             tender.ID,
+		"notice_publication_id": tender.NoticePublicationID,
+	})
+
+	return tender, nil
+}
+
+// GetTenderByID retrieves a tender by ID
+func (s *tenderService) GetTenderByID(ctx context.Context, id string) (*Tender, error) {
+	s.logger.Info("Retrieving tender by ID", map[string]interface{}{
+		"tender_id": id,
+	})
+
+	tender, err := s.repository.GetByID(ctx, id)
+	if err != nil {
+		s.logger.Error("Failed to retrieve tender", map[string]interface{}{
+			"tender_id": id,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to retrieve tender: %w", err)
+	}
+
+	return tender, nil
+}
+
+// UpdateTender updates an existing tender
+func (s *tenderService) UpdateTender(ctx context.Context, req UpdateTenderRequest) (*Tender, error) {
+	s.logger.Info("Updating tender", map[string]interface{}{
+		"tender_id": req.ID,
+	})
+
+	// Get existing tender
+	tender, err := s.repository.GetByID(ctx, req.ID)
+	if err != nil {
+		s.logger.Error("Failed to find tender for update", map[string]interface{}{
+			"tender_id": req.ID,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to find tender: %w", err)
+	}
+
+	// Update fields if provided
+	if req.Title != nil {
+		tender.Title = *req.Title
+	}
+	if req.Description != nil {
+		tender.Description = *req.Description
+	}
+	if req.Status != nil {
+		tender.Status = *req.Status
+	}
+	if req.EstimatedValue != nil {
+		tender.EstimatedValue = *req.EstimatedValue
+	}
+	if req.Currency != nil {
+		tender.Currency = *req.Currency
+	}
+	if req.TenderDeadline != nil {
+		tender.TenderDeadline = *req.TenderDeadline
+	}
+
+	if err := s.repository.Update(ctx, tender); err != nil {
+		s.logger.Error("Failed to update tender", map[string]interface{}{
+			"tender_id": req.ID,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to update tender: %w", err)
+	}
+
+	s.logger.Info("Tender updated successfully", map[string]interface{}{
+		"tender_id": tender.ID,
+	})
+
+	return tender, nil
+}
+
+// DeleteTender deletes a tender
+func (s *tenderService) DeleteTender(ctx context.Context, id string) error {
+	s.logger.Info("Deleting tender", map[string]interface{}{
+		"tender_id": id,
+	})
+
+	if err := s.repository.Delete(ctx, id); err != nil {
+		s.logger.Error("Failed to delete tender", map[string]interface{}{
+			"tender_id": id,
+			"error":     err.Error(),
+		})
+		return fmt.Errorf("failed to delete tender: %w", err)
+	}
+
+	s.logger.Info("Tender deleted successfully", map[string]interface{}{
+		"tender_id": id,
+	})
+
+	return nil
+}
+
+// ListTenders retrieves tenders with pagination and filtering
+func (s *tenderService) ListTenders(ctx context.Context, req ListTendersRequest) (*ListTendersResponse, error) {
+	s.logger.Info("Listing tenders", map[string]interface{}{
+		"limit":  req.Limit,
+		"offset": req.Offset,
+	})
+
+	if req.Limit <= 0 {
+		req.Limit = 20
+	}
+	if req.Limit > 100 {
+		req.Limit = 100
+	}
+
+	tenders, total, err := s.repository.List(ctx, req.Criteria, req.Limit, req.Offset)
+	if err != nil {
+		s.logger.Error("Failed to list tenders", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to list tenders: %w", err)
+	}
+
+	return &ListTendersResponse{
+		Tenders: tenders,
+		Total:   total,
+		Limit:   req.Limit,
+		Offset:  req.Offset,
+	}, nil
+}
+
+// SearchTenders searches tenders based on criteria
+func (s *tenderService) SearchTenders(ctx context.Context, req SearchTendersRequest) (*SearchTendersResponse, error) {
+	s.logger.Info("Searching tenders", map[string]interface{}{
+		"query": req.Query,
+	})
+
+	tenders, err := s.repository.Search(ctx, req.Criteria)
+	if err != nil {
+		s.logger.Error("Failed to search tenders", map[string]interface{}{
+			"error": err.Error(),
+			"query": req.Query,
+		})
+		return nil, fmt.Errorf("failed to search tenders: %w", err)
+	}
+
+	return &SearchTendersResponse{
+		Tenders: tenders,
+		Query:   req.Query,
+		Total:   len(tenders),
+	}, nil
+}
+
+// GetTenderStatistics retrieves tender statistics
+func (s *tenderService) GetTenderStatistics(ctx context.Context) (*TenderStatistics, error) {
+	s.logger.Info("Retrieving tender statistics", map[string]interface{}{})
+
+	stats, err := s.repository.GetStatistics(ctx)
+	if err != nil {
+		s.logger.Error("Failed to retrieve tender statistics", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to retrieve statistics: %w", err)
+	}
+
+	return stats, nil
+}
+
+// GetDashboardData retrieves dashboard data
+func (s *tenderService) GetDashboardData(ctx context.Context) (*DashboardData, error) {
+	s.logger.Info("Retrieving dashboard data", map[string]interface{}{})
+
+	stats, err := s.GetTenderStatistics(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get recent tenders
+	recentTenders, _, err := s.repository.List(ctx, TenderSearchCriteria{}, 10, 0)
+	if err != nil {
+		s.logger.Error("Failed to retrieve recent tenders", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to retrieve recent tenders: %w", err)
+	}
+
+	// Get active scraping jobs
+	activeJobs, err := s.repository.GetActiveScrapingJobs(ctx)
+	if err != nil {
+		s.logger.Error("Failed to retrieve active scraping jobs", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to retrieve active jobs: %w", err)
+	}
+
+	return &DashboardData{
+		Statistics:    stats,
+		RecentTenders: recentTenders,
+		ActiveJobs:    activeJobs,
+	}, nil
+}
+
+// StartScrapingJob starts a new scraping job
+func (s *tenderService) StartScrapingJob(ctx context.Context, req StartScrapingJobRequest) (*ScrapingJob, error) {
+	s.logger.Info("Starting scraping job", map[string]interface{}{
+		"type": req.Type,
+	})
+
+	job := &ScrapingJob{
+		ID:        primitive.NewObjectID().Hex(),
+		Type:      req.Type,
+		Status:    ScrapingJobStatusPending,
+		StartedAt: time.Now().Unix(),
+		Progress:  JobProgress{},
+		Results:   JobResults{},
+		Config:    req.Config,
+	}
+
+	if err := s.repository.CreateScrapingJob(ctx, job); err != nil {
+		s.logger.Error("Failed to create scraping job", map[string]interface{}{
+			"error": err.Error(),
+			"type":  req.Type,
+		})
+		return nil, fmt.Errorf("failed to create scraping job: %w", err)
+	}
+
+	s.logger.Info("Scraping job created successfully", map[string]interface{}{
+		"job_id": job.ID,
+		"type":   job.Type,
+	})
+
+	return job, nil
+}
+
+// GetScrapingJobStatus retrieves scraping job status
+func (s *tenderService) GetScrapingJobStatus(ctx context.Context, jobID string) (*ScrapingJob, error) {
+	s.logger.Info("Retrieving scraping job status", map[string]interface{}{
+		"job_id": jobID,
+	})
+
+	job, err := s.repository.GetScrapingJob(ctx, jobID)
+	if err != nil {
+		s.logger.Error("Failed to retrieve scraping job", map[string]interface{}{
+			"job_id": jobID,
+			"error":  err.Error(),
+		})
+		return nil, fmt.Errorf("failed to retrieve scraping job: %w", err)
+	}
+
+	return job, nil
+}
+
+// ListScrapingJobs lists scraping jobs with pagination
+func (s *tenderService) ListScrapingJobs(ctx context.Context, req ListScrapingJobsRequest) (*ListScrapingJobsResponse, error) {
+	s.logger.Info("Listing scraping jobs", map[string]interface{}{
+		"limit":  req.Limit,
+		"offset": req.Offset,
+	})
+
+	if req.Limit <= 0 {
+		req.Limit = 20
+	}
+	if req.Limit > 100 {
+		req.Limit = 100
+	}
+
+	jobs, total, err := s.repository.ListScrapingJobs(ctx, req.Limit, req.Offset)
+	if err != nil {
+		s.logger.Error("Failed to list scraping jobs", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to list scraping jobs: %w", err)
+	}
+
+	return &ListScrapingJobsResponse{
+		Jobs:   jobs,
+		Total:  total,
+		Limit:  req.Limit,
+		Offset: req.Offset,
+	}, nil
+}
+
+// CancelScrapingJob cancels a running scraping job
+func (s *tenderService) CancelScrapingJob(ctx context.Context, jobID string) error {
+	s.logger.Info("Cancelling scraping job", map[string]interface{}{
+		"job_id": jobID,
+	})
+
+	job, err := s.repository.GetScrapingJob(ctx, jobID)
+	if err != nil {
+		s.logger.Error("Failed to find scraping job for cancellation", map[string]interface{}{
+			"job_id": jobID,
+			"error":  err.Error(),
+		})
+		return fmt.Errorf("failed to find scraping job: %w", err)
+	}
+
+	if job.Status != ScrapingJobStatusRunning && job.Status != ScrapingJobStatusPending {
+		s.logger.Warn("Cannot cancel scraping job in current status", map[string]interface{}{
+			"job_id": jobID,
+			"status": job.Status,
+		})
+		return fmt.Errorf("cannot cancel job in status: %s", job.Status)
+	}
+
+	job.Status = ScrapingJobStatusCancelled
+	completedAt := time.Now().Unix()
+	job.CompletedAt = &completedAt
+	duration := completedAt - job.StartedAt
+	job.Duration = &duration
+
+	if err := s.repository.UpdateScrapingJob(ctx, job); err != nil {
+		s.logger.Error("Failed to update cancelled scraping job", map[string]interface{}{
+			"job_id": jobID,
+			"error":  err.Error(),
+		})
+		return fmt.Errorf("failed to update scraping job: %w", err)
+	}
+
+	s.logger.Info("Scraping job cancelled successfully", map[string]interface{}{
+		"job_id": jobID,
+	})
+
+	return nil
+}
+
+// UpdateExpiredTenders updates status of expired tenders
+func (s *tenderService) UpdateExpiredTenders(ctx context.Context) error {
+	s.logger.Info("Updating expired tenders", map[string]interface{}{})
+
+	// Get tenders that should be expired
+	expiredTenders, err := s.repository.GetExpiredTenders(ctx, time.Now().Unix())
+	if err != nil {
+		s.logger.Error("Failed to get expired tenders", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return fmt.Errorf("failed to get expired tenders: %w", err)
+	}
+
+	updatedCount := 0
+	for _, tender := range expiredTenders {
+		if tender.Status != TenderStatusExpired {
+			tender.UpdateStatus(TenderStatusExpired)
+			if err := s.repository.Update(ctx, &tender); err != nil {
+				s.logger.Error("Failed to update expired tender", map[string]interface{}{
+					"tender_id": tender.ID,
+					"error":     err.Error(),
+				})
+				continue
+			}
+			updatedCount++
+		}
+	}
+
+	s.logger.Info("Expired tenders updated", map[string]interface{}{
+		"total_expired": len(expiredTenders),
+		"updated":       updatedCount,
+	})
+
+	return nil
+}
+
+// CleanupOldData removes old data based on retention policy
+func (s *tenderService) CleanupOldData(ctx context.Context, olderThan time.Duration) error {
+	s.logger.Info("Cleaning up old data", map[string]interface{}{
+		"older_than": olderThan.String(),
+	})
+
+	cutoffTime := time.Now().Add(-olderThan).Unix()
+
+	// For now, we'll just log what would be cleaned up
+	// In production, you might want to actually delete old data
+	criteria := TenderSearchCriteria{
+		Status: []TenderStatus{TenderStatusExpired, TenderStatusCancelled, TenderStatusAwarded},
+	}
+
+	oldTenders, err := s.repository.Search(ctx, criteria)
+	if err != nil {
+		s.logger.Error("Failed to find old tenders for cleanup", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return fmt.Errorf("failed to find old tenders: %w", err)
+	}
+
+	cleanupCount := 0
+	for _, tender := range oldTenders {
+		if tender.CreatedAt < cutoffTime {
+			cleanupCount++
+			// Here you would delete the tender if implementing actual cleanup
+			// s.repository.Delete(ctx, tender.ID)
+		}
+	}
+
+	s.logger.Info("Old data cleanup completed", map[string]interface{}{
+		"found_old_tenders": cleanupCount,
+		"cutoff_time":       cutoffTime,
+	})
+
+	return nil
+}