96c21b8b78
- Introduced a new TED scraper in `cmd/scraper` to handle downloading and parsing TED XML files. - Added configuration management for the scraper, including a new `Config` struct and YAML configuration file. - Created necessary handler, service, and repository layers for tender management, adhering to Clean Architecture principles. - Implemented comprehensive logging and error handling throughout the scraper functionality. - Updated API routes to include tender management operations, enhancing the overall system capabilities.
524 lines
16 KiB
Go
524 lines
16 KiB
Go
package tender
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
"tm/pkg/logger"
|
|
|
|
"go.mongodb.org/mongo-driver/bson/primitive"
|
|
)
|
|
|
|
// TenderService interface defines tender business logic operations
|
|
type TenderService interface {
|
|
// Tender operations
|
|
CreateTender(ctx context.Context, req CreateTenderRequest) (*Tender, error)
|
|
GetTenderByID(ctx context.Context, id string) (*Tender, error)
|
|
UpdateTender(ctx context.Context, req UpdateTenderRequest) (*Tender, error)
|
|
DeleteTender(ctx context.Context, id string) error
|
|
ListTenders(ctx context.Context, req ListTendersRequest) (*ListTendersResponse, error)
|
|
SearchTenders(ctx context.Context, req SearchTendersRequest) (*SearchTendersResponse, error)
|
|
|
|
// Statistics and reporting
|
|
GetTenderStatistics(ctx context.Context) (*TenderStatistics, error)
|
|
GetDashboardData(ctx context.Context) (*DashboardData, error)
|
|
|
|
// Scraping management
|
|
StartScrapingJob(ctx context.Context, req StartScrapingJobRequest) (*ScrapingJob, error)
|
|
GetScrapingJobStatus(ctx context.Context, jobID string) (*ScrapingJob, error)
|
|
ListScrapingJobs(ctx context.Context, req ListScrapingJobsRequest) (*ListScrapingJobsResponse, error)
|
|
CancelScrapingJob(ctx context.Context, jobID string) error
|
|
|
|
// Maintenance operations
|
|
UpdateExpiredTenders(ctx context.Context) error
|
|
CleanupOldData(ctx context.Context, olderThan time.Duration) error
|
|
}
|
|
|
|
// tenderService implements TenderService interface
|
|
type tenderService struct {
|
|
repository TenderRepository
|
|
logger logger.Logger
|
|
}
|
|
|
|
// NewTenderService creates a new tender service
|
|
func NewTenderService(repository TenderRepository, logger logger.Logger) TenderService {
|
|
return &tenderService{
|
|
repository: repository,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
// CreateTender creates a new tender
|
|
func (s *tenderService) CreateTender(ctx context.Context, req CreateTenderRequest) (*Tender, error) {
|
|
s.logger.Info("Creating new tender", map[string]interface{}{
|
|
"notice_publication_id": req.NoticePublicationID,
|
|
"title": req.Title,
|
|
})
|
|
|
|
// Check if tender with same notice publication ID already exists
|
|
if req.NoticePublicationID != "" {
|
|
existing, err := s.repository.GetByNoticePublicationID(ctx, req.NoticePublicationID)
|
|
if err == nil && existing != nil {
|
|
s.logger.Warn("Tender with same notice publication ID already exists", map[string]interface{}{
|
|
"notice_publication_id": req.NoticePublicationID,
|
|
"existing_id": existing.ID,
|
|
})
|
|
return nil, fmt.Errorf("tender with notice publication ID %s already exists", req.NoticePublicationID)
|
|
}
|
|
}
|
|
|
|
tender := &Tender{
|
|
ContractNoticeID: req.ContractNoticeID,
|
|
NoticePublicationID: req.NoticePublicationID,
|
|
ContractFolderID: req.ContractFolderID,
|
|
NoticeTypeCode: req.NoticeTypeCode,
|
|
NoticeSubTypeCode: req.NoticeSubTypeCode,
|
|
NoticeLanguageCode: req.NoticeLanguageCode,
|
|
IssueDate: req.IssueDate,
|
|
IssueTime: req.IssueTime,
|
|
PublicationDate: req.PublicationDate,
|
|
GazetteID: req.GazetteID,
|
|
Title: req.Title,
|
|
Description: req.Description,
|
|
ProcurementTypeCode: req.ProcurementTypeCode,
|
|
ProcedureCode: req.ProcedureCode,
|
|
MainClassification: req.MainClassification,
|
|
AdditionalClassifications: req.AdditionalClassifications,
|
|
EstimatedValue: req.EstimatedValue,
|
|
Currency: req.Currency,
|
|
Duration: req.Duration,
|
|
DurationUnit: req.DurationUnit,
|
|
TenderDeadline: req.TenderDeadline,
|
|
PlaceOfPerformance: req.PlaceOfPerformance,
|
|
CountryCode: req.CountryCode,
|
|
RegionCode: req.RegionCode,
|
|
CityName: req.CityName,
|
|
PostalCode: req.PostalCode,
|
|
DocumentURI: req.DocumentURI,
|
|
TenderURL: req.TenderURL,
|
|
BuyerOrganization: req.BuyerOrganization,
|
|
ReviewOrganization: req.ReviewOrganization,
|
|
Organizations: req.Organizations,
|
|
SelectionCriteria: req.SelectionCriteria,
|
|
OfficialLanguages: req.OfficialLanguages,
|
|
Status: TenderStatusActive,
|
|
Source: req.Source,
|
|
SourceFileURL: req.SourceFileURL,
|
|
SourceFileName: req.SourceFileName,
|
|
ProcessingMetadata: ProcessingMetadata{
|
|
ProcessedAt: time.Now().Unix(),
|
|
ProcessingVersion: "1.0",
|
|
},
|
|
}
|
|
|
|
// Generate tender URL if not provided
|
|
if tender.TenderURL == "" && tender.NoticePublicationID != "" {
|
|
tender.TenderURL = generateTenderURL(tender.NoticePublicationID)
|
|
}
|
|
|
|
if err := s.repository.Create(ctx, tender); err != nil {
|
|
s.logger.Error("Failed to create tender", map[string]interface{}{
|
|
"error": err.Error(),
|
|
"notice_publication_id": req.NoticePublicationID,
|
|
})
|
|
return nil, fmt.Errorf("failed to create tender: %w", err)
|
|
}
|
|
|
|
s.logger.Info("Tender created successfully", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
"notice_publication_id": tender.NoticePublicationID,
|
|
})
|
|
|
|
return tender, nil
|
|
}
|
|
|
|
// GetTenderByID retrieves a tender by ID
|
|
func (s *tenderService) GetTenderByID(ctx context.Context, id string) (*Tender, error) {
|
|
s.logger.Info("Retrieving tender by ID", map[string]interface{}{
|
|
"tender_id": id,
|
|
})
|
|
|
|
tender, err := s.repository.GetByID(ctx, id)
|
|
if err != nil {
|
|
s.logger.Error("Failed to retrieve tender", map[string]interface{}{
|
|
"tender_id": id,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to retrieve tender: %w", err)
|
|
}
|
|
|
|
return tender, nil
|
|
}
|
|
|
|
// UpdateTender updates an existing tender
|
|
func (s *tenderService) UpdateTender(ctx context.Context, req UpdateTenderRequest) (*Tender, error) {
|
|
s.logger.Info("Updating tender", map[string]interface{}{
|
|
"tender_id": req.ID,
|
|
})
|
|
|
|
// Get existing tender
|
|
tender, err := s.repository.GetByID(ctx, req.ID)
|
|
if err != nil {
|
|
s.logger.Error("Failed to find tender for update", map[string]interface{}{
|
|
"tender_id": req.ID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to find tender: %w", err)
|
|
}
|
|
|
|
// Update fields if provided
|
|
if req.Title != nil {
|
|
tender.Title = *req.Title
|
|
}
|
|
if req.Description != nil {
|
|
tender.Description = *req.Description
|
|
}
|
|
if req.Status != nil {
|
|
tender.Status = *req.Status
|
|
}
|
|
if req.EstimatedValue != nil {
|
|
tender.EstimatedValue = *req.EstimatedValue
|
|
}
|
|
if req.Currency != nil {
|
|
tender.Currency = *req.Currency
|
|
}
|
|
if req.TenderDeadline != nil {
|
|
tender.TenderDeadline = *req.TenderDeadline
|
|
}
|
|
|
|
if err := s.repository.Update(ctx, tender); err != nil {
|
|
s.logger.Error("Failed to update tender", map[string]interface{}{
|
|
"tender_id": req.ID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to update tender: %w", err)
|
|
}
|
|
|
|
s.logger.Info("Tender updated successfully", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
})
|
|
|
|
return tender, nil
|
|
}
|
|
|
|
// DeleteTender deletes a tender
|
|
func (s *tenderService) DeleteTender(ctx context.Context, id string) error {
|
|
s.logger.Info("Deleting tender", map[string]interface{}{
|
|
"tender_id": id,
|
|
})
|
|
|
|
if err := s.repository.Delete(ctx, id); err != nil {
|
|
s.logger.Error("Failed to delete tender", map[string]interface{}{
|
|
"tender_id": id,
|
|
"error": err.Error(),
|
|
})
|
|
return fmt.Errorf("failed to delete tender: %w", err)
|
|
}
|
|
|
|
s.logger.Info("Tender deleted successfully", map[string]interface{}{
|
|
"tender_id": id,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// ListTenders retrieves tenders with pagination and filtering
|
|
func (s *tenderService) ListTenders(ctx context.Context, req ListTendersRequest) (*ListTendersResponse, error) {
|
|
s.logger.Info("Listing tenders", map[string]interface{}{
|
|
"limit": req.Limit,
|
|
"offset": req.Offset,
|
|
})
|
|
|
|
if req.Limit <= 0 {
|
|
req.Limit = 20
|
|
}
|
|
if req.Limit > 100 {
|
|
req.Limit = 100
|
|
}
|
|
|
|
tenders, total, err := s.repository.List(ctx, req.Criteria, req.Limit, req.Offset)
|
|
if err != nil {
|
|
s.logger.Error("Failed to list tenders", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to list tenders: %w", err)
|
|
}
|
|
|
|
return &ListTendersResponse{
|
|
Tenders: tenders,
|
|
Total: total,
|
|
Limit: req.Limit,
|
|
Offset: req.Offset,
|
|
}, nil
|
|
}
|
|
|
|
// SearchTenders searches tenders based on criteria
|
|
func (s *tenderService) SearchTenders(ctx context.Context, req SearchTendersRequest) (*SearchTendersResponse, error) {
|
|
s.logger.Info("Searching tenders", map[string]interface{}{
|
|
"query": req.Query,
|
|
})
|
|
|
|
tenders, err := s.repository.Search(ctx, req.Criteria)
|
|
if err != nil {
|
|
s.logger.Error("Failed to search tenders", map[string]interface{}{
|
|
"error": err.Error(),
|
|
"query": req.Query,
|
|
})
|
|
return nil, fmt.Errorf("failed to search tenders: %w", err)
|
|
}
|
|
|
|
return &SearchTendersResponse{
|
|
Tenders: tenders,
|
|
Query: req.Query,
|
|
Total: len(tenders),
|
|
}, nil
|
|
}
|
|
|
|
// GetTenderStatistics retrieves tender statistics
|
|
func (s *tenderService) GetTenderStatistics(ctx context.Context) (*TenderStatistics, error) {
|
|
s.logger.Info("Retrieving tender statistics", map[string]interface{}{})
|
|
|
|
stats, err := s.repository.GetStatistics(ctx)
|
|
if err != nil {
|
|
s.logger.Error("Failed to retrieve tender statistics", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to retrieve statistics: %w", err)
|
|
}
|
|
|
|
return stats, nil
|
|
}
|
|
|
|
// GetDashboardData retrieves dashboard data
|
|
func (s *tenderService) GetDashboardData(ctx context.Context) (*DashboardData, error) {
|
|
s.logger.Info("Retrieving dashboard data", map[string]interface{}{})
|
|
|
|
stats, err := s.GetTenderStatistics(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Get recent tenders
|
|
recentTenders, _, err := s.repository.List(ctx, TenderSearchCriteria{}, 10, 0)
|
|
if err != nil {
|
|
s.logger.Error("Failed to retrieve recent tenders", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to retrieve recent tenders: %w", err)
|
|
}
|
|
|
|
// Get active scraping jobs
|
|
activeJobs, err := s.repository.GetActiveScrapingJobs(ctx)
|
|
if err != nil {
|
|
s.logger.Error("Failed to retrieve active scraping jobs", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to retrieve active jobs: %w", err)
|
|
}
|
|
|
|
return &DashboardData{
|
|
Statistics: stats,
|
|
RecentTenders: recentTenders,
|
|
ActiveJobs: activeJobs,
|
|
}, nil
|
|
}
|
|
|
|
// StartScrapingJob starts a new scraping job
|
|
func (s *tenderService) StartScrapingJob(ctx context.Context, req StartScrapingJobRequest) (*ScrapingJob, error) {
|
|
s.logger.Info("Starting scraping job", map[string]interface{}{
|
|
"type": req.Type,
|
|
})
|
|
|
|
job := &ScrapingJob{
|
|
ID: primitive.NewObjectID().Hex(),
|
|
Type: req.Type,
|
|
Status: ScrapingJobStatusPending,
|
|
StartedAt: time.Now().Unix(),
|
|
Progress: JobProgress{},
|
|
Results: JobResults{},
|
|
Config: req.Config,
|
|
}
|
|
|
|
if err := s.repository.CreateScrapingJob(ctx, job); err != nil {
|
|
s.logger.Error("Failed to create scraping job", map[string]interface{}{
|
|
"error": err.Error(),
|
|
"type": req.Type,
|
|
})
|
|
return nil, fmt.Errorf("failed to create scraping job: %w", err)
|
|
}
|
|
|
|
s.logger.Info("Scraping job created successfully", map[string]interface{}{
|
|
"job_id": job.ID,
|
|
"type": job.Type,
|
|
})
|
|
|
|
return job, nil
|
|
}
|
|
|
|
// GetScrapingJobStatus retrieves scraping job status
|
|
func (s *tenderService) GetScrapingJobStatus(ctx context.Context, jobID string) (*ScrapingJob, error) {
|
|
s.logger.Info("Retrieving scraping job status", map[string]interface{}{
|
|
"job_id": jobID,
|
|
})
|
|
|
|
job, err := s.repository.GetScrapingJob(ctx, jobID)
|
|
if err != nil {
|
|
s.logger.Error("Failed to retrieve scraping job", map[string]interface{}{
|
|
"job_id": jobID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to retrieve scraping job: %w", err)
|
|
}
|
|
|
|
return job, nil
|
|
}
|
|
|
|
// ListScrapingJobs lists scraping jobs with pagination
|
|
func (s *tenderService) ListScrapingJobs(ctx context.Context, req ListScrapingJobsRequest) (*ListScrapingJobsResponse, error) {
|
|
s.logger.Info("Listing scraping jobs", map[string]interface{}{
|
|
"limit": req.Limit,
|
|
"offset": req.Offset,
|
|
})
|
|
|
|
if req.Limit <= 0 {
|
|
req.Limit = 20
|
|
}
|
|
if req.Limit > 100 {
|
|
req.Limit = 100
|
|
}
|
|
|
|
jobs, total, err := s.repository.ListScrapingJobs(ctx, req.Limit, req.Offset)
|
|
if err != nil {
|
|
s.logger.Error("Failed to list scraping jobs", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to list scraping jobs: %w", err)
|
|
}
|
|
|
|
return &ListScrapingJobsResponse{
|
|
Jobs: jobs,
|
|
Total: total,
|
|
Limit: req.Limit,
|
|
Offset: req.Offset,
|
|
}, nil
|
|
}
|
|
|
|
// CancelScrapingJob cancels a running scraping job
|
|
func (s *tenderService) CancelScrapingJob(ctx context.Context, jobID string) error {
|
|
s.logger.Info("Cancelling scraping job", map[string]interface{}{
|
|
"job_id": jobID,
|
|
})
|
|
|
|
job, err := s.repository.GetScrapingJob(ctx, jobID)
|
|
if err != nil {
|
|
s.logger.Error("Failed to find scraping job for cancellation", map[string]interface{}{
|
|
"job_id": jobID,
|
|
"error": err.Error(),
|
|
})
|
|
return fmt.Errorf("failed to find scraping job: %w", err)
|
|
}
|
|
|
|
if job.Status != ScrapingJobStatusRunning && job.Status != ScrapingJobStatusPending {
|
|
s.logger.Warn("Cannot cancel scraping job in current status", map[string]interface{}{
|
|
"job_id": jobID,
|
|
"status": job.Status,
|
|
})
|
|
return fmt.Errorf("cannot cancel job in status: %s", job.Status)
|
|
}
|
|
|
|
job.Status = ScrapingJobStatusCancelled
|
|
completedAt := time.Now().Unix()
|
|
job.CompletedAt = &completedAt
|
|
duration := completedAt - job.StartedAt
|
|
job.Duration = &duration
|
|
|
|
if err := s.repository.UpdateScrapingJob(ctx, job); err != nil {
|
|
s.logger.Error("Failed to update cancelled scraping job", map[string]interface{}{
|
|
"job_id": jobID,
|
|
"error": err.Error(),
|
|
})
|
|
return fmt.Errorf("failed to update scraping job: %w", err)
|
|
}
|
|
|
|
s.logger.Info("Scraping job cancelled successfully", map[string]interface{}{
|
|
"job_id": jobID,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// UpdateExpiredTenders updates status of expired tenders
|
|
func (s *tenderService) UpdateExpiredTenders(ctx context.Context) error {
|
|
s.logger.Info("Updating expired tenders", map[string]interface{}{})
|
|
|
|
// Get tenders that should be expired
|
|
expiredTenders, err := s.repository.GetExpiredTenders(ctx, time.Now().Unix())
|
|
if err != nil {
|
|
s.logger.Error("Failed to get expired tenders", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return fmt.Errorf("failed to get expired tenders: %w", err)
|
|
}
|
|
|
|
updatedCount := 0
|
|
for _, tender := range expiredTenders {
|
|
if tender.Status != TenderStatusExpired {
|
|
tender.UpdateStatus(TenderStatusExpired)
|
|
if err := s.repository.Update(ctx, &tender); err != nil {
|
|
s.logger.Error("Failed to update expired tender", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
"error": err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
updatedCount++
|
|
}
|
|
}
|
|
|
|
s.logger.Info("Expired tenders updated", map[string]interface{}{
|
|
"total_expired": len(expiredTenders),
|
|
"updated": updatedCount,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// CleanupOldData removes old data based on retention policy
|
|
func (s *tenderService) CleanupOldData(ctx context.Context, olderThan time.Duration) error {
|
|
s.logger.Info("Cleaning up old data", map[string]interface{}{
|
|
"older_than": olderThan.String(),
|
|
})
|
|
|
|
cutoffTime := time.Now().Add(-olderThan).Unix()
|
|
|
|
// For now, we'll just log what would be cleaned up
|
|
// In production, you might want to actually delete old data
|
|
criteria := TenderSearchCriteria{
|
|
Status: []TenderStatus{TenderStatusExpired, TenderStatusCancelled, TenderStatusAwarded},
|
|
}
|
|
|
|
oldTenders, err := s.repository.Search(ctx, criteria)
|
|
if err != nil {
|
|
s.logger.Error("Failed to find old tenders for cleanup", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return fmt.Errorf("failed to find old tenders: %w", err)
|
|
}
|
|
|
|
cleanupCount := 0
|
|
for _, tender := range oldTenders {
|
|
if tender.CreatedAt < cutoffTime {
|
|
cleanupCount++
|
|
// Here you would delete the tender if implementing actual cleanup
|
|
// s.repository.Delete(ctx, tender.ID)
|
|
}
|
|
}
|
|
|
|
s.logger.Info("Old data cleanup completed", map[string]interface{}{
|
|
"found_old_tenders": cleanupCount,
|
|
"cutoff_time": cutoffTime,
|
|
})
|
|
|
|
return nil
|
|
}
|