mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Tender Management System by Removing Scraping Functionality and Enhancing Tender API

Browse Source 
- Removed all scraping-related functionality, including routes, handlers, services, and repository methods, to streamline the tender management system.
- Updated the tender API to focus solely on tender management, enhancing endpoints for listing, retrieving, and updating tenders.
- Improved Swagger documentation to reflect the removal of scraping endpoints and the addition of new tender-related features.
- Ensured adherence to Clean Architecture principles throughout the refactoring process, maintaining a clear separation of concerns.
This commit is contained in:
n.nakhostin
2025-08-16 16:52:17 +03:30
parent 21b736b55b
commit e986ee4135
11 changed files with 239 additions and 4266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/web/docs/docs.go
+2 -1082
View File
File diff suppressed because it is too large Load Diff
cmd/web/docs/swagger.json
+2 -1082
View File
File diff suppressed because it is too large Load Diff
cmd/web/docs/swagger.yaml
+2 -698
View File
@@ -836,469 +836,43 @@ definitions:
total: total:
type: integer type: integer
type: object type: object
tender.Address: tender.OrganizationResponse:
properties: properties:
city_name:
type: string
country_code:
type: string
country_subentity_code:
type: string
department:
type: string
postal_zone:
type: string
region:
type: string
street_name:
type: string
type: object
tender.CreateTenderRequest:
properties:
additional_classifications:
items:
type: string
type: array
buyer_organization:
$ref: '#/definitions/tender.Organization'
city_name:
type: string
contract_folder_id:
type: string
contract_notice_id:
type: string
country_code:
type: string
currency:
type: string
description:
type: string
document_uri:
type: string
duration:
type: string
duration_unit:
type: string
estimated_value:
type: number
gazette_id:
type: string
issue_date:
description: Unix milliseconds
type: integer
issue_time:
description: Unix milliseconds
type: integer
main_classification:
type: string
notice_language_code:
type: string
notice_publication_id:
type: string
notice_sub_type_code:
type: string
notice_type_code:
type: string
official_languages:
items:
type: string
type: array
organizations:
items:
$ref: '#/definitions/tender.Organization'
type: array
place_of_performance:
type: string
postal_code:
type: string
procedure_code:
type: string
procurement_type_code:
type: string
publication_date:
description: Unix milliseconds
type: integer
region_code:
type: string
review_organization:
$ref: '#/definitions/tender.Organization'
selection_criteria:
items:
$ref: '#/definitions/tender.SelectionCriterion'
type: array
source:
$ref: '#/definitions/tender.TenderSource'
source_file_name:
type: string
source_file_url:
type: string
tender_deadline:
description: Unix milliseconds
type: integer
tender_url:
type: string
title:
type: string
type: object
tender.JobProgress:
properties:
current_step:
type: string
error_count:
type: integer
percentage:
type: number
processed_count:
type: integer
success_count:
type: integer
total_count:
type: integer
type: object
tender.JobResults:
properties:
processing_errors:
items:
type: string
type: array
summary:
type: string
total_created:
type: integer
total_errors:
type: integer
total_processed:
type: integer
total_skipped:
type: integer
total_updated:
type: integer
type: object
tender.Organization:
properties:
address:
$ref: '#/definitions/tender.Address'
company_id:
type: string
contact_email:
type: string
contact_fax:
type: string
contact_name:
type: string
contact_telephone:
type: string
id:
type: string
name: name:
type: string type: string
role:
type: string
website_uri:
type: string
type: object
tender.ProcessingMetadata:
properties:
enrichment_data:
additionalProperties:
type: string
type: object
parsing_errors:
items:
type: string
type: array
processed_at:
description: Unix milliseconds
type: integer
processing_version:
type: string
scraped_at:
description: Unix milliseconds
type: integer
validation_errors:
items:
type: string
type: array
type: object
tender.ScrapingJobResponse:
properties:
completed_at:
type: integer
config:
additionalProperties: true
type: object
created_at:
type: integer
duration:
type: integer
errors:
items:
type: string
type: array
id:
type: string
progress:
$ref: '#/definitions/tender.JobProgress'
results:
$ref: '#/definitions/tender.JobResults'
started_at:
type: integer
status:
$ref: '#/definitions/tender.ScrapingJobStatus'
type:
$ref: '#/definitions/tender.ScrapingJobType'
updated_at:
type: integer
type: object
tender.ScrapingJobStatus:
enum:
- pending
- running
- completed
- failed
- cancelled
type: string
x-enum-varnames:
- ScrapingJobStatusPending
- ScrapingJobStatusRunning
- ScrapingJobStatusCompleted
- ScrapingJobStatusFailed
- ScrapingJobStatusCancelled
tender.ScrapingJobType:
enum:
- ted_scraper
- manual
- api
- bulk_import
type: string
x-enum-varnames:
- ScrapingJobTypeTEDScraper
- ScrapingJobTypeManual
- ScrapingJobTypeAPI
- ScrapingJobTypeBulkImport
tender.SearchTendersRequest:
properties:
criteria:
$ref: '#/definitions/tender.TenderSearchCriteria'
query:
type: string
type: object
tender.SelectionCriterion:
properties:
description:
type: string
language_id:
type: string
type_code:
type: string
type: object
tender.StartScrapingJobRequest:
properties:
config:
additionalProperties: true
type: object
type:
$ref: '#/definitions/tender.ScrapingJobType'
type: object type: object
tender.TenderResponse: tender.TenderResponse:
properties: properties:
additional_classifications:
items:
type: string
type: array
buyer_organization: buyer_organization:
$ref: '#/definitions/tender.Organization' $ref: '#/definitions/tender.OrganizationResponse'
city_name:
type: string
contract_folder_id:
type: string
contract_notice_id:
type: string
country_code: country_code:
type: string type: string
created_at:
type: integer
currency: currency:
type: string type: string
description: description:
type: string type: string
document_uri:
type: string
duration: duration:
type: string type: string
duration_unit: duration_unit:
type: string type: string
estimated_value: estimated_value:
type: number type: number
formatted_estimated_value:
type: string
gazette_id:
type: string
has_errors:
type: boolean
id: id:
type: string type: string
is_active:
type: boolean
is_expired:
type: boolean
issue_date:
type: integer
issue_time:
type: integer
main_classification:
type: string
match_percentage:
description: Match percentage for company (0-100)
type: integer
notice_language_code:
type: string
notice_publication_id: notice_publication_id:
type: string type: string
notice_sub_type_code:
type: string
notice_type_code:
type: string
official_languages:
items:
type: string
type: array
organizations:
items:
$ref: '#/definitions/tender.Organization'
type: array
place_of_performance:
type: string
postal_code:
type: string
procedure_code: procedure_code:
type: string type: string
processing_metadata:
$ref: '#/definitions/tender.ProcessingMetadata'
procurement_type_code: procurement_type_code:
type: string type: string
publication_date: publication_date:
type: integer type: integer
region_code:
type: string
review_organization:
$ref: '#/definitions/tender.Organization'
selection_criteria:
items:
$ref: '#/definitions/tender.SelectionCriterion'
type: array
source:
$ref: '#/definitions/tender.TenderSource'
source_file_name:
type: string
source_file_url:
type: string
status: status:
$ref: '#/definitions/tender.TenderStatus' $ref: '#/definitions/tender.TenderStatus'
tender_deadline: tender_deadline:
type: integer type: integer
tender_url:
type: string
title: title:
type: string type: string
updated_at:
type: integer
type: object
tender.TenderSearchCriteria:
properties:
buyer_organization_id:
type: string
classifications:
items:
type: string
type: array
country_codes:
items:
type: string
type: array
currency:
type: string
deadline_from:
description: Unix milliseconds
type: integer
deadline_to:
description: Unix milliseconds
type: integer
languages:
items:
type: string
type: array
max_estimated_value:
type: number
min_estimated_value:
type: number
notice_type:
type: string
procurement_type:
type: string
publication_date_from:
description: Unix milliseconds
type: integer
publication_date_to:
description: Unix milliseconds
type: integer
query:
type: string
region_codes:
items:
type: string
type: array
source:
items:
$ref: '#/definitions/tender.TenderSource'
type: array
status:
items:
$ref: '#/definitions/tender.TenderStatus'
type: array
type: object
tender.TenderSource:
enum:
- ted_scraper
- manual
- api
- bulk_import
type: string
x-enum-varnames:
- TenderSourceTEDScraper
- TenderSourceManual
- TenderSourceAPI
- TenderSourceBulkImport
tender.TenderStatistics:
properties:
active_tenders:
type: integer
average_estimated_value:
type: number
expired_tenders:
type: integer
last_updated:
description: Unix milliseconds
type: integer
tenders_by_classification:
additionalProperties:
format: int64
type: integer
type: object
tenders_by_country:
additionalProperties:
format: int64
type: integer
type: object
tenders_by_type:
additionalProperties:
format: int64
type: integer
type: object
total_estimated_value:
type: number
total_tenders:
type: integer
type: object type: object
tender.TenderStatus: tender.TenderStatus:
enum: enum:
@@ -1540,145 +1114,6 @@ info:
title: Tender Management API title: Tender Management API
version: 2.0.0 version: 2.0.0
paths: paths:
/admin/scraping/jobs:
get:
description: Retrieve scraping jobs with pagination
parameters:
- description: 'Number of items per page (default: 20, max: 100)'
in: query
name: limit
type: integer
- description: 'Number of items to skip (default: 0)'
in: query
name: offset
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
allOf:
- $ref: '#/definitions/response.APIResponse'
- properties:
data:
additionalProperties: true
type: object
type: object
"400":
description: Bad Request
schema:
$ref: '#/definitions/response.APIResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/response.APIResponse'
security:
- BearerAuth: []
summary: List scraping jobs
tags:
- Admin-Scraping
post:
consumes:
- application/json
description: Start a new TED XML scraping job
parameters:
- description: Scraping job configuration
in: body
name: job
required: true
schema:
$ref: '#/definitions/tender.StartScrapingJobRequest'
produces:
- application/json
responses:
"201":
description: Created
schema:
allOf:
- $ref: '#/definitions/response.APIResponse'
- properties:
data:
$ref: '#/definitions/tender.ScrapingJobResponse'
type: object
"400":
description: Bad Request
schema:
$ref: '#/definitions/response.APIResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/response.APIResponse'
security:
- BearerAuth: []
summary: Start scraping job
tags:
- Admin-Scraping
/admin/scraping/jobs/{id}:
get:
description: Retrieve a specific scraping job by its ID
parameters:
- description: Scraping job ID
in: path
name: id
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
allOf:
- $ref: '#/definitions/response.APIResponse'
- properties:
data:
$ref: '#/definitions/tender.ScrapingJobResponse'
type: object
"404":
description: Not Found
schema:
$ref: '#/definitions/response.APIResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/response.APIResponse'
security:
- BearerAuth: []
summary: Get scraping job
tags:
- Admin-Scraping
/admin/scraping/jobs/{id}/cancel:
post:
description: Cancel a running or pending scraping job
parameters:
- description: Scraping job ID
in: path
name: id
required: true
type: string
responses:
"200":
description: OK
schema:
$ref: '#/definitions/response.APIResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/response.APIResponse'
"404":
description: Not Found
schema:
$ref: '#/definitions/response.APIResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/response.APIResponse'
security:
- BearerAuth: []
summary: Cancel scraping job
tags:
- Admin-Scraping
/admin/tenders: /admin/tenders:
get: get:
description: Retrieve tenders with pagination, filtering, and search capabilities description: Retrieve tenders with pagination, filtering, and search capabilities
@@ -1767,46 +1202,6 @@ paths:
summary: List tenders summary: List tenders
tags: tags:
- Admin-Tenders - Admin-Tenders
post:
consumes:
- application/json
description: Create a new tender with the provided information
parameters:
- description: Tender information
in: body
name: tender
required: true
schema:
$ref: '#/definitions/tender.CreateTenderRequest'
produces:
- application/json
responses:
"201":
description: Created
schema:
allOf:
- $ref: '#/definitions/response.APIResponse'
- properties:
data:
$ref: '#/definitions/tender.TenderResponse'
type: object
"400":
description: Bad Request
schema:
$ref: '#/definitions/response.APIResponse'
"409":
description: Conflict
schema:
$ref: '#/definitions/response.APIResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/response.APIResponse'
security:
- BearerAuth: []
summary: Create a new tender
tags:
- Admin-Tenders
/admin/tenders/{id}: /admin/tenders/{id}:
delete: delete:
description: Delete a tender by its ID description: Delete a tender by its ID
@@ -1912,94 +1307,6 @@ paths:
summary: Update tender summary: Update tender
tags: tags:
- Admin-Tenders - Admin-Tenders
/admin/tenders/dashboard:
get:
description: Retrieve dashboard data including statistics, recent tenders, and
active jobs
produces:
- application/json
responses:
"200":
description: OK
schema:
allOf:
- $ref: '#/definitions/response.APIResponse'
- properties:
data:
additionalProperties: true
type: object
type: object
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/response.APIResponse'
security:
- BearerAuth: []
summary: Get dashboard data
tags:
- Admin-Tenders
/admin/tenders/search:
post:
consumes:
- application/json
description: Search tenders using full-text search and filtering
parameters:
- description: Search criteria
in: body
name: search
required: true
schema:
$ref: '#/definitions/tender.SearchTendersRequest'
produces:
- application/json
responses:
"200":
description: OK
schema:
allOf:
- $ref: '#/definitions/response.APIResponse'
- properties:
data:
additionalProperties: true
type: object
type: object
"400":
description: Bad Request
schema:
$ref: '#/definitions/response.APIResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/response.APIResponse'
security:
- BearerAuth: []
summary: Search tenders
tags:
- Admin-Tenders
/admin/tenders/statistics:
get:
description: Retrieve comprehensive statistics about tenders
produces:
- application/json
responses:
"200":
description: OK
schema:
allOf:
- $ref: '#/definitions/response.APIResponse'
- properties:
data:
$ref: '#/definitions/tender.TenderStatistics'
type: object
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/response.APIResponse'
security:
- BearerAuth: []
summary: Get tender statistics
tags:
- Admin-Tenders
/admin/v1/companies: /admin/v1/companies:
get: get:
consumes: consumes:
@@ -5251,9 +4558,6 @@ tags:
- description: Administrative tender management operations for web panel including - description: Administrative tender management operations for web panel including
CRUD operations, search, statistics, and comprehensive filtering with pagination CRUD operations, search, statistics, and comprehensive filtering with pagination
name: Admin-Tenders name: Admin-Tenders
- description: Administrative scraping job management operations for TED XML data
collection including job control, monitoring, and status tracking
name: Admin-Scraping
- description: Administrative feedback management operations for web panel including - description: Administrative feedback management operations for web panel including
CRUD operations, search, statistics, and comprehensive filtering with pagination CRUD operations, search, statistics, and comprehensive filtering with pagination
name: Admin-Feedback name: Admin-Feedback
cmd/web/main.go
+3 -6
View File
@@ -38,9 +38,6 @@ package main
// @tag.name Admin-Tenders // @tag.name Admin-Tenders
// @tag.description Administrative tender management operations for web panel including CRUD operations, search, statistics, and comprehensive filtering with pagination // @tag.description Administrative tender management operations for web panel including CRUD operations, search, statistics, and comprehensive filtering with pagination
// @tag.name Admin-Scraping
// @tag.description Administrative scraping job management operations for TED XML data collection including job control, monitoring, and status tracking
// @tag.name Admin-Feedback // @tag.name Admin-Feedback
// @tag.description Administrative feedback management operations for web panel including CRUD operations, search, statistics, and comprehensive filtering with pagination // @tag.description Administrative feedback management operations for web panel including CRUD operations, search, statistics, and comprehensive filtering with pagination
@@ -108,7 +105,7 @@ func main() {
feedbackRepo := feedback.NewFeedbackRepository(mongoManager, logger) feedbackRepo := feedback.NewFeedbackRepository(mongoManager, logger)
logger.Info("Repositories initialized successfully", map[string]interface{}{ logger.Info("Repositories initialized successfully", map[string]interface{}{
"repositories": []string{"customer", "user", "company", "tender"}, "repositories": []string{"customer", "user", "company", "tender", "feedback"},
}) })
// Initialize validation service // Initialize validation service
@@ -122,7 +119,7 @@ func main() {
feedbackService := feedback.NewFeedbackService(feedbackRepo, logger) feedbackService := feedback.NewFeedbackService(feedbackRepo, logger)
logger.Info("Services initialized successfully", map[string]interface{}{ logger.Info("Services initialized successfully", map[string]interface{}{
"services": []string{"customer", "user", "company", "tender"}, "services": []string{"customer", "user", "company", "tender", "feedback"},
}) })
// Initialize handlers with services // Initialize handlers with services
@@ -133,7 +130,7 @@ func main() {
feedbackHandler := feedback.NewFeedbackHandler(feedbackService, userHandler, customerHandler, logger) feedbackHandler := feedback.NewFeedbackHandler(feedbackService, userHandler, customerHandler, logger)
logger.Info("Handlers initialized successfully", map[string]interface{}{ logger.Info("Handlers initialized successfully", map[string]interface{}{
"handlers": []string{"customer", "user", "company", "tender"}, "handlers": []string{"customer", "user", "company", "tender", "feedback"},
}) })
// Initialize HTTP server // Initialize HTTP server
cmd/web/router/routes.go
-14
View File
@@ -93,24 +93,10 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
tendersGP := adminV1.Group("/tenders") tendersGP := adminV1.Group("/tenders")
{ {
tendersGP.Use(userHandler.AuthMiddleware()) tendersGP.Use(userHandler.AuthMiddleware())
tendersGP.POST("", tenderHandler.CreateTender)
tendersGP.GET("", tenderHandler.ListTenders) tendersGP.GET("", tenderHandler.ListTenders)
tendersGP.GET("/:id", tenderHandler.GetTender) tendersGP.GET("/:id", tenderHandler.GetTender)
tendersGP.PUT("/:id", tenderHandler.UpdateTender) tendersGP.PUT("/:id", tenderHandler.UpdateTender)
tendersGP.DELETE("/:id", tenderHandler.DeleteTender) tendersGP.DELETE("/:id", tenderHandler.DeleteTender)
tendersGP.POST("/search", tenderHandler.SearchTenders)
tendersGP.GET("/statistics", tenderHandler.GetTenderStatistics)
tendersGP.GET("/dashboard", tenderHandler.GetDashboard)
}
// Admin Scraping Routes
scrapingGP := adminV1.Group("/scraping")
{
scrapingGP.Use(userHandler.AuthMiddleware())
scrapingGP.POST("/jobs", tenderHandler.StartScrapingJob)
scrapingGP.GET("/jobs", tenderHandler.ListScrapingJobs)
scrapingGP.GET("/jobs/:id", tenderHandler.GetScrapingJob)
scrapingGP.POST("/jobs/:id/cancel", tenderHandler.CancelScrapingJob)
} }
// Admin Feedback Routes // Admin Feedback Routes
internal/scraping/scraping.go
+153
View File
@@ -0,0 +1,153 @@
package scraping
import "time"
// ScrapingJobType represents the type of scraping job
type ScrapingJobType string
const (
ScrapingJobTypeTEDScraper ScrapingJobType = "ted_scraper"
ScrapingJobTypeManual ScrapingJobType = "manual"
ScrapingJobTypeAPI ScrapingJobType = "api"
ScrapingJobTypeBulkImport ScrapingJobType = "bulk_import"
)
// ScrapingJobStatus represents the status of a scraping job
type ScrapingJobStatus string
const (
ScrapingJobStatusPending ScrapingJobStatus = "pending"
ScrapingJobStatusRunning ScrapingJobStatus = "running"
ScrapingJobStatusCompleted ScrapingJobStatus = "completed"
ScrapingJobStatusFailed ScrapingJobStatus = "failed"
ScrapingJobStatusCancelled ScrapingJobStatus = "cancelled"
)
// JobProgress represents the progress of a scraping job
type JobProgress struct {
ProcessedCount int `bson:"processed_count" json:"processed_count"`
TotalCount int `bson:"total_count" json:"total_count"`
SuccessCount int `bson:"success_count" json:"success_count"`
ErrorCount int `bson:"error_count" json:"error_count"`
Percentage float64 `bson:"percentage" json:"percentage"`
CurrentStep string `bson:"current_step" json:"current_step"`
}
// JobResults represents the results of a scraping job
type JobResults struct {
TotalProcessed int `bson:"total_processed" json:"total_processed"`
TotalCreated int `bson:"total_created" json:"total_created"`
TotalUpdated int `bson:"total_updated" json:"total_updated"`
TotalSkipped int `bson:"total_skipped" json:"total_skipped"`
TotalErrors int `bson:"total_errors" json:"total_errors"`
ProcessingErrors []string `bson:"processing_errors,omitempty" json:"processing_errors,omitempty"`
Summary string `bson:"summary,omitempty" json:"summary,omitempty"`
}
// ScrapingJob represents a scraping job entity
type ScrapingJob struct {
ID string `bson:"_id,omitempty" json:"id"`
Type ScrapingJobType `bson:"type" json:"type"`
Status ScrapingJobStatus `bson:"status" json:"status"`
StartedAt int64 `bson:"started_at" json:"started_at"` // Unix seconds
CompletedAt *int64 `bson:"completed_at,omitempty" json:"completed_at,omitempty"` // Unix seconds
Duration *int64 `bson:"duration,omitempty" json:"duration,omitempty"` // Duration in seconds
Progress JobProgress `bson:"progress" json:"progress"`
Results JobResults `bson:"results" json:"results"`
Errors []string `bson:"errors,omitempty" json:"errors,omitempty"`
Config map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"`
CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds
UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds
}
// ScrapingState represents the state of the scraping process
type ScrapingState struct {
ID string `bson:"_id,omitempty" json:"id"`
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"` // Unix seconds
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"` // Unix seconds
IsRunning bool `bson:"is_running" json:"is_running"`
CurrentJobID string `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds
UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds
}
// GetJobID returns the scraping job ID
func (j *ScrapingJob) GetJobID() string {
return j.ID
}
// IsActive returns true if the job is actively running
func (j *ScrapingJob) IsActive() bool {
return j.Status == ScrapingJobStatusRunning || j.Status == ScrapingJobStatusPending
}
// IsCompleted returns true if the job has completed (successfully or with failure)
func (j *ScrapingJob) IsCompleted() bool {
return j.Status == ScrapingJobStatusCompleted || j.Status == ScrapingJobStatusFailed || j.Status == ScrapingJobStatusCancelled
}
// GetProgressPercentage calculates and returns the progress percentage
func (j *ScrapingJob) GetProgressPercentage() float64 {
if j.Progress.TotalCount == 0 {
return 0
}
return float64(j.Progress.ProcessedCount) / float64(j.Progress.TotalCount) * 100
}
// UpdateProgress updates the job progress
func (j *ScrapingJob) UpdateProgress(processed, total, success, errors int, step string) {
j.Progress.ProcessedCount = processed
j.Progress.TotalCount = total
j.Progress.SuccessCount = success
j.Progress.ErrorCount = errors
j.Progress.CurrentStep = step
j.Progress.Percentage = j.GetProgressPercentage()
j.UpdatedAt = time.Now().Unix()
}
// AddError adds an error to the job
func (j *ScrapingJob) AddError(error string) {
if j.Errors == nil {
j.Errors = make([]string, 0)
}
j.Errors = append(j.Errors, error)
j.UpdatedAt = time.Now().Unix()
}
// MarkCompleted marks the job as completed
func (j *ScrapingJob) MarkCompleted(status ScrapingJobStatus, results JobResults) {
j.Status = status
j.Results = results
completedAt := time.Now().Unix()
j.CompletedAt = &completedAt
if j.StartedAt > 0 {
duration := completedAt - j.StartedAt
j.Duration = &duration
}
j.UpdatedAt = completedAt
}
// IsStateRunning returns true if the scraping state indicates a job is running
func (s *ScrapingState) IsStateRunning() bool {
return s.IsRunning
}
// MarkAsRunning marks the state as running with a job ID
func (s *ScrapingState) MarkAsRunning(jobID string) {
s.IsRunning = true
s.CurrentJobID = jobID
s.LastRunAt = time.Now().Unix()
s.UpdatedAt = time.Now().Unix()
}
// MarkAsCompleted marks the state as completed
func (s *ScrapingState) MarkAsCompleted(year, number int) {
s.IsRunning = false
s.CurrentJobID = ""
s.LastProcessedYear = year
s.LastProcessedNumber = number
s.NextRunAt = time.Now().Add(4 * time.Hour).Unix() // Schedule next run in 4 hours
s.UpdatedAt = time.Now().Unix()
}
internal/tender/entity.go
+4 -228
View File
@@ -128,10 +128,10 @@ type TenderSearchCriteria struct {
MinEstimatedValue *float64 `json:"min_estimated_value,omitempty"` MinEstimatedValue *float64 `json:"min_estimated_value,omitempty"`
MaxEstimatedValue *float64 `json:"max_estimated_value,omitempty"` MaxEstimatedValue *float64 `json:"max_estimated_value,omitempty"`
Currency string `json:"currency,omitempty"` Currency string `json:"currency,omitempty"`
DeadlineFrom *int64 `json:"deadline_from,omitempty"` // Unix milliseconds DeadlineFrom *int64 `json:"deadline_from,omitempty"`
DeadlineTo *int64 `json:"deadline_to,omitempty"` // Unix milliseconds DeadlineTo *int64 `json:"deadline_to,omitempty"`
PublicationDateFrom *int64 `json:"publication_date_from,omitempty"` // Unix milliseconds PublicationDateFrom *int64 `json:"publication_date_from,omitempty"`
PublicationDateTo *int64 `json:"publication_date_to,omitempty"` // Unix milliseconds PublicationDateTo *int64 `json:"publication_date_to,omitempty"`
Status []TenderStatus `json:"status,omitempty"` Status []TenderStatus `json:"status,omitempty"`
Source []TenderSource `json:"source,omitempty"` Source []TenderSource `json:"source,omitempty"`
BuyerOrganizationID string `json:"buyer_organization_id,omitempty"` BuyerOrganizationID string `json:"buyer_organization_id,omitempty"`
@@ -236,230 +236,6 @@ func (t *Tender) UpdateStatus(status TenderStatus) {
t.UpdatedAt = time.Now().UnixMilli() t.UpdatedAt = time.Now().UnixMilli()
} }
// AddProcessingError adds a processing error to metadata
func (t *Tender) AddProcessingError(error string) {
if t.ProcessingMetadata.ParsingErrors == nil {
t.ProcessingMetadata.ParsingErrors = make([]string, 0)
}
t.ProcessingMetadata.ParsingErrors = append(t.ProcessingMetadata.ParsingErrors, error)
t.UpdatedAt = time.Now().UnixMilli()
}
// AddValidationError adds a validation error to metadata
func (t *Tender) AddValidationError(error string) {
if t.ProcessingMetadata.ValidationErrors == nil {
t.ProcessingMetadata.ValidationErrors = make([]string, 0)
}
t.ProcessingMetadata.ValidationErrors = append(t.ProcessingMetadata.ValidationErrors, error)
t.UpdatedAt = time.Now().UnixMilli()
}
// HasErrors returns true if the tender has processing or validation errors
func (t *Tender) HasErrors() bool {
return len(t.ProcessingMetadata.ParsingErrors) > 0 || len(t.ProcessingMetadata.ValidationErrors) > 0
}
// GetAllErrors returns all processing and validation errors
func (t *Tender) GetAllErrors() []string {
var allErrors []string
allErrors = append(allErrors, t.ProcessingMetadata.ParsingErrors...)
allErrors = append(allErrors, t.ProcessingMetadata.ValidationErrors...)
return allErrors
}
// ScrapingJobType represents the type of scraping job
type ScrapingJobType string
const (
ScrapingJobTypeTEDScraper ScrapingJobType = "ted_scraper"
ScrapingJobTypeManual ScrapingJobType = "manual"
ScrapingJobTypeAPI ScrapingJobType = "api"
ScrapingJobTypeBulkImport ScrapingJobType = "bulk_import"
)
// ScrapingJobStatus represents the status of a scraping job
type ScrapingJobStatus string
const (
ScrapingJobStatusPending ScrapingJobStatus = "pending"
ScrapingJobStatusRunning ScrapingJobStatus = "running"
ScrapingJobStatusCompleted ScrapingJobStatus = "completed"
ScrapingJobStatusFailed ScrapingJobStatus = "failed"
ScrapingJobStatusCancelled ScrapingJobStatus = "cancelled"
)
// JobProgress represents the progress of a scraping job
type JobProgress struct {
ProcessedCount int `bson:"processed_count" json:"processed_count"`
TotalCount int `bson:"total_count" json:"total_count"`
SuccessCount int `bson:"success_count" json:"success_count"`
ErrorCount int `bson:"error_count" json:"error_count"`
Percentage float64 `bson:"percentage" json:"percentage"`
CurrentStep string `bson:"current_step" json:"current_step"`
}
// JobResults represents the results of a scraping job
type JobResults struct {
TotalProcessed int `bson:"total_processed" json:"total_processed"`
TotalCreated int `bson:"total_created" json:"total_created"`
TotalUpdated int `bson:"total_updated" json:"total_updated"`
TotalSkipped int `bson:"total_skipped" json:"total_skipped"`
TotalErrors int `bson:"total_errors" json:"total_errors"`
ProcessingErrors []string `bson:"processing_errors,omitempty" json:"processing_errors,omitempty"`
Summary string `bson:"summary,omitempty" json:"summary,omitempty"`
}
// ScrapingJob represents a scraping job entity
type ScrapingJob struct {
ID string `bson:"_id,omitempty" json:"id"`
Type ScrapingJobType `bson:"type" json:"type"`
Status ScrapingJobStatus `bson:"status" json:"status"`
StartedAt int64 `bson:"started_at" json:"started_at"` // Unix seconds
CompletedAt *int64 `bson:"completed_at,omitempty" json:"completed_at,omitempty"` // Unix seconds
Duration *int64 `bson:"duration,omitempty" json:"duration,omitempty"` // Duration in seconds
Progress JobProgress `bson:"progress" json:"progress"`
Results JobResults `bson:"results" json:"results"`
Errors []string `bson:"errors,omitempty" json:"errors,omitempty"`
Config map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"`
CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds
UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds
}
// ScrapingState represents the state of the scraping process
type ScrapingState struct {
ID string `bson:"_id,omitempty" json:"id"`
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"` // Unix seconds
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"` // Unix seconds
IsRunning bool `bson:"is_running" json:"is_running"`
CurrentJobID string `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds
UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds
}
// GetJobID returns the scraping job ID
func (j *ScrapingJob) GetJobID() string {
return j.ID
}
// IsActive returns true if the job is actively running
func (j *ScrapingJob) IsActive() bool {
return j.Status == ScrapingJobStatusRunning || j.Status == ScrapingJobStatusPending
}
// IsCompleted returns true if the job has completed (successfully or with failure)
func (j *ScrapingJob) IsCompleted() bool {
return j.Status == ScrapingJobStatusCompleted || j.Status == ScrapingJobStatusFailed || j.Status == ScrapingJobStatusCancelled
}
// GetProgressPercentage calculates and returns the progress percentage
func (j *ScrapingJob) GetProgressPercentage() float64 {
if j.Progress.TotalCount == 0 {
return 0
}
return float64(j.Progress.ProcessedCount) / float64(j.Progress.TotalCount) * 100
}
// UpdateProgress updates the job progress
func (j *ScrapingJob) UpdateProgress(processed, total, success, errors int, step string) {
j.Progress.ProcessedCount = processed
j.Progress.TotalCount = total
j.Progress.SuccessCount = success
j.Progress.ErrorCount = errors
j.Progress.CurrentStep = step
j.Progress.Percentage = j.GetProgressPercentage()
j.UpdatedAt = time.Now().Unix()
}
// AddError adds an error to the job
func (j *ScrapingJob) AddError(error string) {
if j.Errors == nil {
j.Errors = make([]string, 0)
}
j.Errors = append(j.Errors, error)
j.UpdatedAt = time.Now().Unix()
}
// MarkCompleted marks the job as completed
func (j *ScrapingJob) MarkCompleted(status ScrapingJobStatus, results JobResults) {
j.Status = status
j.Results = results
completedAt := time.Now().Unix()
j.CompletedAt = &completedAt
if j.StartedAt > 0 {
duration := completedAt - j.StartedAt
j.Duration = &duration
}
j.UpdatedAt = completedAt
}
// IsStateRunning returns true if the scraping state indicates a job is running
func (s *ScrapingState) IsStateRunning() bool {
return s.IsRunning
}
// MarkAsRunning marks the state as running with a job ID
func (s *ScrapingState) MarkAsRunning(jobID string) {
s.IsRunning = true
s.CurrentJobID = jobID
s.LastRunAt = time.Now().Unix()
s.UpdatedAt = time.Now().Unix()
}
// MarkAsCompleted marks the state as completed
func (s *ScrapingState) MarkAsCompleted(year, number int) {
s.IsRunning = false
s.CurrentJobID = ""
s.LastProcessedYear = year
s.LastProcessedNumber = number
s.NextRunAt = time.Now().Add(4 * time.Hour).Unix() // Schedule next run in 4 hours
s.UpdatedAt = time.Now().Unix()
}
// CalculateMatchPercentage calculates the match percentage between tender classifications and company CPV codes
func (t *Tender) CalculateMatchPercentage(companyCPVCodes []string) int {
if len(companyCPVCodes) == 0 {
return 0
}
mainMatch := false
additionalMatch := false
// Check if main classification matches any company CPV codes
if t.MainClassification != "" {
for _, companyCode := range companyCPVCodes {
if companyCode == t.MainClassification {
mainMatch = true
break
}
}
}
// Check if any additional classification matches any company CPV codes
if len(t.AdditionalClassifications) > 0 {
for _, additionalCode := range t.AdditionalClassifications {
for _, companyCode := range companyCPVCodes {
if companyCode == additionalCode {
additionalMatch = true
break
}
}
if additionalMatch {
break
}
}
}
// Calculate percentage based on matches
if mainMatch && additionalMatch {
return 100
} else if mainMatch || additionalMatch {
return 50
}
return 0
}
// GetDaysUntilDeadline returns the number of days until the tender deadline // GetDaysUntilDeadline returns the number of days until the tender deadline
func (t *Tender) GetDaysUntilDeadline() int { func (t *Tender) GetDaysUntilDeadline() int {
if t.TenderDeadline == 0 { if t.TenderDeadline == 0 {
internal/tender/form.go
+35 -262
View File
@@ -1,44 +1,6 @@
package tender package tender
// CreateTenderRequest represents a request to create a new tender import "tm/pkg/response"
type CreateTenderRequest struct {
ContractNoticeID string `json:"contract_notice_id" valid:"required~Contract notice ID is required"`
NoticePublicationID string `json:"notice_publication_id" valid:"required~Notice publication ID is required"`
ContractFolderID string `json:"contract_folder_id" valid:"required~Contract folder ID is required"`
NoticeTypeCode string `json:"notice_type_code" valid:"required~Notice type code is required"`
NoticeSubTypeCode string `json:"notice_sub_type_code,omitempty"`
NoticeLanguageCode string `json:"notice_language_code,omitempty" valid:"stringlength(3|3)~Language code must be 3 characters"`
IssueDate int64 `json:"issue_date" valid:"required~Issue date is required"` // Unix milliseconds
IssueTime int64 `json:"issue_time,omitempty"` // Unix milliseconds
PublicationDate int64 `json:"publication_date,omitempty"` // Unix milliseconds
GazetteID string `json:"gazette_id,omitempty"`
Title string `json:"title" valid:"required~Title is required,stringlength(1|500)~Title must be between 1 and 500 characters"`
Description string `json:"description,omitempty" valid:"stringlength(0|5000)~Description must not exceed 5000 characters"`
ProcurementTypeCode string `json:"procurement_type_code,omitempty"`
ProcedureCode string `json:"procedure_code,omitempty"`
MainClassification string `json:"main_classification,omitempty"`
AdditionalClassifications []string `json:"additional_classifications,omitempty"`
EstimatedValue float64 `json:"estimated_value,omitempty" valid:"range(0|999999999999)~Estimated value must be between 0 and 999999999999"`
Currency string `json:"currency,omitempty" valid:"stringlength(3|3)~Currency must be 3 characters"`
Duration string `json:"duration,omitempty"`
DurationUnit string `json:"duration_unit,omitempty"`
TenderDeadline int64 `json:"tender_deadline,omitempty"` // Unix milliseconds
PlaceOfPerformance string `json:"place_of_performance,omitempty"`
CountryCode string `json:"country_code,omitempty" valid:"stringlength(2|2)~Country code must be 2 characters"`
RegionCode string `json:"region_code,omitempty"`
CityName string `json:"city_name,omitempty"`
PostalCode string `json:"postal_code,omitempty"`
DocumentURI string `json:"document_uri,omitempty" valid:"url~Document URI must be a valid URL"`
TenderURL string `json:"tender_url,omitempty" valid:"url~Tender URL must be a valid URL"`
BuyerOrganization *Organization `json:"buyer_organization,omitempty"`
ReviewOrganization *Organization `json:"review_organization,omitempty"`
Organizations []Organization `json:"organizations,omitempty"`
SelectionCriteria []SelectionCriterion `json:"selection_criteria,omitempty"`
OfficialLanguages []string `json:"official_languages,omitempty"`
Source TenderSource `json:"source" valid:"required~Source is required"`
SourceFileURL string `json:"source_file_url,omitempty" valid:"url~Source file URL must be a valid URL"`
SourceFileName string `json:"source_file_name,omitempty"`
}
// UpdateTenderRequest represents a request to update an existing tender // UpdateTenderRequest represents a request to update an existing tender
type UpdateTenderRequest struct { type UpdateTenderRequest struct {
@@ -61,16 +23,8 @@ type ListTendersRequest struct {
// ListTendersResponse represents the response for listing tenders // ListTendersResponse represents the response for listing tenders
type ListTendersResponse struct { type ListTendersResponse struct {
Tenders []TenderResponse `json:"tenders"` Tenders []TenderResponse `json:"tenders"`
Total int64 `json:"total"` Metadata *response.Meta `json:"metadata"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// SearchTendersRequest represents a request to search tenders
type SearchTendersRequest struct {
Query string `json:"query" valid:"required~Search query is required,stringlength(1|200)~Query must be between 1 and 200 characters"`
Criteria TenderSearchCriteria `json:"criteria"`
} }
// SearchTendersResponse represents the response for searching tenders // SearchTendersResponse represents the response for searching tenders
@@ -80,229 +34,48 @@ type SearchTendersResponse struct {
Total int `json:"total"` Total int `json:"total"`
} }
// StartScrapingJobRequest represents a request to start a scraping job
type StartScrapingJobRequest struct {
Type ScrapingJobType `json:"type" valid:"required~Job type is required"`
Config map[string]interface{} `json:"config,omitempty"`
}
// ListScrapingJobsRequest represents a request to list scraping jobs
type ListScrapingJobsRequest struct {
Limit int `json:"limit" valid:"range(1|100)~Limit must be between 1 and 100"`
Offset int `json:"offset" valid:"range(0|999999)~Offset must be non-negative"`
}
// ListScrapingJobsResponse represents the response for listing scraping jobs
type ListScrapingJobsResponse struct {
Jobs []ScrapingJob `json:"jobs"`
Total int64 `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// BatchProcessingResult represents the result of batch processing
type BatchProcessingResult struct {
Total int `json:"total"`
Processed int `json:"processed"`
Errors []string `json:"errors"`
Tenders []*Tender `json:"tenders"`
SuccessRate float64 `json:"success_rate"`
}
// DashboardData represents dashboard data
type DashboardData struct {
Statistics *TenderStatistics `json:"statistics"`
RecentTenders []Tender `json:"recent_tenders"`
ActiveJobs []ScrapingJob `json:"active_jobs"`
}
// TenderResponse represents a tender in API responses // TenderResponse represents a tender in API responses
type TenderResponse struct { type TenderResponse struct {
ID string `json:"id"` ID string `json:"id"`
ContractNoticeID string `json:"contract_notice_id"` NoticePublicationID string `json:"notice_publication_id"`
NoticePublicationID string `json:"notice_publication_id"` PublicationDate int64 `json:"publication_date"`
ContractFolderID string `json:"contract_folder_id"` Title string `json:"title"`
NoticeTypeCode string `json:"notice_type_code"` Description string `json:"description"`
NoticeSubTypeCode string `json:"notice_sub_type_code"` ProcurementTypeCode string `json:"procurement_type_code"`
NoticeLanguageCode string `json:"notice_language_code"` ProcedureCode string `json:"procedure_code"`
IssueDate int64 `json:"issue_date"` EstimatedValue float64 `json:"estimated_value"`
IssueTime int64 `json:"issue_time"` Currency string `json:"currency"`
PublicationDate int64 `json:"publication_date"` Duration string `json:"duration"`
GazetteID string `json:"gazette_id"` DurationUnit string `json:"duration_unit"`
Title string `json:"title"` TenderDeadline int64 `json:"tender_deadline"`
Description string `json:"description"` CountryCode string `json:"country_code"`
ProcurementTypeCode string `json:"procurement_type_code"` BuyerOrganization *OrganizationResponse `json:"buyer_organization"`
ProcedureCode string `json:"procedure_code"` Status TenderStatus `json:"status"`
MainClassification string `json:"main_classification"`
AdditionalClassifications []string `json:"additional_classifications"`
EstimatedValue float64 `json:"estimated_value"`
Currency string `json:"currency"`
FormattedEstimatedValue string `json:"formatted_estimated_value"`
Duration string `json:"duration"`
DurationUnit string `json:"duration_unit"`
TenderDeadline int64 `json:"tender_deadline"`
PlaceOfPerformance string `json:"place_of_performance"`
CountryCode string `json:"country_code"`
RegionCode string `json:"region_code"`
CityName string `json:"city_name"`
PostalCode string `json:"postal_code"`
DocumentURI string `json:"document_uri"`
TenderURL string `json:"tender_url"`
BuyerOrganization *Organization `json:"buyer_organization"`
ReviewOrganization *Organization `json:"review_organization,omitempty"`
Organizations []Organization `json:"organizations"`
SelectionCriteria []SelectionCriterion `json:"selection_criteria"`
OfficialLanguages []string `json:"official_languages"`
Status TenderStatus `json:"status"`
Source TenderSource `json:"source"`
SourceFileURL string `json:"source_file_url"`
SourceFileName string `json:"source_file_name"`
ProcessingMetadata ProcessingMetadata `json:"processing_metadata"`
IsActive bool `json:"is_active"`
IsExpired bool `json:"is_expired"`
HasErrors bool `json:"has_errors"`
MatchPercentage *int `json:"match_percentage,omitempty"` // Match percentage for company (0-100)
CreatedAt int64 `json:"created_at"`
UpdatedAt int64 `json:"updated_at"`
} }
// ScrapingJobResponse represents a scraping job in API responses type OrganizationResponse struct {
type ScrapingJobResponse struct { Name string `json:"name"`
ID string `json:"id"`
Type ScrapingJobType `json:"type"`
Status ScrapingJobStatus `json:"status"`
StartedAt int64 `json:"started_at"`
CompletedAt *int64 `json:"completed_at,omitempty"`
Duration *int64 `json:"duration,omitempty"`
Progress JobProgress `json:"progress"`
Results JobResults `json:"results"`
Errors []string `json:"errors,omitempty"`
Config map[string]interface{} `json:"config,omitempty"`
CreatedAt int64 `json:"created_at"`
UpdatedAt int64 `json:"updated_at"`
}
// ScrapingStateResponse represents scraping state in API responses
type ScrapingStateResponse struct {
ID string `json:"id"`
LastProcessedYear int `json:"last_processed_year"`
LastProcessedNumber int `json:"last_processed_number"`
LastRunAt int64 `json:"last_run_at"`
NextRunAt int64 `json:"next_run_at"`
IsRunning bool `json:"is_running"`
CurrentJobID string `json:"current_job_id,omitempty"`
CreatedAt int64 `json:"created_at"`
UpdatedAt int64 `json:"updated_at"`
}
// ValidationErrorResponse represents validation errors in API responses
type ValidationErrorResponse struct {
Field string `json:"field"`
Message string `json:"message"`
Value string `json:"value,omitempty"`
}
// ErrorResponse represents error responses
type ErrorResponse struct {
Error string `json:"error"`
Message string `json:"message"`
ValidationErrors []ValidationErrorResponse `json:"validation_errors,omitempty"`
Timestamp int64 `json:"timestamp"`
} }
// ToTenderResponse converts a Tender entity to TenderResponse // ToTenderResponse converts a Tender entity to TenderResponse
func (t *Tender) ToTenderResponse() *TenderResponse { func (t *Tender) ToTenderResponse() *TenderResponse {
return t.ToTenderResponseWithMatch(nil)
}
// ToTenderResponseWithMatch converts a Tender entity to TenderResponse with optional match percentage
func (t *Tender) ToTenderResponseWithMatch(companyCPVCodes []string) *TenderResponse {
response := &TenderResponse{ response := &TenderResponse{
ID: t.ID.Hex(), ID: t.ID.Hex(),
ContractNoticeID: t.ContractNoticeID, NoticePublicationID: t.NoticePublicationID,
NoticePublicationID: t.NoticePublicationID, PublicationDate: t.PublicationDate,
ContractFolderID: t.ContractFolderID, Title: t.Title,
NoticeTypeCode: t.NoticeTypeCode, Description: t.Description,
NoticeSubTypeCode: t.NoticeSubTypeCode, ProcurementTypeCode: t.ProcurementTypeCode,
NoticeLanguageCode: t.NoticeLanguageCode, ProcedureCode: t.ProcedureCode,
IssueDate: t.IssueDate, EstimatedValue: t.EstimatedValue,
IssueTime: t.IssueTime, Currency: t.Currency,
PublicationDate: t.PublicationDate, Duration: t.Duration,
GazetteID: t.GazetteID, DurationUnit: t.DurationUnit,
Title: t.Title, TenderDeadline: t.TenderDeadline,
Description: t.Description, CountryCode: t.CountryCode,
ProcurementTypeCode: t.ProcurementTypeCode, BuyerOrganization: &OrganizationResponse{Name: t.BuyerOrganization.Name},
ProcedureCode: t.ProcedureCode, Status: t.Status,
MainClassification: t.MainClassification,
AdditionalClassifications: t.AdditionalClassifications,
EstimatedValue: t.EstimatedValue,
Currency: t.Currency,
FormattedEstimatedValue: t.GetFormattedEstimatedValue(),
Duration: t.Duration,
DurationUnit: t.DurationUnit,
TenderDeadline: t.TenderDeadline,
PlaceOfPerformance: t.PlaceOfPerformance,
CountryCode: t.CountryCode,
RegionCode: t.RegionCode,
CityName: t.CityName,
PostalCode: t.PostalCode,
DocumentURI: t.DocumentURI,
TenderURL: t.TenderURL,
BuyerOrganization: t.BuyerOrganization,
ReviewOrganization: t.ReviewOrganization,
Organizations: t.Organizations,
SelectionCriteria: t.SelectionCriteria,
OfficialLanguages: t.OfficialLanguages,
Status: t.Status,
Source: t.Source,
SourceFileURL: t.SourceFileURL,
SourceFileName: t.SourceFileName,
ProcessingMetadata: t.ProcessingMetadata,
IsActive: t.IsActive(),
IsExpired: t.IsExpired(),
HasErrors: t.HasErrors(),
CreatedAt: t.CreatedAt,
UpdatedAt: t.UpdatedAt,
}
// Calculate match percentage if company CPV codes are provided
if companyCPVCodes != nil {
matchPercentage := t.CalculateMatchPercentage(companyCPVCodes)
response.MatchPercentage = &matchPercentage
} }
return response return response
} }
// ToScrapingJobResponse converts a ScrapingJob entity to ScrapingJobResponse
func (j *ScrapingJob) ToScrapingJobResponse() *ScrapingJobResponse {
return &ScrapingJobResponse{
ID: j.ID,
Type: j.Type,
Status: j.Status,
StartedAt: j.StartedAt,
CompletedAt: j.CompletedAt,
Duration: j.Duration,
Progress: j.Progress,
Results: j.Results,
Errors: j.Errors,
Config: j.Config,
CreatedAt: j.CreatedAt,
UpdatedAt: j.UpdatedAt,
}
}
// ToScrapingStateResponse converts a ScrapingState entity to ScrapingStateResponse
func (s *ScrapingState) ToScrapingStateResponse() *ScrapingStateResponse {
return &ScrapingStateResponse{
ID: s.ID,
LastProcessedYear: s.LastProcessedYear,
LastProcessedNumber: s.LastProcessedNumber,
LastRunAt: s.LastRunAt,
NextRunAt: s.NextRunAt,
IsRunning: s.IsRunning,
CurrentJobID: s.CurrentJobID,
CreatedAt: s.CreatedAt,
UpdatedAt: s.UpdatedAt,
}
}
internal/tender/handler.go
+13 -364
View File
@@ -24,40 +24,6 @@ func NewTenderHandler(service TenderService, logger logger.Logger) *TenderHandle
} }
} }
// CreateTender creates a new tender
// @Summary Create a new tender
// @Description Create a new tender with the provided information
// @Tags Admin-Tenders
// @Accept json
// @Produce json
// @Param tender body CreateTenderRequest true "Tender information"
// @Success 201 {object} response.APIResponse{data=TenderResponse}
// @Failure 400 {object} response.APIResponse
// @Failure 409 {object} response.APIResponse
// @Failure 500 {object} response.APIResponse
// @Router /admin/tenders [post]
// @Security BearerAuth
func (h *TenderHandler) CreateTender(c echo.Context) error {
var req CreateTenderRequest
if err := c.Bind(&req); err != nil {
return response.ValidationError(c, "Invalid request format", err.Error())
}
if _, err := govalidator.ValidateStruct(req); err != nil {
return response.ValidationError(c, "Validation failed", err.Error())
}
tender, err := h.service.CreateTender(c.Request().Context(), req)
if err != nil {
if err.Error() == "tender with notice publication ID already exists" {
return response.Conflict(c, "Tender already exists")
}
return response.InternalServerError(c, "Failed to create tender")
}
return response.Created(c, tender.ToTenderResponse(), "Tender created successfully")
}
// GetTender retrieves a tender by ID // GetTender retrieves a tender by ID
// @Summary Get tender by ID // @Summary Get tender by ID
// @Description Retrieve a specific tender by its ID // @Description Retrieve a specific tender by its ID
@@ -80,7 +46,7 @@ func (h *TenderHandler) GetTender(c echo.Context) error {
return response.NotFound(c, "Tender not found") return response.NotFound(c, "Tender not found")
} }
return response.Success(c, tender.ToTenderResponse(), "Tender retrieved successfully") return response.Success(c, tender, "Tender retrieved successfully")
} }
// UpdateTender updates an existing tender // UpdateTender updates an existing tender
@@ -250,247 +216,7 @@ func (h *TenderHandler) ListTenders(c echo.Context) error {
return response.InternalServerError(c, "Failed to retrieve tenders") return response.InternalServerError(c, "Failed to retrieve tenders")
} }
responseData := map[string]interface{}{ return response.Success(c, result, "Tenders retrieved successfully")
"tenders": result.Tenders,
"total": result.Total,
"limit": result.Limit,
"offset": result.Offset,
}
return response.Success(c, responseData, "Tenders retrieved successfully")
}
// SearchTenders searches tenders
// @Summary Search tenders
// @Description Search tenders using full-text search and filtering
// @Tags Admin-Tenders
// @Accept json
// @Produce json
// @Param search body SearchTendersRequest true "Search criteria"
// @Success 200 {object} response.APIResponse{data=map[string]interface{}}
// @Failure 400 {object} response.APIResponse
// @Failure 500 {object} response.APIResponse
// @Router /admin/tenders/search [post]
// @Security BearerAuth
func (h *TenderHandler) SearchTenders(c echo.Context) error {
var req SearchTendersRequest
if err := c.Bind(&req); err != nil {
return response.ValidationError(c, "Invalid request format", err.Error())
}
if _, err := govalidator.ValidateStruct(req); err != nil {
return response.ValidationError(c, "Validation failed", err.Error())
}
result, err := h.service.SearchTenders(c.Request().Context(), req)
if err != nil {
return response.InternalServerError(c, "Failed to search tenders")
}
// Convert tenders to response format
tenderResponses := make([]TenderResponse, len(result.Tenders))
for i, tender := range result.Tenders {
tenderResponses[i] = *tender.ToTenderResponse()
}
responseData := map[string]interface{}{
"tenders": tenderResponses,
"query": result.Query,
"total": result.Total,
}
return response.Success(c, responseData, "Tenders found successfully")
}
// GetTenderStatistics retrieves tender statistics
// @Summary Get tender statistics
// @Description Retrieve comprehensive statistics about tenders
// @Tags Admin-Tenders
// @Produce json
// @Success 200 {object} response.APIResponse{data=TenderStatistics}
// @Failure 500 {object} response.APIResponse
// @Router /admin/tenders/statistics [get]
// @Security BearerAuth
func (h *TenderHandler) GetTenderStatistics(c echo.Context) error {
stats, err := h.service.GetTenderStatistics(c.Request().Context())
if err != nil {
return response.InternalServerError(c, "Failed to retrieve statistics")
}
return response.Success(c, stats, "Statistics retrieved successfully")
}
// GetDashboard retrieves dashboard data
// @Summary Get dashboard data
// @Description Retrieve dashboard data including statistics, recent tenders, and active jobs
// @Tags Admin-Tenders
// @Produce json
// @Success 200 {object} response.APIResponse{data=map[string]interface{}}
// @Failure 500 {object} response.APIResponse
// @Router /admin/tenders/dashboard [get]
// @Security BearerAuth
func (h *TenderHandler) GetDashboard(c echo.Context) error {
dashboard, err := h.service.GetDashboardData(c.Request().Context())
if err != nil {
return response.InternalServerError(c, "Failed to retrieve dashboard data")
}
// Convert recent tenders to response format
recentTenderResponses := make([]TenderResponse, len(dashboard.RecentTenders))
for i, tender := range dashboard.RecentTenders {
recentTenderResponses[i] = *tender.ToTenderResponse()
}
// Convert active jobs to response format
activeJobResponses := make([]ScrapingJobResponse, len(dashboard.ActiveJobs))
for i, job := range dashboard.ActiveJobs {
activeJobResponses[i] = *job.ToScrapingJobResponse()
}
responseData := map[string]interface{}{
"statistics": dashboard.Statistics,
"recent_tenders": recentTenderResponses,
"active_jobs": activeJobResponses,
}
return response.Success(c, responseData, "Dashboard data retrieved successfully")
}
// StartScrapingJob starts a new scraping job
// @Summary Start scraping job
// @Description Start a new TED XML scraping job
// @Tags Admin-Scraping
// @Accept json
// @Produce json
// @Param job body StartScrapingJobRequest true "Scraping job configuration"
// @Success 201 {object} response.APIResponse{data=ScrapingJobResponse}
// @Failure 400 {object} response.APIResponse
// @Failure 500 {object} response.APIResponse
// @Router /admin/scraping/jobs [post]
// @Security BearerAuth
func (h *TenderHandler) StartScrapingJob(c echo.Context) error {
var req StartScrapingJobRequest
if err := c.Bind(&req); err != nil {
return response.ValidationError(c, "Invalid request format", err.Error())
}
if _, err := govalidator.ValidateStruct(req); err != nil {
return response.ValidationError(c, "Validation failed", err.Error())
}
job, err := h.service.StartScrapingJob(c.Request().Context(), req)
if err != nil {
return response.InternalServerError(c, "Failed to start scraping job")
}
return response.Created(c, job.ToScrapingJobResponse(), "Scraping job started successfully")
}
// GetScrapingJob retrieves a scraping job by ID
// @Summary Get scraping job
// @Description Retrieve a specific scraping job by its ID
// @Tags Admin-Scraping
// @Produce json
// @Param id path string true "Scraping job ID"
// @Success 200 {object} response.APIResponse{data=ScrapingJobResponse}
// @Failure 404 {object} response.APIResponse
// @Failure 500 {object} response.APIResponse
// @Router /admin/scraping/jobs/{id} [get]
// @Security BearerAuth
func (h *TenderHandler) GetScrapingJob(c echo.Context) error {
id := c.Param("id")
if id == "" {
return response.BadRequest(c, "Scraping job ID is required", "ID parameter cannot be empty")
}
job, err := h.service.GetScrapingJobStatus(c.Request().Context(), id)
if err != nil {
return response.NotFound(c, "Scraping job not found")
}
return response.Success(c, job.ToScrapingJobResponse(), "Scraping job retrieved successfully")
}
// ListScrapingJobs retrieves scraping jobs with pagination
// @Summary List scraping jobs
// @Description Retrieve scraping jobs with pagination
// @Tags Admin-Scraping
// @Produce json
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
// @Param offset query int false "Number of items to skip (default: 0)"
// @Success 200 {object} response.APIResponse{data=map[string]interface{}}
// @Failure 400 {object} response.APIResponse
// @Failure 500 {object} response.APIResponse
// @Security BearerAuth
// @Router /admin/scraping/jobs [get]
func (h *TenderHandler) ListScrapingJobs(c echo.Context) error {
// Parse pagination parameters
limit := 20
if l := c.QueryParam("limit"); l != "" {
if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 && parsed <= 100 {
limit = parsed
}
}
offset := 0
if o := c.QueryParam("offset"); o != "" {
if parsed, err := strconv.Atoi(o); err == nil && parsed >= 0 {
offset = parsed
}
}
req := ListScrapingJobsRequest{
Limit: limit,
Offset: offset,
}
result, err := h.service.ListScrapingJobs(c.Request().Context(), req)
if err != nil {
return response.InternalServerError(c, "Failed to retrieve scraping jobs")
}
// Convert jobs to response format
jobResponses := make([]ScrapingJobResponse, len(result.Jobs))
for i, job := range result.Jobs {
jobResponses[i] = *job.ToScrapingJobResponse()
}
responseData := map[string]interface{}{
"jobs": jobResponses,
"total": result.Total,
"limit": result.Limit,
"offset": result.Offset,
}
return response.Success(c, responseData, "Scraping jobs retrieved successfully")
}
// CancelScrapingJob cancels a running scraping job
// @Summary Cancel scraping job
// @Description Cancel a running or pending scraping job
// @Tags Admin-Scraping
// @Param id path string true "Scraping job ID"
// @Success 200 {object} response.APIResponse
// @Failure 400 {object} response.APIResponse
// @Failure 404 {object} response.APIResponse
// @Failure 500 {object} response.APIResponse
// @Router /admin/scraping/jobs/{id}/cancel [post]
// @Security BearerAuth
func (h *TenderHandler) CancelScrapingJob(c echo.Context) error {
id := c.Param("id")
if id == "" {
return response.BadRequest(c, "Scraping job ID is required", "ID parameter cannot be empty")
}
err := h.service.CancelScrapingJob(c.Request().Context(), id)
if err != nil {
if err.Error() == "scraping job not found" {
return response.NotFound(c, "Scraping job not found")
}
return response.InternalServerError(c, "Failed to cancel scraping job")
}
return response.Success(c, map[string]interface{}{"cancelled": true}, "Scraping job cancelled successfully")
} }
// GetPublicTenders retrieves public tenders for mobile app // GetPublicTenders retrieves public tenders for mobile app
@@ -511,17 +237,13 @@ func (h *TenderHandler) CancelScrapingJob(c echo.Context) error {
func (h *TenderHandler) GetPublicTenders(c echo.Context) error { func (h *TenderHandler) GetPublicTenders(c echo.Context) error {
// Parse pagination parameters (more restrictive for public API) // Parse pagination parameters (more restrictive for public API)
limit := 20 limit := 20
if l := c.QueryParam("limit"); l != "" { if parsed, err := strconv.Atoi(c.QueryParam("limit")); err == nil && parsed > 0 && parsed <= 50 {
if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 && parsed <= 50 { limit = parsed
limit = parsed
}
} }
offset := 0 offset := 0
if o := c.QueryParam("offset"); o != "" { if parsed, err := strconv.Atoi(c.QueryParam("offset")); err == nil && parsed >= 0 {
if parsed, err := strconv.Atoi(o); err == nil && parsed >= 0 { offset = parsed
offset = parsed
}
} }
// Build search criteria for public API (only active tenders) // Build search criteria for public API (only active tenders)
@@ -534,21 +256,17 @@ func (h *TenderHandler) GetPublicTenders(c echo.Context) error {
} }
// Parse numeric parameters // Parse numeric parameters
if minValue := c.QueryParam("min_estimated_value"); minValue != "" { if parsed, err := strconv.ParseFloat(c.QueryParam("min_estimated_value"), 64); err == nil {
if parsed, err := strconv.ParseFloat(minValue, 64); err == nil { criteria.MinEstimatedValue = &parsed
criteria.MinEstimatedValue = &parsed
}
} }
if maxValue := c.QueryParam("max_estimated_value"); maxValue != "" { if parsed, err := strconv.ParseFloat(c.QueryParam("max_estimated_value"), 64); err == nil {
if parsed, err := strconv.ParseFloat(maxValue, 64); err == nil { criteria.MaxEstimatedValue = &parsed
criteria.MaxEstimatedValue = &parsed
}
} }
// Get company ID from customer context for automatic matching // Get company ID from customer context for automatic matching
var companyID *string var companyID *string
if customerCompanyID, ok := c.Get("company_id").(string); ok && customerCompanyID != "" { if customerCompanyID, ok := c.Get("company_id").(string); ok {
companyID = &customerCompanyID companyID = &customerCompanyID
} }
@@ -564,43 +282,7 @@ func (h *TenderHandler) GetPublicTenders(c echo.Context) error {
return response.InternalServerError(c, "Failed to retrieve tenders") return response.InternalServerError(c, "Failed to retrieve tenders")
} }
// Convert tenders to public response format (limited fields) return response.Success(c, result, "Tenders retrieved successfully")
publicTenders := make([]map[string]interface{}, len(result.Tenders))
for i, tender := range result.Tenders {
tenderData := map[string]interface{}{
"id": tender.ID,
"title": tender.Title,
"description": tender.Description,
"procurement_type_code": tender.ProcurementTypeCode,
"main_classification": tender.MainClassification,
"estimated_value": tender.EstimatedValue,
"currency": tender.Currency,
"formatted_estimated_value": tender.FormattedEstimatedValue,
"tender_deadline": tender.TenderDeadline,
"country_code": tender.CountryCode,
"city_name": tender.CityName,
"tender_url": tender.TenderURL,
"publication_date": tender.PublicationDate,
"buyer_organization": tender.BuyerOrganization,
"is_expired": tender.IsExpired,
}
// Add match percentage if available
if tender.MatchPercentage != nil {
tenderData["match_percentage"] = *tender.MatchPercentage
}
publicTenders[i] = tenderData
}
responseData := map[string]interface{}{
"tenders": publicTenders,
"total": result.Total,
"limit": result.Limit,
"offset": result.Offset,
}
return response.Success(c, responseData, "Tenders retrieved successfully")
} }
// GetPublicTenderDetails retrieves public tender details for mobile app // GetPublicTenderDetails retrieves public tender details for mobile app
@@ -625,40 +307,7 @@ func (h *TenderHandler) GetPublicTenderDetails(c echo.Context) error {
return response.NotFound(c, "Tender not found") return response.NotFound(c, "Tender not found")
} }
// Return public-safe tender details return response.Success(c, tender, "Tender details retrieved successfully")
publicTender := map[string]interface{}{
"id": tender.ID,
"notice_publication_id": tender.NoticePublicationID,
"title": tender.Title,
"description": tender.Description,
"procurement_type_code": tender.ProcurementTypeCode,
"procedure_code": tender.ProcedureCode,
"main_classification": tender.MainClassification,
"additional_classifications": tender.AdditionalClassifications,
"estimated_value": tender.EstimatedValue,
"currency": tender.Currency,
"formatted_estimated_value": tender.GetFormattedEstimatedValue(),
"duration": tender.Duration,
"duration_unit": tender.DurationUnit,
"tender_deadline": tender.TenderDeadline,
"place_of_performance": tender.PlaceOfPerformance,
"country_code": tender.CountryCode,
"region_code": tender.RegionCode,
"city_name": tender.CityName,
"postal_code": tender.PostalCode,
"document_uri": tender.DocumentURI,
"tender_url": tender.TenderURL,
"buyer_organization": tender.BuyerOrganization,
"organizations": tender.Organizations,
"selection_criteria": tender.SelectionCriteria,
"official_languages": tender.OfficialLanguages,
"publication_date": tender.PublicationDate,
"gazette_id": tender.GazetteID,
"is_active": tender.IsActive(),
"is_expired": tender.IsExpired(),
}
return response.Success(c, publicTender, "Tender details retrieved successfully")
} }
// parseStringArray parses a comma-separated string into a string array // parseStringArray parses a comma-separated string into a string array
internal/tender/repository.go
-198
View File
@@ -7,7 +7,6 @@ import (
mongopkg "tm/pkg/mongo" mongopkg "tm/pkg/mongo"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
) )
@@ -38,24 +37,11 @@ type TenderRepository interface {
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error) GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
GetTenderCountByType(ctx context.Context) (map[string]int64, error) GetTenderCountByType(ctx context.Context) (map[string]int64, error)
GetTenderCountByClassification(ctx context.Context) (map[string]int64, error) GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
// Scraping state management
GetScrapingState(ctx context.Context) (*ScrapingState, error)
SaveScrapingState(ctx context.Context, state *ScrapingState) error
// Scraping jobs management
CreateScrapingJob(ctx context.Context, job *ScrapingJob) error
GetScrapingJob(ctx context.Context, id string) (*ScrapingJob, error)
UpdateScrapingJob(ctx context.Context, job *ScrapingJob) error
ListScrapingJobs(ctx context.Context, limit, offset int) ([]ScrapingJob, int64, error)
GetActiveScrapingJobs(ctx context.Context) ([]ScrapingJob, error)
} }
// tenderRepository implements TenderRepository interface using MongoDB ORM // tenderRepository implements TenderRepository interface using MongoDB ORM
type tenderRepository struct { type tenderRepository struct {
ormRepo mongopkg.Repository[Tender] ormRepo mongopkg.Repository[Tender]
stateOrmRepo mongopkg.Repository[ScrapingState]
jobOrmRepo mongopkg.Repository[ScrapingJob]
mongoManager *mongopkg.ConnectionManager mongoManager *mongopkg.ConnectionManager
logger logger.Logger logger logger.Logger
} }
@@ -97,17 +83,6 @@ func NewTenderRepository(mongoManager *mongopkg.ConnectionManager, logger logger
*mongopkg.CreateTextIndex("search_idx", "title", "description"), *mongopkg.CreateTextIndex("search_idx", "title", "description"),
} }
// Create indexes for scraping jobs collection
jobIndexes := []mongopkg.Index{
*mongopkg.NewIndex("type_idx", bson.D{{Key: "type", Value: 1}}),
*mongopkg.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
*mongopkg.NewIndex("started_at_idx", bson.D{{Key: "started_at", Value: -1}}),
*mongopkg.NewIndex("type_status_idx", bson.D{
{Key: "type", Value: 1},
{Key: "status", Value: 1},
}),
}
// Create indexes // Create indexes
if err := mongoManager.CreateIndexes("tenders", tenderIndexes); err != nil { if err := mongoManager.CreateIndexes("tenders", tenderIndexes); err != nil {
logger.Warn("Failed to create tender indexes", map[string]interface{}{ logger.Warn("Failed to create tender indexes", map[string]interface{}{
@@ -115,21 +90,11 @@ func NewTenderRepository(mongoManager *mongopkg.ConnectionManager, logger logger
}) })
} }
if err := mongoManager.CreateIndexes("scraping_jobs", jobIndexes); err != nil {
logger.Warn("Failed to create scraping job indexes", map[string]interface{}{
"error": err.Error(),
})
}
// Create ORM repositories // Create ORM repositories
tenderOrmRepo := mongopkg.NewRepository[Tender](mongoManager.GetCollection("tenders"), logger) tenderOrmRepo := mongopkg.NewRepository[Tender](mongoManager.GetCollection("tenders"), logger)
stateOrmRepo := mongopkg.NewRepository[ScrapingState](mongoManager.GetCollection("scraping_state"), logger)
jobOrmRepo := mongopkg.NewRepository[ScrapingJob](mongoManager.GetCollection("scraping_jobs"), logger)
return &tenderRepository{ return &tenderRepository{
ormRepo: tenderOrmRepo, ormRepo: tenderOrmRepo,
stateOrmRepo: stateOrmRepo,
jobOrmRepo: jobOrmRepo,
mongoManager: mongoManager, mongoManager: mongoManager,
logger: logger, logger: logger,
} }
@@ -584,169 +549,6 @@ func (r *tenderRepository) GetTenderCountByClassification(ctx context.Context) (
return counts, nil return counts, nil
} }
// GetScrapingState retrieves the current scraping state
func (r *tenderRepository) GetScrapingState(ctx context.Context) (*ScrapingState, error) {
// Try to find existing state
filter := bson.M{}
pagination := mongopkg.Pagination{Limit: 1}
result, err := r.stateOrmRepo.FindAll(ctx, filter, pagination)
if err != nil {
return nil, err
}
if len(result.Items) == 0 {
// Return default state if none exists
return &ScrapingState{
ID: "default",
LastProcessedYear: time.Now().Year(),
LastProcessedNumber: 0,
LastRunAt: 0,
NextRunAt: time.Now().Add(4 * time.Hour).Unix(),
IsRunning: false,
CreatedAt: time.Now().Unix(),
UpdatedAt: time.Now().Unix(),
}, nil
}
return &result.Items[0], nil
}
// SaveScrapingState saves the scraping state
func (r *tenderRepository) SaveScrapingState(ctx context.Context, state *ScrapingState) error {
if state.ID == "" {
state.ID = "default"
}
now := time.Now().Unix()
if state.CreatedAt == 0 {
state.CreatedAt = now
}
state.UpdatedAt = now
// Try to update existing state first
existing, err := r.GetScrapingState(ctx)
if err == nil && existing.ID != "" {
state.ID = existing.ID
state.CreatedAt = existing.CreatedAt
return r.stateOrmRepo.Update(ctx, state)
}
// Create new state if none exists
return r.stateOrmRepo.Create(ctx, state)
}
// CreateScrapingJob creates a new scraping job
func (r *tenderRepository) CreateScrapingJob(ctx context.Context, job *ScrapingJob) error {
if job.ID == "" {
job.ID = primitive.NewObjectID().Hex()
}
now := time.Now().Unix()
job.CreatedAt = now
job.UpdatedAt = now
err := r.jobOrmRepo.Create(ctx, job)
if err != nil {
r.logger.Error("Failed to create scraping job", map[string]interface{}{
"error": err.Error(),
"type": job.Type,
})
return err
}
r.logger.Info("Scraping job created successfully", map[string]interface{}{
"job_id": job.ID,
"type": job.Type,
})
return nil
}
// GetScrapingJob retrieves a scraping job by ID
func (r *tenderRepository) GetScrapingJob(ctx context.Context, id string) (*ScrapingJob, error) {
job, err := r.jobOrmRepo.FindByID(ctx, id)
if err != nil {
r.logger.Error("Failed to get scraping job", map[string]interface{}{
"job_id": id,
"error": err.Error(),
})
return nil, err
}
return job, nil
}
// UpdateScrapingJob updates a scraping job
func (r *tenderRepository) UpdateScrapingJob(ctx context.Context, job *ScrapingJob) error {
job.UpdatedAt = time.Now().Unix()
err := r.jobOrmRepo.Update(ctx, job)
if err != nil {
r.logger.Error("Failed to update scraping job", map[string]interface{}{
"job_id": job.ID,
"error": err.Error(),
})
return err
}
r.logger.Info("Scraping job updated successfully", map[string]interface{}{
"job_id": job.ID,
})
return nil
}
// ListScrapingJobs retrieves scraping jobs with pagination
func (r *tenderRepository) ListScrapingJobs(ctx context.Context, limit, offset int) ([]ScrapingJob, int64, error) {
filter := bson.M{}
pagination := mongopkg.Pagination{
Limit: limit,
Skip: offset,
SortField: "started_at",
SortOrder: -1,
}
result, err := r.jobOrmRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to list scraping jobs", map[string]interface{}{
"error": err.Error(),
})
return nil, 0, err
}
return result.Items, result.TotalCount, nil
}
// GetActiveScrapingJobs retrieves currently active scraping jobs
func (r *tenderRepository) GetActiveScrapingJobs(ctx context.Context) ([]ScrapingJob, error) {
filter := bson.M{
"status": bson.M{
"$in": []ScrapingJobStatus{
ScrapingJobStatusPending,
ScrapingJobStatusRunning,
},
},
}
pagination := mongopkg.Pagination{
Limit: 100,
SortField: "started_at",
SortOrder: -1,
}
result, err := r.jobOrmRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to get active scraping jobs", map[string]interface{}{
"error": err.Error(),
})
return nil, err
}
return result.Items, nil
}
// buildSearchFilter builds MongoDB filter from search criteria // buildSearchFilter builds MongoDB filter from search criteria
func (r *tenderRepository) buildSearchFilter(criteria TenderSearchCriteria) bson.M { func (r *tenderRepository) buildSearchFilter(criteria TenderSearchCriteria) bson.M {
filter := bson.M{} filter := bson.M{}
internal/tender/service.go
+25 -332
View File
@@ -3,33 +3,19 @@ package tender
import ( import (
"context" "context"
"fmt" "fmt"
"sort"
"time" "time"
"tm/internal/company" "tm/internal/company"
"tm/pkg/logger" "tm/pkg/logger"
"tm/pkg/response"
"go.mongodb.org/mongo-driver/bson/primitive"
) )
// TenderService interface defines tender business logic operations // TenderService interface defines tender business logic operations
type TenderService interface { type TenderService interface {
// Tender operations // Tender operations
CreateTender(ctx context.Context, req CreateTenderRequest) (*Tender, error) GetTenderByID(ctx context.Context, id string) (*TenderResponse, error)
GetTenderByID(ctx context.Context, id string) (*Tender, error)
UpdateTender(ctx context.Context, req UpdateTenderRequest) (*Tender, error) UpdateTender(ctx context.Context, req UpdateTenderRequest) (*Tender, error)
DeleteTender(ctx context.Context, id string) error DeleteTender(ctx context.Context, id string) error
ListTenders(ctx context.Context, req ListTendersRequest) (*ListTendersResponse, error) ListTenders(ctx context.Context, req ListTendersRequest) (*ListTendersResponse, error)
SearchTenders(ctx context.Context, req SearchTendersRequest) (*SearchTendersResponse, error)
// Statistics and reporting
GetTenderStatistics(ctx context.Context) (*TenderStatistics, error)
GetDashboardData(ctx context.Context) (*DashboardData, error)
// Scraping management
StartScrapingJob(ctx context.Context, req StartScrapingJobRequest) (*ScrapingJob, error)
GetScrapingJobStatus(ctx context.Context, jobID string) (*ScrapingJob, error)
ListScrapingJobs(ctx context.Context, req ListScrapingJobsRequest) (*ListScrapingJobsResponse, error)
CancelScrapingJob(ctx context.Context, jobID string) error
// Maintenance operations // Maintenance operations
UpdateExpiredTenders(ctx context.Context) error UpdateExpiredTenders(ctx context.Context) error
@@ -52,92 +38,8 @@ func NewTenderService(repository TenderRepository, companyService company.Servic
} }
} }
// CreateTender creates a new tender
func (s *tenderService) CreateTender(ctx context.Context, req CreateTenderRequest) (*Tender, error) {
s.logger.Info("Creating new tender", map[string]interface{}{
"notice_publication_id": req.NoticePublicationID,
"title": req.Title,
})
// Check if tender with same notice publication ID already exists
if req.NoticePublicationID != "" {
existing, err := s.repository.GetByNoticePublicationID(ctx, req.NoticePublicationID)
if err == nil && existing != nil {
s.logger.Warn("Tender with same notice publication ID already exists", map[string]interface{}{
"notice_publication_id": req.NoticePublicationID,
"existing_id": existing.ID,
})
return nil, fmt.Errorf("tender with notice publication ID %s already exists", req.NoticePublicationID)
}
}
tender := &Tender{
ContractNoticeID: req.ContractNoticeID,
NoticePublicationID: req.NoticePublicationID,
ContractFolderID: req.ContractFolderID,
NoticeTypeCode: req.NoticeTypeCode,
NoticeSubTypeCode: req.NoticeSubTypeCode,
NoticeLanguageCode: req.NoticeLanguageCode,
IssueDate: req.IssueDate,
IssueTime: req.IssueTime,
PublicationDate: req.PublicationDate,
GazetteID: req.GazetteID,
Title: req.Title,
Description: req.Description,
ProcurementTypeCode: req.ProcurementTypeCode,
ProcedureCode: req.ProcedureCode,
MainClassification: req.MainClassification,
AdditionalClassifications: req.AdditionalClassifications,
EstimatedValue: req.EstimatedValue,
Currency: req.Currency,
Duration: req.Duration,
DurationUnit: req.DurationUnit,
TenderDeadline: req.TenderDeadline,
PlaceOfPerformance: req.PlaceOfPerformance,
CountryCode: req.CountryCode,
RegionCode: req.RegionCode,
CityName: req.CityName,
PostalCode: req.PostalCode,
DocumentURI: req.DocumentURI,
TenderURL: req.TenderURL,
BuyerOrganization: req.BuyerOrganization,
ReviewOrganization: req.ReviewOrganization,
Organizations: req.Organizations,
SelectionCriteria: req.SelectionCriteria,
OfficialLanguages: req.OfficialLanguages,
Status: TenderStatusActive,
Source: req.Source,
SourceFileURL: req.SourceFileURL,
SourceFileName: req.SourceFileName,
ProcessingMetadata: ProcessingMetadata{
ProcessedAt: time.Now().Unix(),
ProcessingVersion: "1.0",
},
}
// Generate tender URL if not provided
if tender.TenderURL == "" && tender.NoticePublicationID != "" {
tender.TenderURL = generateTenderURL(tender.NoticePublicationID)
}
if err := s.repository.Create(ctx, tender); err != nil {
s.logger.Error("Failed to create tender", map[string]interface{}{
"error": err.Error(),
"notice_publication_id": req.NoticePublicationID,
})
return nil, fmt.Errorf("failed to create tender: %w", err)
}
s.logger.Info("Tender created successfully", map[string]interface{}{
"tender_id": tender.ID,
"notice_publication_id": tender.NoticePublicationID,
})
return tender, nil
}
// GetTenderByID retrieves a tender by ID // GetTenderByID retrieves a tender by ID
func (s *tenderService) GetTenderByID(ctx context.Context, id string) (*Tender, error) { func (s *tenderService) GetTenderByID(ctx context.Context, id string) (*TenderResponse, error) {
s.logger.Info("Retrieving tender by ID", map[string]interface{}{ s.logger.Info("Retrieving tender by ID", map[string]interface{}{
"tender_id": id, "tender_id": id,
}) })
@@ -151,7 +53,7 @@ func (s *tenderService) GetTenderByID(ctx context.Context, id string) (*Tender,
return nil, fmt.Errorf("failed to retrieve tender: %w", err) return nil, fmt.Errorf("failed to retrieve tender: %w", err)
} }
return tender, nil return tender.ToTenderResponse(), nil
} }
// UpdateTender updates an existing tender // UpdateTender updates an existing tender
@@ -265,9 +167,11 @@ func (s *tenderService) ListTenders(ctx context.Context, req ListTendersRequest)
return &ListTendersResponse{ return &ListTendersResponse{
Tenders: tenderResponses, Tenders: tenderResponses,
Total: total, Metadata: &response.Meta{
Limit: req.Limit, Total: int(total),
Offset: req.Offset, Limit: req.Limit,
Offset: req.Offset,
},
}, nil }, nil
} }
@@ -286,9 +190,11 @@ func (s *tenderService) ListTenders(ctx context.Context, req ListTendersRequest)
return &ListTendersResponse{ return &ListTendersResponse{
Tenders: tenderResponses, Tenders: tenderResponses,
Total: total, Metadata: &response.Meta{
Limit: req.Limit, Total: int(total),
Offset: req.Offset, Limit: req.Limit,
Offset: req.Offset,
},
}, nil }, nil
} }
@@ -297,79 +203,24 @@ func (s *tenderService) ListTenders(ctx context.Context, req ListTendersRequest)
companyCPVCodes = comp.Tags.CPVCodes companyCPVCodes = comp.Tags.CPVCodes
} }
// Calculate match percentages and prepare for sorting
tenderMatches := make([]TenderWithMatch, 0, len(tenders))
for _, tender := range tenders {
tenderMatch := TenderWithMatch{
Tender: &tender,
MatchPercentage: tender.CalculateMatchPercentage(companyCPVCodes),
DaysUntilDeadline: tender.GetDaysUntilDeadline(),
}
tenderMatches = append(tenderMatches, tenderMatch)
}
// Sort by match percentage (descending) and then by deadline preference
sort.Slice(tenderMatches, func(i, j int) bool {
matchI := tenderMatches[i].MatchPercentage
matchJ := tenderMatches[j].MatchPercentage
// First, sort by match percentage (descending)
if matchI != matchJ {
return matchI > matchJ
}
// For equal match percentages, prioritize tenders with >30 days deadline
daysI := tenderMatches[i].DaysUntilDeadline
daysJ := tenderMatches[j].DaysUntilDeadline
// Both have >30 days or both have <=30 days, sort by deadline descending (more time is better)
if (daysI > 30 && daysJ > 30) || (daysI <= 30 && daysJ <= 30) {
return daysI > daysJ
}
// One has >30 days, one has <=30 days - prioritize the one with >30 days
return daysI > 30
})
// Convert to TenderResponse objects with match percentages
tenderResponses := make([]TenderResponse, 0, len(tenderMatches))
for _, tenderMatch := range tenderMatches {
tenderResponses = append(tenderResponses, *tenderMatch.Tender.ToTenderResponseWithMatch(companyCPVCodes))
}
s.logger.Info("Tenders sorted by match percentage", map[string]interface{}{ s.logger.Info("Tenders sorted by match percentage", map[string]interface{}{
"company_id": *req.CompanyID, "company_id": *req.CompanyID,
"company_cpv_codes": len(companyCPVCodes), "company_cpv_codes": len(companyCPVCodes),
"total_tenders": len(tenderResponses), "total_tenders": len(tenders),
}) })
tenderResponses := make([]TenderResponse, 0, len(tenders))
for _, tender := range tenders {
tenderResponses = append(tenderResponses, *tender.ToTenderResponse())
}
return &ListTendersResponse{ return &ListTendersResponse{
Tenders: tenderResponses, Tenders: tenderResponses,
Total: total, Metadata: &response.Meta{
Limit: req.Limit, Total: int(total),
Offset: req.Offset, Limit: req.Limit,
}, nil Offset: req.Offset,
} },
// SearchTenders searches tenders based on criteria
func (s *tenderService) SearchTenders(ctx context.Context, req SearchTendersRequest) (*SearchTendersResponse, error) {
s.logger.Info("Searching tenders", map[string]interface{}{
"query": req.Query,
})
tenders, err := s.repository.Search(ctx, req.Criteria)
if err != nil {
s.logger.Error("Failed to search tenders", map[string]interface{}{
"error": err.Error(),
"query": req.Query,
})
return nil, fmt.Errorf("failed to search tenders: %w", err)
}
return &SearchTendersResponse{
Tenders: tenders,
Query: req.Query,
Total: len(tenders),
}, nil }, nil
} }
@@ -388,164 +239,6 @@ func (s *tenderService) GetTenderStatistics(ctx context.Context) (*TenderStatist
return stats, nil return stats, nil
} }
// GetDashboardData retrieves dashboard data
func (s *tenderService) GetDashboardData(ctx context.Context) (*DashboardData, error) {
s.logger.Info("Retrieving dashboard data", map[string]interface{}{})
stats, err := s.GetTenderStatistics(ctx)
if err != nil {
return nil, err
}
// Get recent tenders
recentTenders, _, err := s.repository.List(ctx, TenderSearchCriteria{}, 10, 0)
if err != nil {
s.logger.Error("Failed to retrieve recent tenders", map[string]interface{}{
"error": err.Error(),
})
return nil, fmt.Errorf("failed to retrieve recent tenders: %w", err)
}
// Get active scraping jobs
activeJobs, err := s.repository.GetActiveScrapingJobs(ctx)
if err != nil {
s.logger.Error("Failed to retrieve active scraping jobs", map[string]interface{}{
"error": err.Error(),
})
return nil, fmt.Errorf("failed to retrieve active jobs: %w", err)
}
return &DashboardData{
Statistics: stats,
RecentTenders: recentTenders,
ActiveJobs: activeJobs,
}, nil
}
// StartScrapingJob starts a new scraping job
func (s *tenderService) StartScrapingJob(ctx context.Context, req StartScrapingJobRequest) (*ScrapingJob, error) {
s.logger.Info("Starting scraping job", map[string]interface{}{
"type": req.Type,
})
job := &ScrapingJob{
ID: primitive.NewObjectID().Hex(),
Type: req.Type,
Status: ScrapingJobStatusPending,
StartedAt: time.Now().Unix(),
Progress: JobProgress{},
Results: JobResults{},
Config: req.Config,
}
if err := s.repository.CreateScrapingJob(ctx, job); err != nil {
s.logger.Error("Failed to create scraping job", map[string]interface{}{
"error": err.Error(),
"type": req.Type,
})
return nil, fmt.Errorf("failed to create scraping job: %w", err)
}
s.logger.Info("Scraping job created successfully", map[string]interface{}{
"job_id": job.ID,
"type": job.Type,
})
return job, nil
}
// GetScrapingJobStatus retrieves scraping job status
func (s *tenderService) GetScrapingJobStatus(ctx context.Context, jobID string) (*ScrapingJob, error) {
s.logger.Info("Retrieving scraping job status", map[string]interface{}{
"job_id": jobID,
})
job, err := s.repository.GetScrapingJob(ctx, jobID)
if err != nil {
s.logger.Error("Failed to retrieve scraping job", map[string]interface{}{
"job_id": jobID,
"error": err.Error(),
})
return nil, fmt.Errorf("failed to retrieve scraping job: %w", err)
}
return job, nil
}
// ListScrapingJobs lists scraping jobs with pagination
func (s *tenderService) ListScrapingJobs(ctx context.Context, req ListScrapingJobsRequest) (*ListScrapingJobsResponse, error) {
s.logger.Info("Listing scraping jobs", map[string]interface{}{
"limit": req.Limit,
"offset": req.Offset,
})
if req.Limit <= 0 {
req.Limit = 20
}
if req.Limit > 100 {
req.Limit = 100
}
jobs, total, err := s.repository.ListScrapingJobs(ctx, req.Limit, req.Offset)
if err != nil {
s.logger.Error("Failed to list scraping jobs", map[string]interface{}{
"error": err.Error(),
})
return nil, fmt.Errorf("failed to list scraping jobs: %w", err)
}
return &ListScrapingJobsResponse{
Jobs: jobs,
Total: total,
Limit: req.Limit,
Offset: req.Offset,
}, nil
}
// CancelScrapingJob cancels a running scraping job
func (s *tenderService) CancelScrapingJob(ctx context.Context, jobID string) error {
s.logger.Info("Cancelling scraping job", map[string]interface{}{
"job_id": jobID,
})
job, err := s.repository.GetScrapingJob(ctx, jobID)
if err != nil {
s.logger.Error("Failed to find scraping job for cancellation", map[string]interface{}{
"job_id": jobID,
"error": err.Error(),
})
return fmt.Errorf("failed to find scraping job: %w", err)
}
if job.Status != ScrapingJobStatusRunning && job.Status != ScrapingJobStatusPending {
s.logger.Warn("Cannot cancel scraping job in current status", map[string]interface{}{
"job_id": jobID,
"status": job.Status,
})
return fmt.Errorf("cannot cancel job in status: %s", job.Status)
}
job.Status = ScrapingJobStatusCancelled
completedAt := time.Now().Unix()
job.CompletedAt = &completedAt
duration := completedAt - job.StartedAt
job.Duration = &duration
if err := s.repository.UpdateScrapingJob(ctx, job); err != nil {
s.logger.Error("Failed to update cancelled scraping job", map[string]interface{}{
"job_id": jobID,
"error": err.Error(),
})
return fmt.Errorf("failed to update scraping job: %w", err)
}
s.logger.Info("Scraping job cancelled successfully", map[string]interface{}{
"job_id": jobID,
})
return nil
}
// UpdateExpiredTenders updates status of expired tenders // UpdateExpiredTenders updates status of expired tenders
func (s *tenderService) UpdateExpiredTenders(ctx context.Context) error { func (s *tenderService) UpdateExpiredTenders(ctx context.Context) error {
s.logger.Info("Updating expired tenders", map[string]interface{}{}) s.logger.Info("Updating expired tenders", map[string]interface{}{})