Refactor Tender Management System by Removing Scraping Functionality and Enhancing Tender API
- Removed all scraping-related functionality, including routes, handlers, services, and repository methods, to streamline the tender management system. - Updated the tender API to focus solely on tender management, enhancing endpoints for listing, retrieving, and updating tenders. - Improved Swagger documentation to reflect the removal of scraping endpoints and the addition of new tender-related features. - Ensured adherence to Clean Architecture principles throughout the refactoring process, maintaining a clear separation of concerns.
This commit is contained in:
n.nakhostin
@@ -7,7 +7,6 @@ import (
|
||||
mongopkg "tm/pkg/mongo"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
@@ -38,24 +37,11 @@ type TenderRepository interface {
|
||||
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
|
||||
GetTenderCountByType(ctx context.Context) (map[string]int64, error)
|
||||
GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
|
||||
|
||||
// Scraping state management
|
||||
GetScrapingState(ctx context.Context) (*ScrapingState, error)
|
||||
SaveScrapingState(ctx context.Context, state *ScrapingState) error
|
||||
|
||||
// Scraping jobs management
|
||||
CreateScrapingJob(ctx context.Context, job *ScrapingJob) error
|
||||
GetScrapingJob(ctx context.Context, id string) (*ScrapingJob, error)
|
||||
UpdateScrapingJob(ctx context.Context, job *ScrapingJob) error
|
||||
ListScrapingJobs(ctx context.Context, limit, offset int) ([]ScrapingJob, int64, error)
|
||||
GetActiveScrapingJobs(ctx context.Context) ([]ScrapingJob, error)
|
||||
}
|
||||
|
||||
// tenderRepository implements TenderRepository interface using MongoDB ORM
|
||||
type tenderRepository struct {
|
||||
ormRepo mongopkg.Repository[Tender]
|
||||
stateOrmRepo mongopkg.Repository[ScrapingState]
|
||||
jobOrmRepo mongopkg.Repository[ScrapingJob]
|
||||
mongoManager *mongopkg.ConnectionManager
|
||||
logger logger.Logger
|
||||
}
|
||||
@@ -97,17 +83,6 @@ func NewTenderRepository(mongoManager *mongopkg.ConnectionManager, logger logger
|
||||
*mongopkg.CreateTextIndex("search_idx", "title", "description"),
|
||||
}
|
||||
|
||||
// Create indexes for scraping jobs collection
|
||||
jobIndexes := []mongopkg.Index{
|
||||
*mongopkg.NewIndex("type_idx", bson.D{{Key: "type", Value: 1}}),
|
||||
*mongopkg.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
|
||||
*mongopkg.NewIndex("started_at_idx", bson.D{{Key: "started_at", Value: -1}}),
|
||||
*mongopkg.NewIndex("type_status_idx", bson.D{
|
||||
{Key: "type", Value: 1},
|
||||
{Key: "status", Value: 1},
|
||||
}),
|
||||
}
|
||||
|
||||
// Create indexes
|
||||
if err := mongoManager.CreateIndexes("tenders", tenderIndexes); err != nil {
|
||||
logger.Warn("Failed to create tender indexes", map[string]interface{}{
|
||||
@@ -115,21 +90,11 @@ func NewTenderRepository(mongoManager *mongopkg.ConnectionManager, logger logger
|
||||
})
|
||||
}
|
||||
|
||||
if err := mongoManager.CreateIndexes("scraping_jobs", jobIndexes); err != nil {
|
||||
logger.Warn("Failed to create scraping job indexes", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Create ORM repositories
|
||||
tenderOrmRepo := mongopkg.NewRepository[Tender](mongoManager.GetCollection("tenders"), logger)
|
||||
stateOrmRepo := mongopkg.NewRepository[ScrapingState](mongoManager.GetCollection("scraping_state"), logger)
|
||||
jobOrmRepo := mongopkg.NewRepository[ScrapingJob](mongoManager.GetCollection("scraping_jobs"), logger)
|
||||
|
||||
return &tenderRepository{
|
||||
ormRepo: tenderOrmRepo,
|
||||
stateOrmRepo: stateOrmRepo,
|
||||
jobOrmRepo: jobOrmRepo,
|
||||
mongoManager: mongoManager,
|
||||
logger: logger,
|
||||
}
|
||||
@@ -584,169 +549,6 @@ func (r *tenderRepository) GetTenderCountByClassification(ctx context.Context) (
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
// GetScrapingState retrieves the current scraping state
|
||||
func (r *tenderRepository) GetScrapingState(ctx context.Context) (*ScrapingState, error) {
|
||||
// Try to find existing state
|
||||
filter := bson.M{}
|
||||
pagination := mongopkg.Pagination{Limit: 1}
|
||||
|
||||
result, err := r.stateOrmRepo.FindAll(ctx, filter, pagination)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(result.Items) == 0 {
|
||||
// Return default state if none exists
|
||||
return &ScrapingState{
|
||||
ID: "default",
|
||||
LastProcessedYear: time.Now().Year(),
|
||||
LastProcessedNumber: 0,
|
||||
LastRunAt: 0,
|
||||
NextRunAt: time.Now().Add(4 * time.Hour).Unix(),
|
||||
IsRunning: false,
|
||||
CreatedAt: time.Now().Unix(),
|
||||
UpdatedAt: time.Now().Unix(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
return &result.Items[0], nil
|
||||
}
|
||||
|
||||
// SaveScrapingState saves the scraping state
|
||||
func (r *tenderRepository) SaveScrapingState(ctx context.Context, state *ScrapingState) error {
|
||||
if state.ID == "" {
|
||||
state.ID = "default"
|
||||
}
|
||||
|
||||
now := time.Now().Unix()
|
||||
if state.CreatedAt == 0 {
|
||||
state.CreatedAt = now
|
||||
}
|
||||
state.UpdatedAt = now
|
||||
|
||||
// Try to update existing state first
|
||||
existing, err := r.GetScrapingState(ctx)
|
||||
if err == nil && existing.ID != "" {
|
||||
state.ID = existing.ID
|
||||
state.CreatedAt = existing.CreatedAt
|
||||
return r.stateOrmRepo.Update(ctx, state)
|
||||
}
|
||||
|
||||
// Create new state if none exists
|
||||
return r.stateOrmRepo.Create(ctx, state)
|
||||
}
|
||||
|
||||
// CreateScrapingJob creates a new scraping job
|
||||
func (r *tenderRepository) CreateScrapingJob(ctx context.Context, job *ScrapingJob) error {
|
||||
if job.ID == "" {
|
||||
job.ID = primitive.NewObjectID().Hex()
|
||||
}
|
||||
|
||||
now := time.Now().Unix()
|
||||
job.CreatedAt = now
|
||||
job.UpdatedAt = now
|
||||
|
||||
err := r.jobOrmRepo.Create(ctx, job)
|
||||
if err != nil {
|
||||
r.logger.Error("Failed to create scraping job", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"type": job.Type,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
r.logger.Info("Scraping job created successfully", map[string]interface{}{
|
||||
"job_id": job.ID,
|
||||
"type": job.Type,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetScrapingJob retrieves a scraping job by ID
|
||||
func (r *tenderRepository) GetScrapingJob(ctx context.Context, id string) (*ScrapingJob, error) {
|
||||
job, err := r.jobOrmRepo.FindByID(ctx, id)
|
||||
if err != nil {
|
||||
r.logger.Error("Failed to get scraping job", map[string]interface{}{
|
||||
"job_id": id,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return job, nil
|
||||
}
|
||||
|
||||
// UpdateScrapingJob updates a scraping job
|
||||
func (r *tenderRepository) UpdateScrapingJob(ctx context.Context, job *ScrapingJob) error {
|
||||
job.UpdatedAt = time.Now().Unix()
|
||||
|
||||
err := r.jobOrmRepo.Update(ctx, job)
|
||||
if err != nil {
|
||||
r.logger.Error("Failed to update scraping job", map[string]interface{}{
|
||||
"job_id": job.ID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
r.logger.Info("Scraping job updated successfully", map[string]interface{}{
|
||||
"job_id": job.ID,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListScrapingJobs retrieves scraping jobs with pagination
|
||||
func (r *tenderRepository) ListScrapingJobs(ctx context.Context, limit, offset int) ([]ScrapingJob, int64, error) {
|
||||
filter := bson.M{}
|
||||
|
||||
pagination := mongopkg.Pagination{
|
||||
Limit: limit,
|
||||
Skip: offset,
|
||||
SortField: "started_at",
|
||||
SortOrder: -1,
|
||||
}
|
||||
|
||||
result, err := r.jobOrmRepo.FindAll(ctx, filter, pagination)
|
||||
if err != nil {
|
||||
r.logger.Error("Failed to list scraping jobs", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
return result.Items, result.TotalCount, nil
|
||||
}
|
||||
|
||||
// GetActiveScrapingJobs retrieves currently active scraping jobs
|
||||
func (r *tenderRepository) GetActiveScrapingJobs(ctx context.Context) ([]ScrapingJob, error) {
|
||||
filter := bson.M{
|
||||
"status": bson.M{
|
||||
"$in": []ScrapingJobStatus{
|
||||
ScrapingJobStatusPending,
|
||||
ScrapingJobStatusRunning,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pagination := mongopkg.Pagination{
|
||||
Limit: 100,
|
||||
SortField: "started_at",
|
||||
SortOrder: -1,
|
||||
}
|
||||
|
||||
result, err := r.jobOrmRepo.FindAll(ctx, filter, pagination)
|
||||
if err != nil {
|
||||
r.logger.Error("Failed to get active scraping jobs", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result.Items, nil
|
||||
}
|
||||
|
||||
// buildSearchFilter builds MongoDB filter from search criteria
|
||||
func (r *tenderRepository) buildSearchFilter(criteria TenderSearchCriteria) bson.M {
|
||||
filter := bson.M{}
|
||||
|
||||
Reference in New Issue
Block a user