Implemented the integration with scraper python server, tenders summarise, and some fixes.

internal/tender/repository.go

@@ -23,6 +23,8 @@ type TenderRepository interface {
 	GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
 	Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) ([]Tender, int64, error)
 	GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
+	GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
+	GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
 	Update(ctx context.Context, tender *Tender) error
 	Delete(ctx context.Context, id string) error
 	GetStatistics(ctx context.Context) (*TenderStatistics, error)
@@ -275,6 +277,66 @@ func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int
 	return result.Items, nil
 }
 
+// GetUnScrapedTenders retrieves tenders that haven't been processed for document scraping
+func (r *tenderRepository) GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
+	filter := bson.M{
+		"$or": []bson.M{
+			{"processing_metadata.documents_scraped": bson.M{"$exists": false}},
+			{"processing_metadata.documents_scraped": false},
+			{"processing_metadata.documents_scraped": nil},
+		},
+	}
+
+	pagination := orm.Pagination{
+		Limit:     limit,
+		Skip:      skip,
+		SortField: "created_at",
+		SortOrder: 1, // Oldest first
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to get un-scraped tenders", map[string]interface{}{
+			"limit": limit,
+			"skip":  skip,
+			"error": err.Error(),
+		})
+		return nil, 0, err
+	}
+
+	return result.Items, result.TotalCount, nil
+}
+
+// GetUnSummarizedTenders retrieves tenders that have documents scraped but not summarized
+func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
+	filter := bson.M{
+		"processing_metadata.documents_scraped": true,
+		"$or": []bson.M{
+			{"processing_metadata.documents_summarized": bson.M{"$exists": false}},
+			{"processing_metadata.documents_summarized": false},
+		},
+	}
+
+	pagination := orm.Pagination{
+		Limit:     limit,
+		Skip:      skip,
+		SortField: "processing_metadata.documents_scraped_at",
+		SortOrder: 1, // Oldest first (process oldest scraped documents first)
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
+			"limit": limit,
+			"skip":  skip,
+			"error": err.Error(),
+		})
+		return nil, 0, err
+	}
+
+	return result.Items, result.TotalCount, nil
+}
+
 // GetStatistics calculates tender statistics using aggregation
 func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
 	stats := &TenderStatistics{