Merge branch 'develop' into Phone-Uniqueness

internal/dashboard/entity.go

@@ -85,3 +85,24 @@ type RecentItem struct {
 	CreatedAt       int64  `json:"created_at"`
 	PublicationDate int64  `json:"publication_date"`
 }
+
+// StatisticsReportResponse powers the statistics and charts report.
+type StatisticsReportResponse struct {
+	Days        int                      `json:"days"`
+	GeneratedAt int64                    `json:"generated_at"`
+	Daily       StatisticsDailySeries    `json:"daily"`
+	Totals      StatisticsLifetimeTotals `json:"totals"`
+}
+
+// StatisticsDailySeries contains per-day chart series.
+type StatisticsDailySeries struct {
+	ScrapedTED       []TrendPoint `json:"scraped_ted"`
+	ScrapedDocuments []TrendPoint `json:"scraped_documents"`
+	TranslatedNotices []TrendPoint `json:"translated_notices"`
+}
+
+// StatisticsLifetimeTotals contains all-time totals.
+type StatisticsLifetimeTotals struct {
+	ScrapedDocuments  int64 `json:"scraped_documents"`
+	TranslatedTenders int64 `json:"translated_tenders"`
+}

internal/dashboard/form.go

@@ -27,3 +27,8 @@ type RecentQuery struct {
 	Limit  int    `query:"limit"`
 	Cursor string `query:"cursor"`
 }
+
+// StatisticsQuery binds query params for GET /dashboard/statistics.
+type StatisticsQuery struct {
+	Days int `query:"days"`
+}

internal/dashboard/handler.go

@@ -161,6 +161,30 @@ func (h *Handler) Recent(c echo.Context) error {
 	return response.Success(c, out, "Recent tenders retrieved successfully")
 }
 
+// Statistics returns scraping and translation statistics for charts.
+// @Summary Dashboard statistics report
+// @Description Daily and lifetime statistics for TED scraping, document scraping, and AI translation
+// @Tags Admin-Dashboard
+// @Produce json
+// @Param days query int false "Days back for daily series (default 14, max 90)"
+// @Success 200 {object} response.APIResponse{data=StatisticsReportResponse}
+// @Failure 500 {object} response.APIResponse
+// @Router /admin/v1/dashboard/statistics [get]
+// @Security BearerAuth
+func (h *Handler) Statistics(c echo.Context) error {
+	query := StatisticsQuery{
+		Days: parseIntQuery(c, "days", 0),
+	}
+
+	out, err := h.service.Statistics(c.Request().Context(), query)
+	if err != nil {
+		return response.InternalServerError(c, "Failed to load dashboard statistics")
+	}
+
+	setPrivateCache(c, 60)
+	return response.Success(c, out, "Dashboard statistics retrieved successfully")
+}
+
 func parseIntQuery(c echo.Context, name string, fallback int) int {
 	raw := c.QueryParam(name)
 	if raw == "" {

internal/dashboard/repository.go

@@ -23,18 +23,23 @@ type Repository interface {
 	NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
 	ClosingSoon(ctx context.Context, limit int, windowSec int64) ([]ClosingSoonItem, error)
 	Recent(ctx context.Context, limit int, cursor string) ([]RecentItem, *string, error)
+	Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error)
 }
 
 type repository struct {
-	ormRepo orm.Repository[tender.Tender]
-	logger  logger.Logger
+	ormRepo        orm.Repository[tender.Tender]
+	mongoManager   *orm.ConnectionManager
+	metricsCounter *orm.Counter
+	logger         logger.Logger
 }
 
 // NewRepository creates a dashboard repository backed by the tenders collection.
 func NewRepository(mongoManager *orm.ConnectionManager, log logger.Logger) Repository {
 	return &repository{
-		ormRepo: orm.NewRepository[tender.Tender](mongoManager.GetCollection(tenderCollectionName()), log),
-		logger:  log,
+		ormRepo:        orm.NewRepository[tender.Tender](mongoManager.GetCollection(tenderCollectionName()), log),
+		mongoManager:   mongoManager,
+		metricsCounter: orm.NewCounter(mongoManager),
+		logger:         log,
 	}
 }
 

internal/dashboard/service.go

@@ -25,6 +25,7 @@ type Service interface {
 	NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
 	ClosingSoon(ctx context.Context, query ClosingSoonQuery) (*ClosingSoonResponse, error)
 	Recent(ctx context.Context, query RecentQuery) (*RecentResponse, error)
+	Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error)
 }
 
 type service struct {
@@ -157,6 +158,25 @@ func (s *service) Recent(ctx context.Context, query RecentQuery) (*RecentRespons
 	}, nil
 }
 
+func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error) {
+	days := trendDays(query.Days)
+	startUnix := trendStartUnix(days)
+
+	s.logger.Info("Fetching dashboard statistics report", map[string]interface{}{
+		"days": days,
+	})
+
+	out, err := s.repo.Statistics(ctx, days, startUnix)
+	if err != nil {
+		s.logger.Error("Failed to fetch dashboard statistics report", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("dashboard statistics: %w", err)
+	}
+
+	return out, nil
+}
+
 func closingWindowHours(hours int) int {
 	if hours <= 0 {
 		return defaultClosingWindowHours

internal/dashboard/statistics_repository.go

@@ -0,0 +1,189 @@
+package dashboard
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"tm/internal/notice"
+	orm "tm/pkg/mongo"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+	mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
+)
+
+const noticesCollectionName = "notices"
+
+func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) {
+	now := time.Now().UTC()
+	endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
+	startDay := endDay.AddDate(0, 0, -(days - 1))
+
+	scrapedTED, err := r.dailyCountByTimestamp(ctx, noticesCollectionName, bson.M{
+		"source": notice.TenderSourceTEDScraper,
+		"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
+	}, "processing_metadata.scraped_at")
+	if err != nil {
+		return nil, fmt.Errorf("scraped ted per day: %w", err)
+	}
+
+	scrapedDocuments, err := r.scrapedDocumentsPerDay(ctx, startUnix)
+	if err != nil {
+		return nil, fmt.Errorf("scraped documents per day: %w", err)
+	}
+
+	translatedNotices, err := r.translatedNoticesPerDay(ctx, startDay, endDay)
+	if err != nil {
+		return nil, fmt.Errorf("translated notices per day: %w", err)
+	}
+
+	totalDocuments, err := r.totalScrapedDocuments(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("total scraped documents: %w", err)
+	}
+
+	totalTranslated, err := r.metricsCounter.Get(ctx, orm.AITranslationSuccessCounterKey)
+	if err != nil {
+		return nil, fmt.Errorf("total translated tenders: %w", err)
+	}
+
+	return &StatisticsReportResponse{
+		Days:        days,
+		GeneratedAt: now.Unix(),
+		Daily: StatisticsDailySeries{
+			ScrapedTED:        fillTrendSeries(days, scrapedTED),
+			ScrapedDocuments:  fillTrendSeries(days, scrapedDocuments),
+			TranslatedNotices: fillTrendSeries(days, translatedNotices),
+		},
+		Totals: StatisticsLifetimeTotals{
+			ScrapedDocuments:  totalDocuments,
+			TranslatedTenders: totalTranslated,
+		},
+	}, nil
+}
+
+func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) {
+	pipeline := mongodriver.Pipeline{
+		{{Key: "$match", Value: match}},
+		{{Key: "$addFields", Value: bson.M{
+			"metric_ts": normalizeTimestampExpr(timestampField),
+		}}},
+		{{Key: "$group", Value: bson.M{
+			"_id": bson.M{
+				"$dateToString": bson.M{
+					"format":   "%Y-%m-%d",
+					"date":     bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
+					"timezone": "UTC",
+				},
+			},
+			"count": bson.M{"$sum": 1},
+		}}},
+	}
+
+	cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline)
+	if err != nil {
+		return nil, err
+	}
+	defer cursor.Close(ctx)
+
+	return decodeDailyCounts(ctx, cursor)
+}
+
+func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) {
+	pipeline := mongodriver.Pipeline{
+		{{Key: "$match", Value: bson.M{
+			"processing_metadata.documents_scraped": true,
+			"scraped_documents":                   bson.M{"$exists": true, "$ne": bson.A{}},
+		}}},
+		{{Key: "$unwind", Value: "$scraped_documents"}},
+		{{Key: "$addFields", Value: bson.M{
+			"metric_ts": bson.M{
+				"$cond": bson.A{
+					bson.M{"$gt": bson.A{"$scraped_documents.scraped_at", 0}},
+					"$scraped_documents.scraped_at",
+					"$processing_metadata.documents_scraped_at",
+				},
+			},
+		}}},
+		{{Key: "$match", Value: bson.M{
+			"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
+		}}},
+		{{Key: "$addFields", Value: bson.M{
+			"metric_ts": normalizeTimestampExpr("metric_ts"),
+		}}},
+		{{Key: "$group", Value: bson.M{
+			"_id": bson.M{
+				"$dateToString": bson.M{
+					"format":   "%Y-%m-%d",
+					"date":     bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
+					"timezone": "UTC",
+				},
+			},
+			"count": bson.M{"$sum": 1},
+		}}},
+	}
+
+	cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
+	if err != nil {
+		return nil, err
+	}
+	defer cursor.Close(ctx)
+
+	return decodeDailyCounts(ctx, cursor)
+}
+
+func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
+	return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay)
+}
+
+func (r *repository) totalScrapedDocuments(ctx context.Context) (int64, error) {
+	pipeline := mongodriver.Pipeline{
+		{{Key: "$match", Value: bson.M{
+			"processing_metadata.documents_scraped": true,
+			"scraped_documents":                   bson.M{"$exists": true, "$ne": bson.A{}},
+		}}},
+		{{Key: "$project", Value: bson.M{
+			"doc_count": bson.M{"$size": "$scraped_documents"},
+		}}},
+		{{Key: "$group", Value: bson.M{
+			"_id":   nil,
+			"total": bson.M{"$sum": "$doc_count"},
+		}}},
+	}
+
+	cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
+	if err != nil {
+		return 0, err
+	}
+	defer cursor.Close(ctx)
+
+	if !cursor.Next(ctx) {
+		return 0, nil
+	}
+
+	var row bson.M
+	if err := cursor.Decode(&row); err != nil {
+		return 0, err
+	}
+
+	return aggregateCount(row, "total"), nil
+}
+
+func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) {
+	counts := make(map[string]int64)
+	for cursor.Next(ctx) {
+		var row bson.M
+		if err := cursor.Decode(&row); err != nil {
+			return nil, err
+		}
+		date, ok := row["_id"].(string)
+		if !ok || date == "" {
+			continue
+		}
+		counts[date] = aggregateCount(row, "count")
+	}
+	if err := cursor.Err(); err != nil {
+		return nil, err
+	}
+	return counts, nil
+}

internal/notice/repository.go

@@ -43,7 +43,7 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
 		*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
 		// One row per TED ContractNoticeID when present (parallel scrape races).
 		*orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}).
-			WithPartialFilterExpression(bson.M{"contract_notice_id": bson.M{"$type": "string", "$ne": ""}}),
+			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_notice_id")),
 	}
 
 	// Create indexes

internal/tender/repository.go

@@ -94,16 +94,16 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
 		*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
 		// One tender row per TED ContractNoticeID when present (parallel ingest / lookup gaps).
 		*orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}).
-			WithPartialFilterExpression(bson.M{"contract_notice_id": bson.M{"$type": "string", "$ne": ""}}),
+			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_notice_id")),
 		*orm.NewIndex("contract_folder_id_idx", bson.D{{Key: "contract_folder_id", Value: 1}}),
 		// One tender row per TED publication / procedure when ids are present (prevents parallel worker races).
 		*orm.CreateUniqueIndex("notice_publication_id_unique", bson.D{{Key: "notice_publication_id", Value: 1}}).
-			WithPartialFilterExpression(bson.M{"notice_publication_id": bson.M{"$type": "string", "$ne": ""}}),
+			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("notice_publication_id")),
 		*orm.CreateUniqueIndex("contract_folder_id_unique", bson.D{{Key: "contract_folder_id", Value: 1}}).
-			WithPartialFilterExpression(bson.M{"contract_folder_id": bson.M{"$type": "string", "$ne": ""}}),
+			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_folder_id")),
 		// One tender per UBL procurement project (merges per-lot notices that share ProcurementProject/ID).
 		*orm.CreateUniqueIndex("procurement_project_id_unique", bson.D{{Key: "procurement_project_id", Value: 1}}).
-			WithPartialFilterExpression(bson.M{"procurement_project_id": bson.M{"$type": "string", "$ne": ""}}),
+			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("procurement_project_id")),
 		*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
 		*orm.NewIndex("source_idx", bson.D{{Key: "source", Value: 1}}),
 		*orm.NewIndex("country_code_idx", bson.D{{Key: "country_code", Value: 1}}),

internal/tender/service.go

@@ -1476,7 +1476,7 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str
 		return nil, fmt.Errorf("tender has no contract folder ID")
 	}
 
-	title, description, err := s.resolveTranslation(ctx, t, targetLanguage)
+	title, description, fromAPI, err := s.resolveTranslation(ctx, t, targetLanguage, ai_summarizer.TranslationRequestSourceManualTrigger)
 	if err != nil {
 		s.logger.Error("AI translation failed", map[string]interface{}{
 			"tender_id": id,
@@ -1487,6 +1487,14 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str
 		return nil, fmt.Errorf("AI translation request failed: %w", err)
 	}
 
+	if fromAPI {
+		s.logger.Info("Manual AI translation request succeeded", map[string]interface{}{
+			"tender_id": id,
+			"notice_id": t.NoticePublicationID,
+			"language":  targetLanguage,
+		})
+	}
+
 	return s.buildAITranslateResponse(t, targetLanguage, title, description), nil
 }
 
@@ -1500,11 +1508,11 @@ func (s *tenderService) buildAITranslateResponse(t *Tender, language, title, des
 	}
 }
 
-func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, language string) (string, string, error) {
+func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, language, requestSource string) (string, string, bool, error) {
 	if s.aiSummarizerStorage != nil {
 		stored, err := s.aiSummarizerStorage.GetTranslationFromStorage(ctx, t.ContractFolderID, t.NoticePublicationID, language)
 		if err == nil {
-			return stored.Title, stored.Description, nil
+			return stored.Title, stored.Description, false, nil
 		}
 		if !errors.Is(err, ai_summarizer.ErrTranslationNotReady) &&
 			!errors.Is(err, ai_summarizer.ErrObjectNotFound) &&
@@ -1523,11 +1531,12 @@ func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, langu
 		Title:               t.Title,
 		Description:         t.Description,
 		Language:            language,
+		RequestSource:       requestSource,
 	})
 	if err != nil {
-		return "", "", err
+		return "", "", false, err
 	}
-	return resp.TranslatedTitle, resp.TranslatedDescription, nil
+	return resp.TranslatedTitle, resp.TranslatedDescription, true, nil
 }
 
 func (s *tenderService) pickResponseLanguage(language *string) string {