diff --git a/internal/dashboard/entity.go b/internal/dashboard/entity.go index 670107b..4b975b0 100644 --- a/internal/dashboard/entity.go +++ b/internal/dashboard/entity.go @@ -105,4 +105,5 @@ type StatisticsDailySeries struct { type StatisticsLifetimeTotals struct { ScrapedDocuments int64 `json:"scraped_documents"` TranslatedTenders int64 `json:"translated_tenders"` + ScrapedTEDNotices int64 `json:"scraped_ted_notices"` } diff --git a/internal/dashboard/statistics_repository.go b/internal/dashboard/statistics_repository.go index 8ba1d4f..6039283 100644 --- a/internal/dashboard/statistics_repository.go +++ b/internal/dashboard/statistics_repository.go @@ -6,7 +6,6 @@ import ( "sync" "time" - "tm/internal/notice" "tm/pkg/ai_summarizer" orm "tm/pkg/mongo" @@ -14,8 +13,6 @@ import ( mongodriver "go.mongodb.org/mongo-driver/v2/mongo" ) -const noticesCollectionName = "notices" - // scrapedScopeCacheTTL avoids listing every MinIO object on each statistics request while // keeping counts aligned with the tender panel (MinIO is the source of truth for scraped docs). const scrapedScopeCacheTTL = 15 * time.Minute @@ -49,6 +46,7 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) translatedNotices map[string]int64 totalDocuments int64 totalTranslated int64 + totalScrapedTED int64 scrapedScope scrapedTendersScope ) @@ -70,10 +68,10 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout) defer cancel() - counts, err := r.dailyCountByTimestamp(qctx, noticesCollectionName, bson.M{ - "source": notice.TenderSourceTEDScraper, - "created_at": bson.M{"$gte": startUnix, "$gt": 0}, - }, "created_at") + // Read from the persistent metrics counter rather than the "notices" collection: the + // notice worker deletes processed notices shortly after promoting them into tenders, so + // counting from "notices" directly would undercount (often to zero) recent scrape activity. + counts, err := r.metricsCounter.GetTEDNoticeScrapedDailyCounts(qctx, startDay, endDay) if err != nil { recordFailure("scraped_ted", err) scrapedTED = map[string]int64{} @@ -82,6 +80,20 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) scrapedTED = counts }() + wg.Add(1) + go func() { + defer wg.Done() + qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout) + defer cancel() + + total, err := r.metricsCounter.Get(qctx, orm.TEDNoticeScrapedCounterKey) + if err != nil { + recordFailure("scraped_ted_total", err) + return + } + totalScrapedTED = total + }() + wg.Add(1) go func() { defer wg.Done() @@ -167,37 +179,11 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) Totals: StatisticsLifetimeTotals{ ScrapedDocuments: totalDocuments, TranslatedTenders: totalTranslated, + ScrapedTEDNotices: totalScrapedTED, }, }, nil } -func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) { - pipeline := mongodriver.Pipeline{ - {{Key: "$match", Value: match}}, - {{Key: "$addFields", Value: bson.M{ - "metric_ts": normalizeTimestampExpr(timestampField), - }}}, - {{Key: "$group", Value: bson.M{ - "_id": bson.M{ - "$dateToString": bson.M{ - "format": "%Y-%m-%d", - "date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}}, - "timezone": "UTC", - }, - }, - "count": bson.M{"$sum": 1}, - }}}, - } - - cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline) - if err != nil { - return nil, err - } - defer cursor.Close(ctx) - - return decodeDailyCounts(ctx, cursor) -} - func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64, scope scrapedTendersScope) (map[string]int64, error) { if scope.zero { return map[string]int64{}, nil diff --git a/pkg/mongo/counter.go b/pkg/mongo/counter.go index 9d92e23..f4856cc 100644 --- a/pkg/mongo/counter.go +++ b/pkg/mongo/counter.go @@ -16,6 +16,11 @@ const ( AITranslationSuccessCounterKey = "ai_translation_successful_requests" AITranslationSuccessCounterKeyDailyJob = "ai_translation_successful_requests_daily_job" AITranslationSuccessCounterKeyManualTrigger = "ai_translation_successful_requests_manual_trigger" + // TEDNoticeScrapedCounterKey tracks TED notices successfully scraped/ingested. This is + // recorded independently of the "notices" collection because that collection is a transient + // processing queue: the worker deletes notices once they've been promoted into tenders, so + // counting from "notices" directly undercounts (or zeroes out) historical scrape activity. + TEDNoticeScrapedCounterKey = "ted_notice_scraped" ) // Counter provides atomic increment/read operations for named metrics stored in MongoDB. @@ -96,8 +101,36 @@ func AITranslationSuccessDailyCounterKey(day time.Time) string { return AITranslationSuccessCounterKey + "_day:" + day.UTC().Format("2006-01-02") } -// GetDailyCounts returns counter values for each UTC day from start through end inclusive. +// TEDNoticeScrapedDailyCounterKey returns the metrics key for TED notices scraped on a given UTC day. +func TEDNoticeScrapedDailyCounterKey(day time.Time) string { + return TEDNoticeScrapedCounterKey + "_day:" + day.UTC().Format("2006-01-02") +} + +// IncrementTEDNoticeScraped atomically increments the lifetime and today's daily counters for +// successfully scraped/ingested TED notices. Call this once per newly created notice document +// (not on updates/refreshes) so totals reflect distinct scrape events. +func (c *Counter) IncrementTEDNoticeScraped(ctx context.Context) error { + if _, err := c.Increment(ctx, TEDNoticeScrapedCounterKey); err != nil { + return err + } + _, err := c.Increment(ctx, TEDNoticeScrapedDailyCounterKey(time.Now().UTC())) + return err +} + +// GetDailyCounts returns counter values for each UTC day from start through end inclusive, for +// the AI translation daily counter. func (c *Counter) GetDailyCounts(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) { + return c.getDailyCounts(ctx, startDay, endDay, AITranslationSuccessDailyCounterKey) +} + +// GetTEDNoticeScrapedDailyCounts returns counter values for each UTC day from start through end +// inclusive, for the TED-notice-scraped daily counter. +func (c *Counter) GetTEDNoticeScrapedDailyCounts(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) { + return c.getDailyCounts(ctx, startDay, endDay, TEDNoticeScrapedDailyCounterKey) +} + +// getDailyCounts is the shared implementation behind the per-metric GetXDailyCounts helpers above. +func (c *Counter) getDailyCounts(ctx context.Context, startDay, endDay time.Time, dailyKey func(time.Time) string) (map[string]int64, error) { startDay = time.Date(startDay.Year(), startDay.Month(), startDay.Day(), 0, 0, 0, 0, time.UTC) endDay = time.Date(endDay.Year(), endDay.Month(), endDay.Day(), 0, 0, 0, 0, time.UTC) if endDay.Before(startDay) { @@ -108,7 +141,7 @@ func (c *Counter) GetDailyCounts(ctx context.Context, startDay, endDay time.Time dates := make([]string, 0) for day := startDay; !day.After(endDay); day = day.AddDate(0, 0, 1) { dates = append(dates, day.Format("2006-01-02")) - keys = append(keys, AITranslationSuccessDailyCounterKey(day)) + keys = append(keys, dailyKey(day)) } counts := make(map[string]int64, len(dates)) diff --git a/ted/scraper.go b/ted/scraper.go index 4224b4e..9dbdd21 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -34,13 +34,14 @@ type Config struct { // TEDScraper handles downloading and parsing TED XML files type TEDScraper struct { - notify notification.SDK - config *Config - logger logger.Logger - httpClient *http.Client - xmlParser *TEDParser - mongoManager *orm.ConnectionManager - noticeRepo notice.Repository + notify notification.SDK + config *Config + logger logger.Logger + httpClient *http.Client + xmlParser *TEDParser + mongoManager *orm.ConnectionManager + noticeRepo notice.Repository + metricsCounter *orm.Counter } // NewTEDScraper creates a new TED scraper instance @@ -56,13 +57,14 @@ func NewTEDScraper( } return &TEDScraper{ - notify: notify, - config: config, - logger: logger, - httpClient: httpClient, - xmlParser: NewTEDParser(), - mongoManager: mongoManager, - noticeRepo: noticeRepo, + notify: notify, + config: config, + logger: logger, + httpClient: httpClient, + xmlParser: NewTEDParser(), + mongoManager: mongoManager, + noticeRepo: noticeRepo, + metricsCounter: orm.NewCounter(mongoManager), } } @@ -420,6 +422,18 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file return fmt.Errorf("failed to import notice: %w", err) } + // Track scraping activity in a persistent counter (independent of the "notices" + // collection) since the worker deletes processed notices, which would otherwise make + // scrape history unrecoverable for dashboard reporting. + if s.metricsCounter != nil { + if incErr := s.metricsCounter.IncrementTEDNoticeScraped(ctx); incErr != nil { + s.logger.Warn("Failed to increment TED scraped notice counter", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + "error": incErr.Error(), + }) + } + } + s.logger.Info("Successfully imported new notice", map[string]interface{}{ "contract_notice_id": t.ContractNoticeID, })