Add translated notices scanning functionality to dashboard repository
continuous-integration/drone/push Build is passing

- Introduced a new `translatedNoticesScanner` interface for scanning MinIO for daily translated notice counts.
- Enhanced the `repository` struct to include fields for managing translated notice scope and caching.
- Implemented the `ScanTranslatedNotices` method in the `StorageClient` to retrieve daily counts and total from MinIO.
- Updated the `Statistics` method in the repository to utilize the new translated notices scope, improving data accuracy.
- Added unit tests for the translated notices functionality, ensuring robust validation of the new features.

This update significantly enhances the dashboard's ability to handle translated notice statistics, improving overall data management and reporting capabilities.
This commit is contained in:
Mazyar
2026-07-10 17:05:38 +03:30
parent 843e1508df
commit 1df44ec8ca
5 changed files with 260 additions and 11 deletions
+22
View File
@@ -389,3 +389,25 @@ func (t *TenderJSON) translationForLanguage(language string) (StoredTranslation,
}
return t.storedTranslationText(language)
}
// hasAnyTranslation reports whether tender.json contains at least one non-empty
// title translation (via translation_status.done or the translations map).
func (t *TenderJSON) hasAnyTranslation() bool {
if t == nil {
return false
}
for _, lang := range t.translationsDone() {
if _, ok := t.storedTranslationText(lang); ok {
return true
}
}
if t.Translations == nil {
return false
}
for _, entry := range t.Translations {
if strings.TrimSpace(entry.Title) != "" {
return true
}
}
return false
}
+83
View File
@@ -654,6 +654,89 @@ func (s *StorageClient) ScanScrapedDocuments(ctx context.Context) ([]ProcedureDo
return results, dailyCounts, nil
}
// ScanTranslatedNotices scans MinIO for tender.json objects with stored translations
// and returns daily notice counts keyed by UTC date (YYYY-MM-DD) plus the total.
func (s *StorageClient) ScanTranslatedNotices(ctx context.Context) (map[string]int64, int64, error) {
s.logger.Debug("Scanning translated notices from MinIO", map[string]interface{}{
"bucket": s.config.MinioBucket,
"prefix": procedurePrefix,
})
dailyCounts := make(map[string]int64)
var total int64
seen := make(map[string]struct{})
var fatalErr error
appendFromPrefix := func(prefix string) bool {
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: true,
})
for object := range objectCh {
if object.Err != nil {
fatalErr = s.logMinIOFailure("list_translated_notices", object.Err, map[string]interface{}{
"prefix": prefix,
})
return false
}
key := object.Key
if !strings.HasSuffix(key, "/tender.json") {
continue
}
if _, dup := seen[key]; dup {
continue
}
seen[key] = struct{}{}
tenderJSON, err := s.readTenderJSONAtKey(ctx, key)
if err != nil {
if errors.Is(err, ErrObjectNotFound) {
continue
}
if errors.Is(err, ErrMinIOUnavailable) {
fatalErr = err
return false
}
s.logger.Debug("Skipping tender.json during translation scan", map[string]interface{}{
"object_key": key,
"error": err.Error(),
})
continue
}
if !tenderJSON.hasAnyTranslation() {
continue
}
total++
modified := normalizeUnixSeconds(object.LastModified.Unix())
if modified > 0 {
date := time.Unix(modified, 0).UTC().Format("2006-01-02")
dailyCounts[date]++
}
}
return true
}
if !appendFromPrefix(procedurePrefix) {
return nil, 0, fatalErr
}
if len(seen) == 0 {
if !appendFromPrefix("") {
return nil, 0, fatalErr
}
}
if fatalErr != nil {
return nil, 0, fatalErr
}
s.logger.Info("Scanned translated notices from storage", map[string]interface{}{
"bucket": s.config.MinioBucket,
"total_count": total,
})
return dailyCounts, total, nil
}
func normalizeUnixSeconds(ts int64) int64 {
if ts > 1_000_000_000_000 {
return ts / 1000