Add translated notices scanning functionality to dashboard repository
continuous-integration/drone/push Build is passing
continuous-integration/drone/push Build is passing
- Introduced a new `translatedNoticesScanner` interface for scanning MinIO for daily translated notice counts. - Enhanced the `repository` struct to include fields for managing translated notice scope and caching. - Implemented the `ScanTranslatedNotices` method in the `StorageClient` to retrieve daily counts and total from MinIO. - Updated the `Statistics` method in the repository to utilize the new translated notices scope, improving data accuracy. - Added unit tests for the translated notices functionality, ensuring robust validation of the new features. This update significantly enhances the dashboard's ability to handle translated notice statistics, improving overall data management and reporting capabilities.
This commit is contained in:
@@ -389,3 +389,25 @@ func (t *TenderJSON) translationForLanguage(language string) (StoredTranslation,
|
||||
}
|
||||
return t.storedTranslationText(language)
|
||||
}
|
||||
|
||||
// hasAnyTranslation reports whether tender.json contains at least one non-empty
|
||||
// title translation (via translation_status.done or the translations map).
|
||||
func (t *TenderJSON) hasAnyTranslation() bool {
|
||||
if t == nil {
|
||||
return false
|
||||
}
|
||||
for _, lang := range t.translationsDone() {
|
||||
if _, ok := t.storedTranslationText(lang); ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if t.Translations == nil {
|
||||
return false
|
||||
}
|
||||
for _, entry := range t.Translations {
|
||||
if strings.TrimSpace(entry.Title) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -654,6 +654,89 @@ func (s *StorageClient) ScanScrapedDocuments(ctx context.Context) ([]ProcedureDo
|
||||
return results, dailyCounts, nil
|
||||
}
|
||||
|
||||
// ScanTranslatedNotices scans MinIO for tender.json objects with stored translations
|
||||
// and returns daily notice counts keyed by UTC date (YYYY-MM-DD) plus the total.
|
||||
func (s *StorageClient) ScanTranslatedNotices(ctx context.Context) (map[string]int64, int64, error) {
|
||||
s.logger.Debug("Scanning translated notices from MinIO", map[string]interface{}{
|
||||
"bucket": s.config.MinioBucket,
|
||||
"prefix": procedurePrefix,
|
||||
})
|
||||
|
||||
dailyCounts := make(map[string]int64)
|
||||
var total int64
|
||||
seen := make(map[string]struct{})
|
||||
var fatalErr error
|
||||
|
||||
appendFromPrefix := func(prefix string) bool {
|
||||
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
|
||||
Prefix: prefix,
|
||||
Recursive: true,
|
||||
})
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
fatalErr = s.logMinIOFailure("list_translated_notices", object.Err, map[string]interface{}{
|
||||
"prefix": prefix,
|
||||
})
|
||||
return false
|
||||
}
|
||||
key := object.Key
|
||||
if !strings.HasSuffix(key, "/tender.json") {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[key]; dup {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
|
||||
tenderJSON, err := s.readTenderJSONAtKey(ctx, key)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrObjectNotFound) {
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, ErrMinIOUnavailable) {
|
||||
fatalErr = err
|
||||
return false
|
||||
}
|
||||
s.logger.Debug("Skipping tender.json during translation scan", map[string]interface{}{
|
||||
"object_key": key,
|
||||
"error": err.Error(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if !tenderJSON.hasAnyTranslation() {
|
||||
continue
|
||||
}
|
||||
|
||||
total++
|
||||
modified := normalizeUnixSeconds(object.LastModified.Unix())
|
||||
if modified > 0 {
|
||||
date := time.Unix(modified, 0).UTC().Format("2006-01-02")
|
||||
dailyCounts[date]++
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
if !appendFromPrefix(procedurePrefix) {
|
||||
return nil, 0, fatalErr
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
if !appendFromPrefix("") {
|
||||
return nil, 0, fatalErr
|
||||
}
|
||||
}
|
||||
if fatalErr != nil {
|
||||
return nil, 0, fatalErr
|
||||
}
|
||||
|
||||
s.logger.Info("Scanned translated notices from storage", map[string]interface{}{
|
||||
"bucket": s.config.MinioBucket,
|
||||
"total_count": total,
|
||||
})
|
||||
|
||||
return dailyCounts, total, nil
|
||||
}
|
||||
|
||||
func normalizeUnixSeconds(ts int64) int64 {
|
||||
if ts > 1_000_000_000_000 {
|
||||
return ts / 1000
|
||||
|
||||
Reference in New Issue
Block a user