Added missing fields - scraper and worker fixes

internal/notice/repository.go

@@ -16,6 +16,7 @@ type Repository interface {
 	BulkImport(ctx context.Context, notices []Notice) error
 	GetByID(ctx context.Context, id string) (*Notice, error)
 	GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error)
+	FindNoticesWithContentXML(ctx context.Context, limit, skip int) ([]Notice, error)
 	GetUnProcessedNotices(ctx context.Context, ProcessingLimit int, skip int) ([]Notice, int64, error)
 	GetProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error)
 	Delete(ctx context.Context, id string) error
@@ -39,6 +40,7 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
 		// processing_metadata.processed
 		*orm.NewIndex("processing_metadata_processed_idx", bson.D{{Key: "processing_metadata.processed", Value: 1}}),
 		*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
+		*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
 	}
 
 	// Create indexes
@@ -144,6 +146,27 @@ func (r *noticeRepository) GetByContractNoticeID(ctx context.Context, contractNo
 	return &result.Items[0], nil
 }
 
+// FindNoticesWithContentXML returns notices that have non-empty stored TED XML (for backfill / remapping).
+func (r *noticeRepository) FindNoticesWithContentXML(ctx context.Context, limit, skip int) ([]Notice, error) {
+	filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}
+	pagination := orm.Pagination{
+		Limit:     limit,
+		Skip:      skip,
+		SortField: "_id",
+		SortOrder: 1,
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to list notices with content_xml", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, err
+	}
+
+	return result.Items, nil
+}
+
 // GetUnProcessedNotices retrieves unprocessed notices
 
 func (r *noticeRepository) GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) {