Enhance Scraper Dockerfile and Notice Entity

- Added an entrypoint script to the scraper Dockerfile to handle one-time scraping with date range support.
- Introduced a new ContentXML field in the Notice entity to store XML content, improving data structure for notices.
- Updated the scraper logic to map XML content to the tender entity, enhancing data processing capabilities.

cmd/scraper/Dockerfile

@@ -17,6 +17,16 @@ WORKDIR /app
 
 COPY --from=go-builder /app/scraper-build .
 
+# Create entrypoint script
+RUN echo '#!/bin/bash' > /app/entrypoint.sh && \
+    echo 'set -e' >> /app/entrypoint.sh && \
+    echo 'if [ "$SCRAPER_ONETIME" = "true" ] && [ -n "$FROM_DATE" ] && [ -n "$TO_DATE" ]; then' >> /app/entrypoint.sh && \
+    echo '  exec /app/scraper-build -onetime -from "$FROM_DATE" -to "$TO_DATE"' >> /app/entrypoint.sh && \
+    echo 'else' >> /app/entrypoint.sh && \
+    echo '  exec /app/scraper-build' >> /app/entrypoint.sh && \
+    echo 'fi' >> /app/entrypoint.sh && \
+    chmod +x /app/entrypoint.sh
+
 EXPOSE 80
 
-ENTRYPOINT /app/scraper-build
+ENTRYPOINT ["/app/entrypoint.sh"]

internal/notice/entity.go

@@ -52,6 +52,7 @@ type Notice struct {
 	SourceFileURL             string               `bson:"source_file_url" json:"source_file_url"`
 	SourceFileName            string               `bson:"source_file_name" json:"source_file_name"`
 	ProcessingMetadata        ProcessingMetadata   `bson:"processing_metadata" json:"processing_metadata"`
+	ContentXML                string               `bson:"content_xml" json:"content_xml"`
 
 	// Status-specific fields
 	CancellationReason string               `bson:"cancellation_reason,omitempty" json:"cancellation_reason,omitempty"`

ted/scraper.go

@@ -311,38 +311,9 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 
 	// Map to tender entity
 	t := s.mapToTenderFromParsedDoc(notice, "", fileName)
+	t.ContentXML = string(content)
 
-	// existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID)
-	// if err != nil {
-	// 	s.logger.Warn("Error checking for existing tender", map[string]interface{}{
-	// 		"contract_notice_id": t.ContractNoticeID,
-	// 		"error":              err.Error(),
-	// 	})
-	// }
-
-	// if err == nil && existingTender != nil {
-	// 	// Update existing tender
-	// 	s.logger.Info("Found existing tender, updating", map[string]interface{}{
-	// 		"existing_tender_id": existingTender.ID.Hex(),
-	// 		"contract_notice_id": t.ContractNoticeID,
-	// 	})
-
-	// 	t.ID = existingTender.ID
-	// 	t.UpdatedAt = time.Now().Unix()
-	// 	err = s.tenderRepo.Update(ctx, t)
-	// 	if err != nil {
-	// 		s.logger.Error("Failed to update existing tender", map[string]interface{}{
-	// 			"tender_id": t.ID.Hex(),
-	// 			"error":     err.Error(),
-	// 		})
-	// 		return fmt.Errorf("failed to update existing tender: %w", err)
-	// 	}
-
-	// 	s.logger.Info("Successfully updated tender", map[string]interface{}{
-	// 		"tender_id": t.ID.Hex(),
-	// 	})
-	// } else {
-	// 	// Create new tender
+	// Create new tender
 	s.logger.Info("Creating new tender", map[string]interface{}{
 		"contract_notice_id": t.ContractNoticeID,
 	})