diff --git a/cmd/scraper/Dockerfile b/cmd/scraper/Dockerfile index a4c7e33..f0a961f 100644 --- a/cmd/scraper/Dockerfile +++ b/cmd/scraper/Dockerfile @@ -17,6 +17,16 @@ WORKDIR /app COPY --from=go-builder /app/scraper-build . +# Create entrypoint script +RUN echo '#!/bin/bash' > /app/entrypoint.sh && \ + echo 'set -e' >> /app/entrypoint.sh && \ + echo 'if [ "$SCRAPER_ONETIME" = "true" ] && [ -n "$FROM_DATE" ] && [ -n "$TO_DATE" ]; then' >> /app/entrypoint.sh && \ + echo ' exec /app/scraper-build -onetime -from "$FROM_DATE" -to "$TO_DATE"' >> /app/entrypoint.sh && \ + echo 'else' >> /app/entrypoint.sh && \ + echo ' exec /app/scraper-build' >> /app/entrypoint.sh && \ + echo 'fi' >> /app/entrypoint.sh && \ + chmod +x /app/entrypoint.sh + EXPOSE 80 -ENTRYPOINT /app/scraper-build +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/internal/notice/entity.go b/internal/notice/entity.go index 104aa5d..0b917f3 100644 --- a/internal/notice/entity.go +++ b/internal/notice/entity.go @@ -52,6 +52,7 @@ type Notice struct { SourceFileURL string `bson:"source_file_url" json:"source_file_url"` SourceFileName string `bson:"source_file_name" json:"source_file_name"` ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"` + ContentXML string `bson:"content_xml" json:"content_xml"` // Status-specific fields CancellationReason string `bson:"cancellation_reason,omitempty" json:"cancellation_reason,omitempty"` diff --git a/ted/scraper.go b/ted/scraper.go index 425d365..403fd72 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -311,38 +311,9 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file // Map to tender entity t := s.mapToTenderFromParsedDoc(notice, "", fileName) + t.ContentXML = string(content) - // existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID) - // if err != nil { - // s.logger.Warn("Error checking for existing tender", map[string]interface{}{ - // "contract_notice_id": t.ContractNoticeID, - // "error": err.Error(), - // }) - // } - - // if err == nil && existingTender != nil { - // // Update existing tender - // s.logger.Info("Found existing tender, updating", map[string]interface{}{ - // "existing_tender_id": existingTender.ID.Hex(), - // "contract_notice_id": t.ContractNoticeID, - // }) - - // t.ID = existingTender.ID - // t.UpdatedAt = time.Now().Unix() - // err = s.tenderRepo.Update(ctx, t) - // if err != nil { - // s.logger.Error("Failed to update existing tender", map[string]interface{}{ - // "tender_id": t.ID.Hex(), - // "error": err.Error(), - // }) - // return fmt.Errorf("failed to update existing tender: %w", err) - // } - - // s.logger.Info("Successfully updated tender", map[string]interface{}{ - // "tender_id": t.ID.Hex(), - // }) - // } else { - // // Create new tender + // Create new tender s.logger.Info("Creating new tender", map[string]interface{}{ "contract_notice_id": t.ContractNoticeID, })