Added missing fields - scraper and worker fixes

ted/notice_upsert.go

@@ -0,0 +1,80 @@
+package ted
+
+import (
+	"strconv"
+	"strings"
+	"time"
+	"tm/internal/notice"
+)
+
+// noticeNeedsRefresh reports whether incoming scraped data should replace an existing notice row.
+func noticeNeedsRefresh(existing, incoming *notice.Notice) bool {
+	if incoming == nil || existing == nil {
+		return true
+	}
+	if strings.TrimSpace(incoming.ContentXML) != "" &&
+		strings.TrimSpace(existing.ContentXML) != strings.TrimSpace(incoming.ContentXML) {
+		return true
+	}
+	if normalizePublicationID(existing.NoticePublicationID) != normalizePublicationID(incoming.NoticePublicationID) {
+		return true
+	}
+	if compareNoticeVersions(incoming.NoticeVersion, existing.NoticeVersion) > 0 {
+		return true
+	}
+	return false
+}
+
+func normalizePublicationID(s string) string {
+	return strings.TrimSpace(s)
+}
+
+// compareNoticeVersions compares BT-757 notice versions (e.g. "01", "02"). Empty parses as 0.
+func compareNoticeVersions(incoming, existing string) int {
+	vi := parseNoticeVersion(incoming)
+	ve := parseNoticeVersion(existing)
+	switch {
+	case vi > ve:
+		return 1
+	case vi < ve:
+		return -1
+	default:
+		return 0
+	}
+}
+
+func parseNoticeVersion(s string) int {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return 0
+	}
+	n, err := strconv.Atoi(s)
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+// MergeNoticeUpdate replaces scrape-derived fields from src into dst while preserving identity,
+// creation time, and optional enrichment metadata already stored on dst.
+func MergeNoticeUpdate(dst *notice.Notice, src *notice.Notice) {
+	preservedID := dst.ID
+	preservedCreated := dst.CreatedAt
+	oldEnrichment := dst.ProcessingMetadata.EnrichmentData
+	oldTranslated := dst.ProcessingMetadata.TranslatedData
+	oldTranslatedAt := dst.ProcessingMetadata.TranslatedAt
+
+	*dst = *src
+	dst.ID = preservedID
+	dst.CreatedAt = preservedCreated
+	dst.UpdatedAt = time.Now().Unix()
+	dst.ProcessingMetadata.Processed = false
+
+	if len(dst.ProcessingMetadata.EnrichmentData) == 0 && len(oldEnrichment) > 0 {
+		dst.ProcessingMetadata.EnrichmentData = oldEnrichment
+	}
+	if len(dst.ProcessingMetadata.TranslatedData) == 0 && len(oldTranslated) > 0 {
+		dst.ProcessingMetadata.TranslatedData = oldTranslated
+		dst.ProcessingMetadata.TranslatedAt = oldTranslatedAt
+	}
+}