Added missing fields - scraper and worker fixes

ted/scraper.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"compress/gzip"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -12,7 +13,7 @@ import (
 	"time"
 	"tm/internal/notice"
 	"tm/pkg/logger"
-	"tm/pkg/mongo"
+	orm "tm/pkg/mongo"
 	"tm/pkg/notification"
 )
 
@@ -38,16 +39,16 @@ type TEDScraper struct {
 	logger       logger.Logger
 	httpClient   *http.Client
 	xmlParser    *TEDParser
-	mongoManager *mongo.ConnectionManager
-	tenderRepo   notice.Repository
+	mongoManager *orm.ConnectionManager
+	noticeRepo   notice.Repository
 }
 
 // NewTEDScraper creates a new TED scraper instance
 func NewTEDScraper(
 	config *Config,
 	logger logger.Logger,
-	mongoManager *mongo.ConnectionManager,
-	tenderRepo notice.Repository,
+	mongoManager *orm.ConnectionManager,
+	noticeRepo notice.Repository,
 	notify notification.SDK,
 ) *TEDScraper {
 	httpClient := &http.Client{
@@ -61,7 +62,7 @@ func NewTEDScraper(
 		httpClient:   httpClient,
 		xmlParser:    NewTEDParser(),
 		mongoManager: mongoManager,
-		tenderRepo:   tenderRepo,
+		noticeRepo:   noticeRepo,
 	}
 }
 
@@ -289,16 +290,18 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
 	return result, nil
 }
 
-func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) {
-	existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID)
+func (s *TEDScraper) findNoticeByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) {
+	if strings.TrimSpace(contractNoticeID) == "" {
+		return nil, nil
+	}
+	existing, err := s.noticeRepo.GetByContractNoticeID(ctx, contractNoticeID)
 	if err != nil {
-		if err.Error() == "document not found" {
+		if errors.Is(err, orm.ErrDocumentNotFound) {
 			return nil, nil
 		}
 		return nil, err
 	}
-
-	return existingTender, nil
+	return existing, nil
 }
 
 func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, result *FileProcessingResult) error {
@@ -331,45 +334,66 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 	}
 	t.ContentXML = string(content)
 
-	// Check if tender already exists to prevent duplicates
-	existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID)
+	existingNotice, err := s.findNoticeByContractNoticeID(ctx, t.ContractNoticeID)
 	if err != nil {
-		s.logger.Error("Failed to check existing tender", map[string]interface{}{
+		s.logger.Error("Failed to check existing notice", map[string]interface{}{
 			"contract_notice_id": t.ContractNoticeID,
 			"error":              err.Error(),
 		})
-		return fmt.Errorf("failed to check existing tender: %w", err)
+		return fmt.Errorf("failed to check existing notice: %w", err)
 	}
 
-	if existingTender != nil {
-		s.logger.Info("Tender already exists, skipping duplicate", map[string]interface{}{
-			"contract_notice_id": t.ContractNoticeID,
-			"existing_id":        existingTender.ID,
+	if existingNotice != nil {
+		if !noticeNeedsRefresh(existingNotice, t) {
+			s.logger.Debug("Notice unchanged, skipping", map[string]interface{}{
+				"contract_notice_id": t.ContractNoticeID,
+				"notice_doc_id":      existingNotice.ID.Hex(),
+			})
+			return nil
+		}
+
+		MergeNoticeUpdate(existingNotice, t)
+		err = s.noticeRepo.Update(ctx, existingNotice)
+		if err != nil {
+			s.logger.Error("Failed to update existing notice", map[string]interface{}{
+				"contract_notice_id": t.ContractNoticeID,
+				"notice_doc_id":      existingNotice.ID.Hex(),
+				"error":              err.Error(),
+			})
+			return fmt.Errorf("failed to update notice: %w", err)
+		}
+
+		s.logger.Info("Notice updated (refresh)", map[string]interface{}{
+			"contract_notice_id":      t.ContractNoticeID,
+			"notice_publication_id":   t.NoticePublicationID,
+			"notice_version":          t.NoticeVersion,
+			"notice_doc_id":           existingNotice.ID.Hex(),
+			"processing_reset_worker": true,
 		})
-		return nil
-	}
-
-	// Create new tender
-	s.logger.Info("Creating new tender", map[string]interface{}{
-		"contract_notice_id": t.ContractNoticeID,
-	})
-
-	t.CreatedAt = time.Now().Unix()
-	t.UpdatedAt = t.CreatedAt
-
-	err = s.tenderRepo.Import(ctx, t)
-	if err != nil {
-		s.logger.Error("Failed to create new tender", map[string]interface{}{
+		result.SuccessCount++
+	} else {
+		s.logger.Info("Creating new notice", map[string]interface{}{
 			"contract_notice_id": t.ContractNoticeID,
-			"error":              err.Error(),
 		})
-		return fmt.Errorf("failed to create tender: %w", err)
-	}
 
-	s.logger.Info("Successfully created new tender", map[string]interface{}{
-		"contract_notice_id": t.ContractNoticeID,
-	})
-	// }
+		now := time.Now().Unix()
+		t.CreatedAt = now
+		t.UpdatedAt = now
+
+		err = s.noticeRepo.Import(ctx, t)
+		if err != nil {
+			s.logger.Error("Failed to import notice", map[string]interface{}{
+				"contract_notice_id": t.ContractNoticeID,
+				"error":              err.Error(),
+			})
+			return fmt.Errorf("failed to import notice: %w", err)
+		}
+
+		s.logger.Info("Successfully imported new notice", map[string]interface{}{
+			"contract_notice_id": t.ContractNoticeID,
+		})
+		result.SuccessCount++
+	}
 
 	// Log detailed parsed information based on notice type
 	s.logger.Info("Successfully parsed notice", map[string]interface{}{
@@ -379,7 +403,6 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 		"tender_url":  t.TenderURL,
 	})
 
-	result.SuccessCount++
 	return nil
 }