Mazyar
45cfa24a72
- Updated the document scraper service to include a new ScrapePortalsProvider interface, allowing for dynamic retrieval of supported scraping portals. - Modified the ListPendingTenders and GetTenderByNoticeID methods to filter tenders based on document URLs that match the configured portals. - Introduced new error handling for cases when the scrape portals provider is not configured, returning appropriate service unavailable responses. - Enhanced API documentation to reflect changes in tender retrieval logic and added error response details for unsupported portal scenarios. This update improves the document scraping functionality by integrating AI portal support, enhancing the overall reliability and flexibility of the tender management system.
34 lines
748 B
Go
34 lines
748 B
Go
package document_scraper
|
|
|
|
import (
|
|
"strings"
|
|
|
|
"tm/internal/tender"
|
|
)
|
|
|
|
// documentURL returns the URL used for portal matching and the scraper API payload.
|
|
func documentURL(t *tender.Tender) string {
|
|
if t == nil {
|
|
return ""
|
|
}
|
|
if u := strings.TrimSpace(t.DocumentURI); u != "" {
|
|
return u
|
|
}
|
|
return strings.TrimSpace(t.TenderURL)
|
|
}
|
|
|
|
// matchesScrapePortals reports whether url contains any supported portal identifier.
|
|
func matchesScrapePortals(url string, portals []string) bool {
|
|
if url == "" || len(portals) == 0 {
|
|
return false
|
|
}
|
|
urlLower := strings.ToLower(url)
|
|
for _, portal := range portals {
|
|
p := strings.TrimSpace(portal)
|
|
if p != "" && strings.Contains(urlLower, strings.ToLower(p)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|