mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance document scraper service with AI portal integration and error handling

Browse Source 
- Updated the document scraper service to include a new ScrapePortalsProvider interface, allowing for dynamic retrieval of supported scraping portals.
- Modified the ListPendingTenders and GetTenderByNoticeID methods to filter tenders based on document URLs that match the configured portals.
- Introduced new error handling for cases when the scrape portals provider is not configured, returning appropriate service unavailable responses.
- Enhanced API documentation to reflect changes in tender retrieval logic and added error response details for unsupported portal scenarios.

This update improves the document scraping functionality by integrating AI portal support, enhancing the overall reliability and flexibility of the tender management system.
This commit is contained in:
Mazyar
2026-06-21 09:58:11 +03:30
parent 550f11a77e
commit 45cfa24a72
7 changed files with 143 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/document_scraper/handler.go
+10 -2
View File
@@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler {
// ListPendingTenders retrieves tenders that need document scraping
// @Summary List pending tenders for document scraping
// @Description Retrieve tenders that have not yet been scraped for documents. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with active deadlines are returned; set include_expired=true to include expired tenders.
// @Description Retrieve tenders that have not yet been scraped for documents and whose document URL matches a portal supported by the AI scraping service. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with active deadlines are returned; set include_expired=true to include expired tenders.
// @Tags Document-Scraper
// @Produce json
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
@@ -35,6 +35,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler {
// @Success 200 {object} response.APIResponse{data=DocumentScraperListResponse} "List of pending tenders"
// @Failure 400 {object} response.APIResponse "Invalid query parameters"
// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
// @Failure 503 {object} response.APIResponse "Scrape portals service is not configured"
// @Failure 500 {object} response.APIResponse "Internal server error"
// @Security ApiKeyAuth
// @Router /api/v1/scraper/tenders [get]
@@ -49,6 +50,9 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
if errors.Is(err, ErrInvalidDateRange) {
return response.BadRequest(c, "Invalid date range", err.Error())
}
if errors.Is(err, ErrScrapePortalsUnavailable) {
return response.ServiceUnavailable(c, "Scrape portals service is not configured")
}
return response.InternalServerError(c, "Failed to list pending tenders")
}
@@ -57,7 +61,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
// @Summary Get tender by notice ID for document scraping
// @Description Retrieve a specific tender by its notice publication ID (contract_folder_id, notice_publication_id, document_url, title, description).
// @Description Retrieve a specific unexpired tender whose document URL matches a supported scrape portal.
// @Tags Document-Scraper
// @Produce json
// @Param notice_id path string true "Notice Publication ID"
@@ -65,6 +69,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
// @Failure 400 {object} response.APIResponse "Invalid notice ID"
// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
// @Failure 404 {object} response.APIResponse "Tender not found"
// @Failure 503 {object} response.APIResponse "Scrape portals service is not configured"
// @Failure 500 {object} response.APIResponse "Internal server error"
// @Security ApiKeyAuth
// @Router /api/v1/scraper/tenders/{notice_id} [get]
@@ -76,6 +81,9 @@ func (h *Handler) GetTenderByNoticeID(c echo.Context) error {
tender, err := h.service.GetTenderByNoticeID(c.Request().Context(), noticeID)
if err != nil {
if errors.Is(err, ErrScrapePortalsUnavailable) {
return response.ServiceUnavailable(c, "Scrape portals service is not configured")
}
if err.Error() == "tender not found" {
return response.NotFound(c, "Tender not found")
}