Implemented the integration with scraper python server, tenders summarise, and some fixes.

pkg/config/config.go

@@ -95,6 +95,21 @@ type HCaptchaConfig struct {
 	Timeout   time.Duration `env:"HCAPTCHA_TIMEOUT"`
 }
 
+type ScraperConfig struct {
+	BaseURL string        `env:"SCRAPER_BASE_URL"`
+	Timeout time.Duration `env:"SCRAPER_TIMEOUT"`
+}
+
+type MinIOConfig struct {
+	Endpoint           string `env:"MINIO_ENDPOINT" default:"localhost:9000"`
+	AccessKeyID        string `env:"MINIO_ACCESS_KEY_ID" default:""`
+	SecretAccessKey    string `env:"MINIO_SECRET_ACCESS_KEY" default:""`
+	UseSSL             bool   `env:"MINIO_USE_SSL" default:"false"`
+	Region             string `env:"MINIO_REGION" default:"us-east-1"`
+	DefaultBucket      string `env:"MINIO_DEFAULT_BUCKET" default:"files"`
+	HierarchicalBucket string `env:"MINIO_HIERARCHICAL_BUCKET" default:"documents"`
+}
+
 // LoadConfig loads configuration with priority: OS environment variables > .env file
 // OS environment variables take precedence over .env file values
 func LoadConfig[T any](path string, config T) (T, error) {

pkg/minio/connection.go

@@ -3,6 +3,7 @@ package minio
 import (
 	"context"
 	"net/http"
+	"time"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -36,6 +37,8 @@ func (cm *ConnectionManager) Connect() error {
 		Region: cm.config.Region,
 		Transport: &http.Transport{
 			ResponseHeaderTimeout: cm.config.RequestTimeout,
+			TLSHandshakeTimeout:   10 * time.Second,
+			DisableKeepAlives:     false,
 		},
 	})
 	if err != nil {
@@ -48,14 +51,16 @@ func (cm *ConnectionManager) Connect() error {
 
 	cm.client = minioClient
 
-	// Test connection by listing buckets
-	ctx, cancel := context.WithTimeout(context.Background(), cm.config.ConnectTimeout)
+	// Test connection by checking if we can reach the server
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	_, err = cm.client.ListBuckets(ctx)
+	// Try bucket exists check with shorter timeout
+	exists, err := cm.client.BucketExists(ctx, cm.config.DefaultBucket)
 	if err != nil {
 		cm.logger.Error("Failed to connect to MinIO", map[string]interface{}{
 			"endpoint": cm.config.Endpoint,
+			"bucket":   cm.config.DefaultBucket,
 			"error":    err.Error(),
 		})
 		return NewErrorWithCause(ErrCodeConnectionFailed, "failed to connect to MinIO", err)
@@ -64,8 +69,9 @@ func (cm *ConnectionManager) Connect() error {
 	cm.logger.Info("Successfully connected to MinIO", map[string]interface{}{
 		"endpoint": cm.config.Endpoint,
 		"region":   cm.config.Region,
+		"bucket":   cm.config.DefaultBucket,
+		"exists":   exists,
 	})
-
 	return nil
 }
 
@@ -240,7 +246,7 @@ func (cm *ConnectionManager) GetBucketInfo(ctx context.Context, bucketName strin
 	// For now, return basic bucket info
 	// In a real implementation, you might want to get more detailed info
 	info := map[string]interface{}{
-		"name": bucketName,
+		"name":   bucketName,
 		"region": cm.config.Region,
 	}
 

pkg/scraper/sdk.go

@@ -71,23 +71,16 @@ type Client struct {
 	httpClient *http.Client
 	baseURL    string
 	logger     logger.Logger
-	// Stored credentials
-	username string
-	password string
-	loginURL string
 }
 
 // NewClient creates a new scraper SDK client with stored credentials
-func NewClient(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) SDK {
+func NewClient(baseURL string, timeout time.Duration, logger logger.Logger) SDK {
 	return &Client{
 		httpClient: &http.Client{
 			Timeout: timeout,
 		},
-		baseURL:  baseURL,
-		logger:   logger,
-		username: username,
-		password: password,
-		loginURL: loginURL,
+		baseURL: baseURL,
+		logger:  logger,
 	}
 }
 
@@ -101,9 +94,6 @@ func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*Scra
 	// Prepare request body with stored credentials + dynamic fields
 	requestBody := map[string]interface{}{
 		"notice_id":  req.NoticeID,
-		"username":   c.username,
-		"password":   c.password,
-		"login_url":  c.loginURL,
 		"notice_url": req.NoticeURL,
 	}