mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
50c018af627daff9416fe0afdcb7400a9ccd4595
tm_back/pkg/minio/USAGE.md
T
Nima Nakhostin 1a56070556 Add MinIO Package for Object Storage Management 
- Introduced a comprehensive MinIO package for managing object storage, including support for hierarchical document storage and general file management.
- Implemented core components such as Config, ConnectionManager, and Service, following Clean Architecture principles.
- Added functionality for file uploads, downloads, bucket management, and structured logging, enhancing usability and maintainability.
- Included detailed error handling and validation for configuration and operations, ensuring robustness.
- Updated documentation with usage examples and configuration guidelines to facilitate integration and understanding of the package's functionality.
2025-11-09 09:52:09 +03:30

12 KiB
Raw Blame History

MinIO Package Usage Examples

Basic Setup

package main

import (
    "context"
    "log"
    "path/to/pkg/minio"
)

// Implement the Logger interface
type SimpleLogger struct{}

func (l *SimpleLogger) Info(message string, fields map[string]interface{}) {
    log.Printf("[INFO] %s: %+v", message, fields)
}

func (l *SimpleLogger) Error(message string, fields map[string]interface{}) {
    log.Printf("[ERROR] %s: %+v", message, fields)
}

func (l *SimpleLogger) Debug(message string, fields map[string]interface{}) {
    log.Printf("[DEBUG] %s: %+v", message, fields)
}

func (l *SimpleLogger) Warn(message string, fields map[string]interface{}) {
    log.Printf("[WARN] %s: %+v", message, fields)
}

func main() {
    // Create configuration
    config := minio.Config{
        Endpoint:        "localhost:9000",
        AccessKeyID:     "minioadmin",
        SecretAccessKey: "minioadmin",
        UseSSL:          false,
        DefaultBucket:   "files",
        HierarchicalBucket: "documents",
    }

    // Create logger
    logger := &SimpleLogger{}

    // Create connection manager
    connManager, err := minio.NewConnectionManager(config, logger)
    if err != nil {
        log.Fatal("Failed to create connection manager:", err)
    }

    // Connect to MinIO
    if err := connManager.Connect(); err != nil {
        log.Fatal("Failed to connect to MinIO:", err)
    }
    defer connManager.Disconnect(context.Background())

    // Create service
    service := minio.NewService(connManager, config, logger)

    // Use the service...
}

Normal File Storage

Upload an Image

func uploadImage(service *minio.Service) error {
    // Open image file
    file, err := os.Open("profile.jpg")
    if err != nil {
        return err
    }
    defer file.Close()

    // Upload to normal storage
    return service.Upload(context.Background(), "files", "images/profile.jpg", file, minio.UploadOptions{
        ContentType: "image/jpeg",
        Metadata: map[string]string{
            "user_id": "12345",
            "type":    "profile_image",
        },
    })
}

Download and Serve a File

func serveFile(service *minio.Service, w http.ResponseWriter, bucket, objectName string) error {
    // Download file
    reader, err := service.Download(context.Background(), bucket, objectName, minio.DownloadOptions{})
    if err != nil {
        return err
    }
    defer reader.Close()

    // Get file info for headers
    info, err := service.GetInfo(context.Background(), bucket, objectName)
    if err != nil {
        return err
    }

    // Set response headers
    w.Header().Set("Content-Type", info.ContentType)
    w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))

    // Copy file content to response
    _, err = io.Copy(w, reader)
    return err
}

Upload with Stream Processing

func uploadWithProgress(service *minio.Service, file io.Reader, filename string) error {
    // Create a progress reader
    progressReader := &ProgressReader{
        Reader:   file,
        Total:    0, // Set total size if known
        Reporter: func(bytesRead int64) {
            log.Printf("Uploaded %d bytes", bytesRead)
        },
    }

    return service.Upload(context.Background(), "files", "uploads/"+filename, progressReader, minio.UploadOptions{
        ContentType: mime.TypeByExtension(filepath.Ext(filename)),
    })
}

type ProgressReader struct {
    Reader   io.Reader
    Total    int64
    Reporter func(bytesRead int64)
}

func (pr *ProgressReader) Read(p []byte) (int, error) {
    n, err := pr.Reader.Read(p)
    if n > 0 && pr.Reporter != nil {
        pr.Total += int64(n)
        pr.Reporter(pr.Total)
    }
    return n, err
}

Hierarchical Document Storage

Upload a Contract

func uploadContract(service *minio.Service) error {
    // Open contract file
    file, err := os.Open("contract.pdf")
    if err != nil {
        return err
    }
    defer file.Close()

    // Upload to hierarchical storage
    docPath, err := service.UploadDocument(context.Background(), file, "service_contract.pdf", minio.HierarchicalUploadOptions{
        UploadOptions: minio.UploadOptions{
            ContentType: "application/pdf",
            Metadata: map[string]string{
                "contract_id": "CTR-2024-001",
                "client_id":   "CLIENT-123",
                "value":       "50000.00",
                "signed":      "true",
                "department":  "legal",
            },
        },
        Category:    "contracts",
        SubCategory: "2024",
        Tags:        []string{"legal", "service", "active"},
    })

    if err != nil {
        return err
    }

    log.Printf("Contract uploaded to: %s", docPath)
    // Output: contracts/2024/legal_service_active/1640995200/service_contract.pdf

    return nil
}

List Documents by Category

func listContractsByYear(service *minio.Service, year string) error {
    result, err := service.ListDocuments(context.Background(), "contracts", year, minio.ListOptions{
        MaxKeys: 50,
    })
    if err != nil {
        return err
    }

    log.Printf("Found %d documents in contracts/%s", len(result.Objects), year)

    for _, doc := range result.Objects {
        log.Printf("- %s (%d bytes)", doc.Name, doc.Size)
    }

    return nil
}

Search Documents

func searchLegalDocuments(service *minio.Service) error {
    result, err := service.SearchDocuments(context.Background(), "legal", "contracts", "2024", minio.ListOptions{
        MaxKeys: 20,
    })
    if err != nil {
        return err
    }

    log.Printf("Found %d legal documents", len(result.Objects))

    return nil
}

File Operations

Copy Files Between Buckets

func backupFile(service *minio.Service, srcBucket, srcObject, destBucket, destObject string) error {
    return service.Copy(context.Background(), srcBucket, srcObject, destBucket, destObject)
}

Generate Presigned URLs

func generateDownloadLink(service *minio.Service, bucket, objectName string) (string, error) {
    // Generate URL valid for 1 hour
    return service.GetPresignedURL(context.Background(), bucket, objectName, 3600)
}

Check File Existence

func fileExists(service *minio.Service, bucket, objectName string) bool {
    exists, err := service.Exists(context.Background(), bucket, objectName)
    if err != nil {
        log.Printf("Error checking file existence: %v", err)
        return false
    }
    return exists
}

Bucket Management

Create Required Buckets

func ensureBucketsExist(connManager *minio.ConnectionManager) error {
    ctx := context.Background()

    // Create default bucket
    if err := connManager.CreateBucket(ctx, "files"); err != nil {
        return fmt.Errorf("failed to create files bucket: %w", err)
    }

    // Create hierarchical bucket
    if err := connManager.CreateBucket(ctx, "documents"); err != nil {
        return fmt.Errorf("failed to create documents bucket: %w", err)
    }

    return nil
}

List All Buckets

func listAllBuckets(connManager *minio.ConnectionManager) error {
    buckets, err := connManager.ListBuckets(context.Background())
    if err != nil {
        return err
    }

    log.Printf("Available buckets:")
    for _, bucket := range buckets {
        log.Printf("- %s", bucket)
    }

    return nil
}

Error Handling

func handleUploadError(err error) {
    switch {
    case errors.Is(err, minio.ErrInvalidConfig):
        log.Println("Configuration error:", err)
    case errors.Is(err, minio.ErrNotConnected):
        log.Println("Connection error:", err)
    case errors.Is(err, minio.ErrUploadFailed):
        log.Println("Upload failed:", err)
    case errors.Is(err, minio.ErrFileTooLarge):
        log.Println("File too large:", err)
    case errors.Is(err, minio.ErrInvalidFileType):
        log.Println("Invalid file type:", err)
    default:
        log.Println("Unknown error:", err)
    }
}

Integration with HTTP Handlers

File Upload Handler

func uploadHandler(service *minio.Service) gin.HandlerFunc {
    return func(c *gin.Context) {
        // Get uploaded file
        file, err := c.FormFile("file")
        if err != nil {
            c.JSON(http.StatusBadRequest, gin.H{"error": "No file provided"})
            return
        }

        // Open uploaded file
        src, err := file.Open()
        if err != nil {
            c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to open file"})
            return
        }
        defer src.Close()

        // Generate unique filename
        filename := fmt.Sprintf("%d_%s", time.Now().Unix(), file.Filename)
        objectName := fmt.Sprintf("uploads/%s", filename)

        // Upload file
        err = service.Upload(c.Request.Context(), "files", objectName, src, minio.UploadOptions{
            ContentType: file.Header.Get("Content-Type"),
            Metadata: map[string]string{
                "original_name": file.Filename,
                "uploaded_by": "user_id", // Get from context
            },
        })

        if err != nil {
            handleUploadError(err)
            c.JSON(http.StatusInternalServerError, gin.H{"error": "Upload failed"})
            return
        }

        c.JSON(http.StatusOK, gin.H{
            "message": "File uploaded successfully",
            "filename": filename,
            "path": objectName,
        })
    }
}

Document Upload Handler

func uploadDocumentHandler(service *minio.Service) gin.HandlerFunc {
    return func(c *gin.Context) {
        // Get form data
        category := c.PostForm("category")
        subCategory := c.PostForm("subcategory")
        tags := c.PostFormArray("tags[]")

        // Get uploaded file
        file, err := c.FormFile("document")
        if err != nil {
            c.JSON(http.StatusBadRequest, gin.H{"error": "No document provided"})
            return
        }

        // Open document
        src, err := file.Open()
        if err != nil {
            c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to open document"})
            return
        }
        defer src.Close()

        // Upload to hierarchical storage
        docPath, err := service.UploadDocument(c.Request.Context(), src, file.Filename, minio.HierarchicalUploadOptions{
            UploadOptions: minio.UploadOptions{
                ContentType: file.Header.Get("Content-Type"),
                Metadata: map[string]string{
                    "uploaded_by": "user_id", // Get from context
                    "department":  c.PostForm("department"),
                },
            },
            Category:    category,
            SubCategory: subCategory,
            Tags:        tags,
        })

        if err != nil {
            handleUploadError(err)
            c.JSON(http.StatusInternalServerError, gin.H{"error": "Document upload failed"})
            return
        }

        c.JSON(http.StatusOK, gin.H{
            "message": "Document uploaded successfully",
            "path": docPath,
        })
    }
}

Configuration Examples

Development Configuration

config := minio.Config{
    Endpoint:        "localhost:9000",
    AccessKeyID:     "minioadmin",
    SecretAccessKey: "minioadmin",
    UseSSL:          false,
    Region:          "us-east-1",
    DefaultBucket:   "files",
    HierarchicalBucket: "documents",
    MaxFileSize:     50 * 1024 * 1024, // 50MB
    AllowedTypes:    []string{"image/", "application/pdf", "text/"},
}

Production Configuration

config := minio.Config{
    Endpoint:        "minio.example.com:443",
    AccessKeyID:     os.Getenv("MINIO_ACCESS_KEY"),
    SecretAccessKey: os.Getenv("MINIO_SECRET_KEY"),
    UseSSL:          true,
    Region:          "us-west-2",
    DefaultBucket:   "production-files",
    HierarchicalBucket: "production-documents",
    MaxFileSize:     100 * 1024 * 1024, // 100MB
    AllowedTypes:    []string{"image/jpeg", "image/png", "application/pdf"},
    ConnectTimeout:  30 * time.Second,
    RequestTimeout:  60 * time.Second,
}
Reference in New Issue View Git Blame Copy Permalink