tm_back/pkg/minio

MinIO Storage Package

A comprehensive Go package for MinIO object storage with support for hierarchical document storage and general file management. This package provides both normal file storage for images, videos, and other media files, as well as classified hierarchical storage for documents.

Features

✅ Hierarchical Document Storage - Organize documents in categorized folders with metadata ✅ Normal File Storage - Standard object storage for images, videos, and other files ✅ File Upload/Download - Stream-based operations with progress tracking ✅ Metadata Management - Rich metadata support for classification and search ✅ Presigned URLs - Temporary access URLs for secure file sharing ✅ Bucket Management - Create, delete, and manage storage buckets ✅ File Operations - Copy, move, delete, and list files ✅ Connection Management - Robust connection pooling and health checks ✅ Structured Logging - Comprehensive logging with structured fields ✅ Error Handling - Detailed error types and context ✅ Type Safety - Full Go generics support

Installation

go get github.com/minio/minio-go/v7

Quick Start

1. Configuration

import "path/to/pkg/minio"

// Create configuration
config := minio.Config{
    Endpoint:        "localhost:9000",
    AccessKeyID:     "your-access-key",
    SecretAccessKey: "your-secret-key",
    UseSSL:          false,
    Region:          "us-east-1",
    DefaultBucket:   "files",
    HierarchicalBucket: "documents",
}

// Validate configuration
if err := config.Validate(); err != nil {
    log.Fatal(err)
}

2. Initialize Service

// Create logger (implement minio.Logger interface)
logger := &YourLogger{}

// Create connection manager
connManager, err := minio.NewConnectionManager(config, logger)
if err != nil {
    log.Fatal(err)
}

// Connect to MinIO
if err := connManager.Connect(); err != nil {
    log.Fatal(err)
}
defer connManager.Disconnect(context.Background())

// Create service
service := minio.NewService(connManager, config, logger)

3. Normal File Storage

// Upload a file
file, err := os.Open("image.jpg")
if err != nil {
    log.Fatal(err)
}
defer file.Close()

err = service.Upload(context.Background(), "files", "images/profile.jpg", file, minio.UploadOptions{
    ContentType: "image/jpeg",
    Metadata: map[string]string{
        "user_id": "123",
        "type": "profile",
    },
})

// Download a file
reader, err := service.Download(context.Background(), "files", "images/profile.jpg", minio.DownloadOptions{})
if err != nil {
    log.Fatal(err)
}
defer reader.Close()

// Copy file data to response
io.Copy(w, reader)

4. Hierarchical Document Storage

// Upload a document with classification
docFile, err := os.Open("contract.pdf")
if err != nil {
    log.Fatal(err)
}
defer docFile.Close()

docPath, err := service.UploadDocument(context.Background(), docFile, "contract.pdf", minio.HierarchicalUploadOptions{
    UploadOptions: minio.UploadOptions{
        ContentType: "application/pdf",
        Metadata: map[string]string{
            "title": "Service Contract",
            "version": "1.0",
        },
    },
    Category:    "contracts",
    SubCategory: "2024",
    Tags:        []string{"legal", "service"},
})

fmt.Printf("Document uploaded to: %s\n", docPath)
// Output: contracts/2024/legal_service/1640995200/contract.pdf

Storage Types

Normal Storage

• Use Case: Images, videos, general files
• Structure: Flat object storage with custom paths
• Example: images/profile.jpg, videos/tutorial.mp4

Hierarchical Storage

• Use Case: Documents requiring classification and organization
• Structure: Categorized folder structure with metadata
• Example: contracts/2024/legal_service/1640995200/contract.pdf

API Reference

File Operations

Upload File

func (s *Service) Upload(ctx context.Context, bucket, objectName string, reader io.Reader, options UploadOptions) error

Upload Hierarchical Document

func (s *Service) UploadHierarchical(ctx context.Context, file io.Reader, filename string, options HierarchicalUploadOptions) (string, error)

Download File

func (s *Service) Download(ctx context.Context, bucket, objectName string, options DownloadOptions) (io.ReadCloser, error)

Delete File

func (s *Service) Delete(ctx context.Context, bucket, objectName string) error

Get File Info

func (s *Service) GetInfo(ctx context.Context, bucket, objectName string) (*FileInfo, error)

List Files

func (s *Service) List(ctx context.Context, bucket string, options ListOptions) (*ListResult, error)

Copy File

func (s *Service) Copy(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) error

Move File

func (s *Service) Move(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) error

Document Operations

List Documents

func (s *Service) ListDocuments(ctx context.Context, category, subCategory string, options ListOptions) (*ListResult, error)

Search Documents

func (s *Service) SearchDocuments(ctx context.Context, query string, category, subCategory string, options ListOptions) (*ListResult, error)

Bucket Operations

Create Bucket

func (cm *ConnectionManager) CreateBucket(ctx context.Context, bucketName string) error

Delete Bucket

func (cm *ConnectionManager) DeleteBucket(ctx context.Context, bucketName string) error

List Buckets

func (cm *ConnectionManager) ListBuckets(ctx context.Context) ([]string, error)

Configuration Options

type Config struct {
    Endpoint        string        `json:"endpoint"`        // MinIO server endpoint
    AccessKeyID     string        `json:"accessKeyID"`     // Access key
    SecretAccessKey string        `json:"secretAccessKey"` // Secret key
    UseSSL          bool          `json:"useSSL"`          // Use HTTPS
    Region          string        `json:"region"`          // Region
    ConnectTimeout  time.Duration `json:"connectTimeout"`  // Connection timeout
    RequestTimeout  time.Duration `json:"requestTimeout"`  // Request timeout
    DefaultBucket   string        `json:"defaultBucket"`   // Default bucket for normal files
    HierarchicalBucket string     `json:"hierarchicalBucket"` // Bucket for hierarchical storage
    PathSeparator   string        `json:"pathSeparator"`   // Path separator (default: "/")
    MaxFileSize     int64         `json:"maxFileSize"`     // Max file size in bytes
    AllowedTypes    []string      `json:"allowedTypes"`    // Allowed MIME types
    ChunkSize       int64         `json:"chunkSize"`       // Upload chunk size
}

Hierarchical Storage Structure

Documents are automatically organized in a hierarchical structure:

{hierarchicalBucket}/
├── {category}/
│   ├── {subCategory}/
│   │   ├── {tags}/
│   │   │   ├── {timestamp}/
│   │   │   │   └── {filename}
│   │   │   └── ...
│   │   └── ...
│   └── ...
└── ...

Example:

documents/
├── contracts/
│   ├── 2024/
│   │   ├── legal_service/
│   │   │   ├── 1640995200/
│   │   │   │   └── contract.pdf
│   │   │   └── 1640995300/
│   │   │       └── amendment.pdf
│   │   └── financial/
│   │       └── 1640995400/
│   │           └── invoice.pdf
│   └── 2023/
│       └── legal/
│           └── 1638360000/
│               └── old_contract.pdf
└── reports/
    └── monthly/
        └── 1640995500/
            └── sales_report.pdf

Metadata Management

Files can have rich metadata attached:

metadata := map[string]string{
    "user_id":     "12345",
    "category":    "contracts",
    "subcategory": "2024",
    "tags":        "legal,service",
    "version":     "1.0",
    "title":       "Service Contract",
    "uploaded_by": "john.doe",
    "department":  "legal",
}

Error Handling

The package provides specific error types:

var (
    ErrInvalidConfig      = errors.New("invalid configuration")
    ErrNotConnected       = errors.New("not connected to MinIO")
    ErrBucketNotFound     = errors.New("bucket not found")
    ErrObjectNotFound     = errors.New("object not found")
    ErrBucketAlreadyExists = errors.New("bucket already exists")
    ErrUploadFailed       = errors.New("upload failed")
    ErrDownloadFailed     = errors.New("download failed")
    ErrFileTooLarge       = errors.New("file too large")
    ErrInvalidFileType    = errors.New("invalid file type")
)

// Handle specific errors
file, err := service.GetInfo(ctx, bucket, objectName)
if err != nil {
    if errors.Is(err, minio.ErrObjectNotFound) {
        // Handle not found
        return
    }
    // Handle other errors
}

Presigned URLs

Generate temporary access URLs:

// Generate URL valid for 1 hour
url, err := service.GetPresignedURL(context.Background(), "files", "document.pdf", 3600)
if err != nil {
    log.Fatal(err)
}

fmt.Printf("Access URL: %s\n", url)
// Output: https://minio.example.com/files/document.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&...

Logger Interface

Implement structured logging:

type YourLogger struct{}

func (l *YourLogger) Info(message string, fields map[string]interface{}) {
    log.Printf("[INFO] %s: %+v", message, fields)
}

func (l *YourLogger) Error(message string, fields map[string]interface{}) {
    log.Printf("[ERROR] %s: %+v", message, fields)
}

func (l *YourLogger) Debug(message string, fields map[string]interface{}) {
    log.Printf("[DEBUG] %s: %+v", message, fields)
}

func (l *YourLogger) Warn(message string, fields map[string]interface{}) {
    log.Printf("[WARN] %s: %+v", message, fields)
}

Performance Considerations

1. Connection Pooling

Configure appropriate connection pool settings:

config := minio.Config{
    // ... other settings
    ConnectTimeout: 10 * time.Second,
    RequestTimeout: 30 * time.Second,
}

2. Chunked Uploads

Large files are automatically chunked for efficient upload:

config := minio.Config{
    MaxFileSize: 100 * 1024 * 1024, // 100MB
    ChunkSize:   5 * 1024 * 1024,   // 5MB chunks
}

3. Streaming Operations

Use streaming for large files to minimize memory usage:

// Streaming upload
file, _ := os.Open("large_file.zip")
defer file.Close()

err := service.Upload(ctx, bucket, objectName, file, options)

// Streaming download
reader, _ := service.Download(ctx, bucket, objectName, options)
defer reader.Close()

io.Copy(destination, reader)

Security Best Practices

1. Access Control

• Use IAM policies to restrict access
• Generate presigned URLs for temporary access
• Never expose access keys in client-side code

2. Input Validation

// Validate file type and size
if err := service.validateFileType(contentType); err != nil {
    return err
}

if err := service.validateFileSize(fileSize); err != nil {
    return err
}

3. Secure Configuration

• Store credentials securely (environment variables, secret management)
• Use HTTPS in production
• Rotate access keys regularly

Testing

Unit Testing

func TestService_Upload(t *testing.T) {
    // Create mock connection manager
    mockConn := &MockConnectionManager{}

    // Create service
    service := minio.NewService(mockConn, config, logger)

    // Test upload
    reader := strings.NewReader("test content")
    err := service.Upload(context.Background(), "test-bucket", "test-object", reader, minio.UploadOptions{})

    assert.NoError(t, err)
}

Integration Testing

func TestMinIOIntegration(t *testing.T) {
    if testing.Short() {
        t.Skip("Skipping integration test")
    }

    // Set up MinIO test server
    // ... test implementation
}

Examples

Image Upload with Metadata

func uploadProfileImage(userID string, imageFile io.Reader) error {
    objectName := fmt.Sprintf("profiles/%s/avatar.jpg", userID)

    return service.Upload(context.Background(), "files", objectName, imageFile, minio.UploadOptions{
        ContentType: "image/jpeg",
        Metadata: map[string]string{
            "user_id":   userID,
            "type":      "profile",
            "uploaded":  fmt.Sprintf("%d", time.Now().Unix()),
        },
    })
}

Document Management System

func uploadContract(contractFile io.Reader, contractData ContractData) (string, error) {
    return service.UploadDocument(context.Background(), contractFile, contractData.Filename, minio.HierarchicalUploadOptions{
        UploadOptions: minio.UploadOptions{
            ContentType: "application/pdf",
            Metadata: map[string]string{
                "contract_id": contractData.ID,
                "client_id":   contractData.ClientID,
                "value":       fmt.Sprintf("%.2f", contractData.Value),
                "signed":      fmt.Sprintf("%v", contractData.Signed),
            },
        },
        Category:    "contracts",
        SubCategory: contractData.Year,
        Tags:        []string{"legal", contractData.Type},
    })
}

func listContractsByYear(year string) (*minio.ListResult, error) {
    return service.ListDocuments(context.Background(), "contracts", year, minio.ListOptions{
        MaxKeys: 100,
    })
}

Migration Guide

From Direct MinIO Usage

// Before
client, err := minio.New(endpoint, &minio.Options{...})
_, err = client.PutObject(ctx, bucket, objectName, reader, -1, minio.PutObjectOptions{...})

// After
service := minio.NewService(connManager, config, logger)
err := service.Upload(ctx, bucket, objectName, reader, options)

From Other Storage Solutions

// From AWS S3 SDK
// Before: s3Client.PutObject(...)

// After
service := minio.NewService(connManager, config, logger)
err := service.Upload(ctx, bucket, objectName, reader, options)

Contributing

1. Fork the repository
2. Create a feature branch
3. Add tests for new functionality
4. Ensure all tests pass
5. Submit a pull request

License

This project is licensed under the MIT License.