Add MinIO Package for Object Storage Management
- Introduced a comprehensive MinIO package for managing object storage, including support for hierarchical document storage and general file management. - Implemented core components such as Config, ConnectionManager, and Service, following Clean Architecture principles. - Added functionality for file uploads, downloads, bucket management, and structured logging, enhancing usability and maintainability. - Included detailed error handling and validation for configuration and operations, ensuring robustness. - Updated documentation with usage examples and configuration guidelines to facilitate integration and understanding of the package's functionality.
This commit is contained in:
Nima Nakhostin
@@ -0,0 +1,532 @@
|
||||
# MinIO Storage Package
|
||||
|
||||
A comprehensive Go package for MinIO object storage with support for hierarchical document storage and general file management. This package provides both normal file storage for images, videos, and other media files, as well as classified hierarchical storage for documents.
|
||||
|
||||
## Features
|
||||
|
||||
✅ **Hierarchical Document Storage** - Organize documents in categorized folders with metadata
|
||||
✅ **Normal File Storage** - Standard object storage for images, videos, and other files
|
||||
✅ **File Upload/Download** - Stream-based operations with progress tracking
|
||||
✅ **Metadata Management** - Rich metadata support for classification and search
|
||||
✅ **Presigned URLs** - Temporary access URLs for secure file sharing
|
||||
✅ **Bucket Management** - Create, delete, and manage storage buckets
|
||||
✅ **File Operations** - Copy, move, delete, and list files
|
||||
✅ **Connection Management** - Robust connection pooling and health checks
|
||||
✅ **Structured Logging** - Comprehensive logging with structured fields
|
||||
✅ **Error Handling** - Detailed error types and context
|
||||
✅ **Type Safety** - Full Go generics support
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
go get github.com/minio/minio-go/v7
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Configuration
|
||||
|
||||
```go
|
||||
import "path/to/pkg/minio"
|
||||
|
||||
// Create configuration
|
||||
config := minio.Config{
|
||||
Endpoint: "localhost:9000",
|
||||
AccessKeyID: "your-access-key",
|
||||
SecretAccessKey: "your-secret-key",
|
||||
UseSSL: false,
|
||||
Region: "us-east-1",
|
||||
DefaultBucket: "files",
|
||||
HierarchicalBucket: "documents",
|
||||
}
|
||||
|
||||
// Validate configuration
|
||||
if err := config.Validate(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Initialize Service
|
||||
|
||||
```go
|
||||
// Create logger (implement minio.Logger interface)
|
||||
logger := &YourLogger{}
|
||||
|
||||
// Create connection manager
|
||||
connManager, err := minio.NewConnectionManager(config, logger)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Connect to MinIO
|
||||
if err := connManager.Connect(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer connManager.Disconnect(context.Background())
|
||||
|
||||
// Create service
|
||||
service := minio.NewService(connManager, config, logger)
|
||||
```
|
||||
|
||||
### 3. Normal File Storage
|
||||
|
||||
```go
|
||||
// Upload a file
|
||||
file, err := os.Open("image.jpg")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
err = service.Upload(context.Background(), "files", "images/profile.jpg", file, minio.UploadOptions{
|
||||
ContentType: "image/jpeg",
|
||||
Metadata: map[string]string{
|
||||
"user_id": "123",
|
||||
"type": "profile",
|
||||
},
|
||||
})
|
||||
|
||||
// Download a file
|
||||
reader, err := service.Download(context.Background(), "files", "images/profile.jpg", minio.DownloadOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// Copy file data to response
|
||||
io.Copy(w, reader)
|
||||
```
|
||||
|
||||
### 4. Hierarchical Document Storage
|
||||
|
||||
```go
|
||||
// Upload a document with classification
|
||||
docFile, err := os.Open("contract.pdf")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer docFile.Close()
|
||||
|
||||
docPath, err := service.UploadDocument(context.Background(), docFile, "contract.pdf", minio.HierarchicalUploadOptions{
|
||||
UploadOptions: minio.UploadOptions{
|
||||
ContentType: "application/pdf",
|
||||
Metadata: map[string]string{
|
||||
"title": "Service Contract",
|
||||
"version": "1.0",
|
||||
},
|
||||
},
|
||||
Category: "contracts",
|
||||
SubCategory: "2024",
|
||||
Tags: []string{"legal", "service"},
|
||||
})
|
||||
|
||||
fmt.Printf("Document uploaded to: %s\n", docPath)
|
||||
// Output: contracts/2024/legal_service/1640995200/contract.pdf
|
||||
```
|
||||
|
||||
## Storage Types
|
||||
|
||||
### Normal Storage
|
||||
- **Use Case**: Images, videos, general files
|
||||
- **Structure**: Flat object storage with custom paths
|
||||
- **Example**: `images/profile.jpg`, `videos/tutorial.mp4`
|
||||
|
||||
### Hierarchical Storage
|
||||
- **Use Case**: Documents requiring classification and organization
|
||||
- **Structure**: Categorized folder structure with metadata
|
||||
- **Example**: `contracts/2024/legal_service/1640995200/contract.pdf`
|
||||
|
||||
## API Reference
|
||||
|
||||
### File Operations
|
||||
|
||||
#### Upload File
|
||||
```go
|
||||
func (s *Service) Upload(ctx context.Context, bucket, objectName string, reader io.Reader, options UploadOptions) error
|
||||
```
|
||||
|
||||
#### Upload Hierarchical Document
|
||||
```go
|
||||
func (s *Service) UploadHierarchical(ctx context.Context, file io.Reader, filename string, options HierarchicalUploadOptions) (string, error)
|
||||
```
|
||||
|
||||
#### Download File
|
||||
```go
|
||||
func (s *Service) Download(ctx context.Context, bucket, objectName string, options DownloadOptions) (io.ReadCloser, error)
|
||||
```
|
||||
|
||||
#### Delete File
|
||||
```go
|
||||
func (s *Service) Delete(ctx context.Context, bucket, objectName string) error
|
||||
```
|
||||
|
||||
#### Get File Info
|
||||
```go
|
||||
func (s *Service) GetInfo(ctx context.Context, bucket, objectName string) (*FileInfo, error)
|
||||
```
|
||||
|
||||
#### List Files
|
||||
```go
|
||||
func (s *Service) List(ctx context.Context, bucket string, options ListOptions) (*ListResult, error)
|
||||
```
|
||||
|
||||
#### Copy File
|
||||
```go
|
||||
func (s *Service) Copy(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) error
|
||||
```
|
||||
|
||||
#### Move File
|
||||
```go
|
||||
func (s *Service) Move(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) error
|
||||
```
|
||||
|
||||
### Document Operations
|
||||
|
||||
#### List Documents
|
||||
```go
|
||||
func (s *Service) ListDocuments(ctx context.Context, category, subCategory string, options ListOptions) (*ListResult, error)
|
||||
```
|
||||
|
||||
#### Search Documents
|
||||
```go
|
||||
func (s *Service) SearchDocuments(ctx context.Context, query string, category, subCategory string, options ListOptions) (*ListResult, error)
|
||||
```
|
||||
|
||||
### Bucket Operations
|
||||
|
||||
#### Create Bucket
|
||||
```go
|
||||
func (cm *ConnectionManager) CreateBucket(ctx context.Context, bucketName string) error
|
||||
```
|
||||
|
||||
#### Delete Bucket
|
||||
```go
|
||||
func (cm *ConnectionManager) DeleteBucket(ctx context.Context, bucketName string) error
|
||||
```
|
||||
|
||||
#### List Buckets
|
||||
```go
|
||||
func (cm *ConnectionManager) ListBuckets(ctx context.Context) ([]string, error)
|
||||
```
|
||||
|
||||
## Configuration Options
|
||||
|
||||
```go
|
||||
type Config struct {
|
||||
Endpoint string `json:"endpoint"` // MinIO server endpoint
|
||||
AccessKeyID string `json:"accessKeyID"` // Access key
|
||||
SecretAccessKey string `json:"secretAccessKey"` // Secret key
|
||||
UseSSL bool `json:"useSSL"` // Use HTTPS
|
||||
Region string `json:"region"` // Region
|
||||
ConnectTimeout time.Duration `json:"connectTimeout"` // Connection timeout
|
||||
RequestTimeout time.Duration `json:"requestTimeout"` // Request timeout
|
||||
DefaultBucket string `json:"defaultBucket"` // Default bucket for normal files
|
||||
HierarchicalBucket string `json:"hierarchicalBucket"` // Bucket for hierarchical storage
|
||||
PathSeparator string `json:"pathSeparator"` // Path separator (default: "/")
|
||||
MaxFileSize int64 `json:"maxFileSize"` // Max file size in bytes
|
||||
AllowedTypes []string `json:"allowedTypes"` // Allowed MIME types
|
||||
ChunkSize int64 `json:"chunkSize"` // Upload chunk size
|
||||
}
|
||||
```
|
||||
|
||||
## Hierarchical Storage Structure
|
||||
|
||||
Documents are automatically organized in a hierarchical structure:
|
||||
|
||||
```
|
||||
{hierarchicalBucket}/
|
||||
├── {category}/
|
||||
│ ├── {subCategory}/
|
||||
│ │ ├── {tags}/
|
||||
│ │ │ ├── {timestamp}/
|
||||
│ │ │ │ └── {filename}
|
||||
│ │ │ └── ...
|
||||
│ │ └── ...
|
||||
│ └── ...
|
||||
└── ...
|
||||
```
|
||||
|
||||
**Example:**
|
||||
```
|
||||
documents/
|
||||
├── contracts/
|
||||
│ ├── 2024/
|
||||
│ │ ├── legal_service/
|
||||
│ │ │ ├── 1640995200/
|
||||
│ │ │ │ └── contract.pdf
|
||||
│ │ │ └── 1640995300/
|
||||
│ │ │ └── amendment.pdf
|
||||
│ │ └── financial/
|
||||
│ │ └── 1640995400/
|
||||
│ │ └── invoice.pdf
|
||||
│ └── 2023/
|
||||
│ └── legal/
|
||||
│ └── 1638360000/
|
||||
│ └── old_contract.pdf
|
||||
└── reports/
|
||||
└── monthly/
|
||||
└── 1640995500/
|
||||
└── sales_report.pdf
|
||||
```
|
||||
|
||||
## Metadata Management
|
||||
|
||||
Files can have rich metadata attached:
|
||||
|
||||
```go
|
||||
metadata := map[string]string{
|
||||
"user_id": "12345",
|
||||
"category": "contracts",
|
||||
"subcategory": "2024",
|
||||
"tags": "legal,service",
|
||||
"version": "1.0",
|
||||
"title": "Service Contract",
|
||||
"uploaded_by": "john.doe",
|
||||
"department": "legal",
|
||||
}
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The package provides specific error types:
|
||||
|
||||
```go
|
||||
var (
|
||||
ErrInvalidConfig = errors.New("invalid configuration")
|
||||
ErrNotConnected = errors.New("not connected to MinIO")
|
||||
ErrBucketNotFound = errors.New("bucket not found")
|
||||
ErrObjectNotFound = errors.New("object not found")
|
||||
ErrBucketAlreadyExists = errors.New("bucket already exists")
|
||||
ErrUploadFailed = errors.New("upload failed")
|
||||
ErrDownloadFailed = errors.New("download failed")
|
||||
ErrFileTooLarge = errors.New("file too large")
|
||||
ErrInvalidFileType = errors.New("invalid file type")
|
||||
)
|
||||
|
||||
// Handle specific errors
|
||||
file, err := service.GetInfo(ctx, bucket, objectName)
|
||||
if err != nil {
|
||||
if errors.Is(err, minio.ErrObjectNotFound) {
|
||||
// Handle not found
|
||||
return
|
||||
}
|
||||
// Handle other errors
|
||||
}
|
||||
```
|
||||
|
||||
## Presigned URLs
|
||||
|
||||
Generate temporary access URLs:
|
||||
|
||||
```go
|
||||
// Generate URL valid for 1 hour
|
||||
url, err := service.GetPresignedURL(context.Background(), "files", "document.pdf", 3600)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Printf("Access URL: %s\n", url)
|
||||
// Output: https://minio.example.com/files/document.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&...
|
||||
```
|
||||
|
||||
## Logger Interface
|
||||
|
||||
Implement structured logging:
|
||||
|
||||
```go
|
||||
type YourLogger struct{}
|
||||
|
||||
func (l *YourLogger) Info(message string, fields map[string]interface{}) {
|
||||
log.Printf("[INFO] %s: %+v", message, fields)
|
||||
}
|
||||
|
||||
func (l *YourLogger) Error(message string, fields map[string]interface{}) {
|
||||
log.Printf("[ERROR] %s: %+v", message, fields)
|
||||
}
|
||||
|
||||
func (l *YourLogger) Debug(message string, fields map[string]interface{}) {
|
||||
log.Printf("[DEBUG] %s: %+v", message, fields)
|
||||
}
|
||||
|
||||
func (l *YourLogger) Warn(message string, fields map[string]interface{}) {
|
||||
log.Printf("[WARN] %s: %+v", message, fields)
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
### 1. Connection Pooling
|
||||
Configure appropriate connection pool settings:
|
||||
|
||||
```go
|
||||
config := minio.Config{
|
||||
// ... other settings
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
RequestTimeout: 30 * time.Second,
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Chunked Uploads
|
||||
Large files are automatically chunked for efficient upload:
|
||||
|
||||
```go
|
||||
config := minio.Config{
|
||||
MaxFileSize: 100 * 1024 * 1024, // 100MB
|
||||
ChunkSize: 5 * 1024 * 1024, // 5MB chunks
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Streaming Operations
|
||||
Use streaming for large files to minimize memory usage:
|
||||
|
||||
```go
|
||||
// Streaming upload
|
||||
file, _ := os.Open("large_file.zip")
|
||||
defer file.Close()
|
||||
|
||||
err := service.Upload(ctx, bucket, objectName, file, options)
|
||||
|
||||
// Streaming download
|
||||
reader, _ := service.Download(ctx, bucket, objectName, options)
|
||||
defer reader.Close()
|
||||
|
||||
io.Copy(destination, reader)
|
||||
```
|
||||
|
||||
## Security Best Practices
|
||||
|
||||
### 1. Access Control
|
||||
- Use IAM policies to restrict access
|
||||
- Generate presigned URLs for temporary access
|
||||
- Never expose access keys in client-side code
|
||||
|
||||
### 2. Input Validation
|
||||
```go
|
||||
// Validate file type and size
|
||||
if err := service.validateFileType(contentType); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := service.validateFileSize(fileSize); err != nil {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Secure Configuration
|
||||
- Store credentials securely (environment variables, secret management)
|
||||
- Use HTTPS in production
|
||||
- Rotate access keys regularly
|
||||
|
||||
## Testing
|
||||
|
||||
### Unit Testing
|
||||
|
||||
```go
|
||||
func TestService_Upload(t *testing.T) {
|
||||
// Create mock connection manager
|
||||
mockConn := &MockConnectionManager{}
|
||||
|
||||
// Create service
|
||||
service := minio.NewService(mockConn, config, logger)
|
||||
|
||||
// Test upload
|
||||
reader := strings.NewReader("test content")
|
||||
err := service.Upload(context.Background(), "test-bucket", "test-object", reader, minio.UploadOptions{})
|
||||
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
```
|
||||
|
||||
### Integration Testing
|
||||
|
||||
```go
|
||||
func TestMinIOIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
// Set up MinIO test server
|
||||
// ... test implementation
|
||||
}
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Image Upload with Metadata
|
||||
```go
|
||||
func uploadProfileImage(userID string, imageFile io.Reader) error {
|
||||
objectName := fmt.Sprintf("profiles/%s/avatar.jpg", userID)
|
||||
|
||||
return service.Upload(context.Background(), "files", objectName, imageFile, minio.UploadOptions{
|
||||
ContentType: "image/jpeg",
|
||||
Metadata: map[string]string{
|
||||
"user_id": userID,
|
||||
"type": "profile",
|
||||
"uploaded": fmt.Sprintf("%d", time.Now().Unix()),
|
||||
},
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
### Document Management System
|
||||
```go
|
||||
func uploadContract(contractFile io.Reader, contractData ContractData) (string, error) {
|
||||
return service.UploadDocument(context.Background(), contractFile, contractData.Filename, minio.HierarchicalUploadOptions{
|
||||
UploadOptions: minio.UploadOptions{
|
||||
ContentType: "application/pdf",
|
||||
Metadata: map[string]string{
|
||||
"contract_id": contractData.ID,
|
||||
"client_id": contractData.ClientID,
|
||||
"value": fmt.Sprintf("%.2f", contractData.Value),
|
||||
"signed": fmt.Sprintf("%v", contractData.Signed),
|
||||
},
|
||||
},
|
||||
Category: "contracts",
|
||||
SubCategory: contractData.Year,
|
||||
Tags: []string{"legal", contractData.Type},
|
||||
})
|
||||
}
|
||||
|
||||
func listContractsByYear(year string) (*minio.ListResult, error) {
|
||||
return service.ListDocuments(context.Background(), "contracts", year, minio.ListOptions{
|
||||
MaxKeys: 100,
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
## Migration Guide
|
||||
|
||||
### From Direct MinIO Usage
|
||||
|
||||
```go
|
||||
// Before
|
||||
client, err := minio.New(endpoint, &minio.Options{...})
|
||||
_, err = client.PutObject(ctx, bucket, objectName, reader, -1, minio.PutObjectOptions{...})
|
||||
|
||||
// After
|
||||
service := minio.NewService(connManager, config, logger)
|
||||
err := service.Upload(ctx, bucket, objectName, reader, options)
|
||||
```
|
||||
|
||||
### From Other Storage Solutions
|
||||
|
||||
```go
|
||||
// From AWS S3 SDK
|
||||
// Before: s3Client.PutObject(...)
|
||||
|
||||
// After
|
||||
service := minio.NewService(connManager, config, logger)
|
||||
err := service.Upload(ctx, bucket, objectName, reader, options)
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Add tests for new functionality
|
||||
4. Ensure all tests pass
|
||||
5. Submit a pull request
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the MIT License.
|
||||
Reference in New Issue
Block a user