Add MinIO Package for Object Storage Management
- Introduced a comprehensive MinIO package for managing object storage, including support for hierarchical document storage and general file management. - Implemented core components such as Config, ConnectionManager, and Service, following Clean Architecture principles. - Added functionality for file uploads, downloads, bucket management, and structured logging, enhancing usability and maintainability. - Included detailed error handling and validation for configuration and operations, ensuring robustness. - Updated documentation with usage examples and configuration guidelines to facilitate integration and understanding of the package's functionality.
This commit is contained in:
Nima Nakhostin
@@ -0,0 +1,460 @@
|
||||
# MinIO Package Usage Examples
|
||||
|
||||
## Basic Setup
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"path/to/pkg/minio"
|
||||
)
|
||||
|
||||
// Implement the Logger interface
|
||||
type SimpleLogger struct{}
|
||||
|
||||
func (l *SimpleLogger) Info(message string, fields map[string]interface{}) {
|
||||
log.Printf("[INFO] %s: %+v", message, fields)
|
||||
}
|
||||
|
||||
func (l *SimpleLogger) Error(message string, fields map[string]interface{}) {
|
||||
log.Printf("[ERROR] %s: %+v", message, fields)
|
||||
}
|
||||
|
||||
func (l *SimpleLogger) Debug(message string, fields map[string]interface{}) {
|
||||
log.Printf("[DEBUG] %s: %+v", message, fields)
|
||||
}
|
||||
|
||||
func (l *SimpleLogger) Warn(message string, fields map[string]interface{}) {
|
||||
log.Printf("[WARN] %s: %+v", message, fields)
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Create configuration
|
||||
config := minio.Config{
|
||||
Endpoint: "localhost:9000",
|
||||
AccessKeyID: "minioadmin",
|
||||
SecretAccessKey: "minioadmin",
|
||||
UseSSL: false,
|
||||
DefaultBucket: "files",
|
||||
HierarchicalBucket: "documents",
|
||||
}
|
||||
|
||||
// Create logger
|
||||
logger := &SimpleLogger{}
|
||||
|
||||
// Create connection manager
|
||||
connManager, err := minio.NewConnectionManager(config, logger)
|
||||
if err != nil {
|
||||
log.Fatal("Failed to create connection manager:", err)
|
||||
}
|
||||
|
||||
// Connect to MinIO
|
||||
if err := connManager.Connect(); err != nil {
|
||||
log.Fatal("Failed to connect to MinIO:", err)
|
||||
}
|
||||
defer connManager.Disconnect(context.Background())
|
||||
|
||||
// Create service
|
||||
service := minio.NewService(connManager, config, logger)
|
||||
|
||||
// Use the service...
|
||||
}
|
||||
```
|
||||
|
||||
## Normal File Storage
|
||||
|
||||
### Upload an Image
|
||||
|
||||
```go
|
||||
func uploadImage(service *minio.Service) error {
|
||||
// Open image file
|
||||
file, err := os.Open("profile.jpg")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Upload to normal storage
|
||||
return service.Upload(context.Background(), "files", "images/profile.jpg", file, minio.UploadOptions{
|
||||
ContentType: "image/jpeg",
|
||||
Metadata: map[string]string{
|
||||
"user_id": "12345",
|
||||
"type": "profile_image",
|
||||
},
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
### Download and Serve a File
|
||||
|
||||
```go
|
||||
func serveFile(service *minio.Service, w http.ResponseWriter, bucket, objectName string) error {
|
||||
// Download file
|
||||
reader, err := service.Download(context.Background(), bucket, objectName, minio.DownloadOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// Get file info for headers
|
||||
info, err := service.GetInfo(context.Background(), bucket, objectName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set response headers
|
||||
w.Header().Set("Content-Type", info.ContentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
|
||||
// Copy file content to response
|
||||
_, err = io.Copy(w, reader)
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
### Upload with Stream Processing
|
||||
|
||||
```go
|
||||
func uploadWithProgress(service *minio.Service, file io.Reader, filename string) error {
|
||||
// Create a progress reader
|
||||
progressReader := &ProgressReader{
|
||||
Reader: file,
|
||||
Total: 0, // Set total size if known
|
||||
Reporter: func(bytesRead int64) {
|
||||
log.Printf("Uploaded %d bytes", bytesRead)
|
||||
},
|
||||
}
|
||||
|
||||
return service.Upload(context.Background(), "files", "uploads/"+filename, progressReader, minio.UploadOptions{
|
||||
ContentType: mime.TypeByExtension(filepath.Ext(filename)),
|
||||
})
|
||||
}
|
||||
|
||||
type ProgressReader struct {
|
||||
Reader io.Reader
|
||||
Total int64
|
||||
Reporter func(bytesRead int64)
|
||||
}
|
||||
|
||||
func (pr *ProgressReader) Read(p []byte) (int, error) {
|
||||
n, err := pr.Reader.Read(p)
|
||||
if n > 0 && pr.Reporter != nil {
|
||||
pr.Total += int64(n)
|
||||
pr.Reporter(pr.Total)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
```
|
||||
|
||||
## Hierarchical Document Storage
|
||||
|
||||
### Upload a Contract
|
||||
|
||||
```go
|
||||
func uploadContract(service *minio.Service) error {
|
||||
// Open contract file
|
||||
file, err := os.Open("contract.pdf")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Upload to hierarchical storage
|
||||
docPath, err := service.UploadDocument(context.Background(), file, "service_contract.pdf", minio.HierarchicalUploadOptions{
|
||||
UploadOptions: minio.UploadOptions{
|
||||
ContentType: "application/pdf",
|
||||
Metadata: map[string]string{
|
||||
"contract_id": "CTR-2024-001",
|
||||
"client_id": "CLIENT-123",
|
||||
"value": "50000.00",
|
||||
"signed": "true",
|
||||
"department": "legal",
|
||||
},
|
||||
},
|
||||
Category: "contracts",
|
||||
SubCategory: "2024",
|
||||
Tags: []string{"legal", "service", "active"},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Contract uploaded to: %s", docPath)
|
||||
// Output: contracts/2024/legal_service_active/1640995200/service_contract.pdf
|
||||
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
### List Documents by Category
|
||||
|
||||
```go
|
||||
func listContractsByYear(service *minio.Service, year string) error {
|
||||
result, err := service.ListDocuments(context.Background(), "contracts", year, minio.ListOptions{
|
||||
MaxKeys: 50,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Found %d documents in contracts/%s", len(result.Objects), year)
|
||||
|
||||
for _, doc := range result.Objects {
|
||||
log.Printf("- %s (%d bytes)", doc.Name, doc.Size)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
### Search Documents
|
||||
|
||||
```go
|
||||
func searchLegalDocuments(service *minio.Service) error {
|
||||
result, err := service.SearchDocuments(context.Background(), "legal", "contracts", "2024", minio.ListOptions{
|
||||
MaxKeys: 20,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Found %d legal documents", len(result.Objects))
|
||||
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
## File Operations
|
||||
|
||||
### Copy Files Between Buckets
|
||||
|
||||
```go
|
||||
func backupFile(service *minio.Service, srcBucket, srcObject, destBucket, destObject string) error {
|
||||
return service.Copy(context.Background(), srcBucket, srcObject, destBucket, destObject)
|
||||
}
|
||||
```
|
||||
|
||||
### Generate Presigned URLs
|
||||
|
||||
```go
|
||||
func generateDownloadLink(service *minio.Service, bucket, objectName string) (string, error) {
|
||||
// Generate URL valid for 1 hour
|
||||
return service.GetPresignedURL(context.Background(), bucket, objectName, 3600)
|
||||
}
|
||||
```
|
||||
|
||||
### Check File Existence
|
||||
|
||||
```go
|
||||
func fileExists(service *minio.Service, bucket, objectName string) bool {
|
||||
exists, err := service.Exists(context.Background(), bucket, objectName)
|
||||
if err != nil {
|
||||
log.Printf("Error checking file existence: %v", err)
|
||||
return false
|
||||
}
|
||||
return exists
|
||||
}
|
||||
```
|
||||
|
||||
## Bucket Management
|
||||
|
||||
### Create Required Buckets
|
||||
|
||||
```go
|
||||
func ensureBucketsExist(connManager *minio.ConnectionManager) error {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create default bucket
|
||||
if err := connManager.CreateBucket(ctx, "files"); err != nil {
|
||||
return fmt.Errorf("failed to create files bucket: %w", err)
|
||||
}
|
||||
|
||||
// Create hierarchical bucket
|
||||
if err := connManager.CreateBucket(ctx, "documents"); err != nil {
|
||||
return fmt.Errorf("failed to create documents bucket: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
### List All Buckets
|
||||
|
||||
```go
|
||||
func listAllBuckets(connManager *minio.ConnectionManager) error {
|
||||
buckets, err := connManager.ListBuckets(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Available buckets:")
|
||||
for _, bucket := range buckets {
|
||||
log.Printf("- %s", bucket)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
```go
|
||||
func handleUploadError(err error) {
|
||||
switch {
|
||||
case errors.Is(err, minio.ErrInvalidConfig):
|
||||
log.Println("Configuration error:", err)
|
||||
case errors.Is(err, minio.ErrNotConnected):
|
||||
log.Println("Connection error:", err)
|
||||
case errors.Is(err, minio.ErrUploadFailed):
|
||||
log.Println("Upload failed:", err)
|
||||
case errors.Is(err, minio.ErrFileTooLarge):
|
||||
log.Println("File too large:", err)
|
||||
case errors.Is(err, minio.ErrInvalidFileType):
|
||||
log.Println("Invalid file type:", err)
|
||||
default:
|
||||
log.Println("Unknown error:", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Integration with HTTP Handlers
|
||||
|
||||
### File Upload Handler
|
||||
|
||||
```go
|
||||
func uploadHandler(service *minio.Service) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// Get uploaded file
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "No file provided"})
|
||||
return
|
||||
}
|
||||
|
||||
// Open uploaded file
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to open file"})
|
||||
return
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// Generate unique filename
|
||||
filename := fmt.Sprintf("%d_%s", time.Now().Unix(), file.Filename)
|
||||
objectName := fmt.Sprintf("uploads/%s", filename)
|
||||
|
||||
// Upload file
|
||||
err = service.Upload(c.Request.Context(), "files", objectName, src, minio.UploadOptions{
|
||||
ContentType: file.Header.Get("Content-Type"),
|
||||
Metadata: map[string]string{
|
||||
"original_name": file.Filename,
|
||||
"uploaded_by": "user_id", // Get from context
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
handleUploadError(err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Upload failed"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"message": "File uploaded successfully",
|
||||
"filename": filename,
|
||||
"path": objectName,
|
||||
})
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Document Upload Handler
|
||||
|
||||
```go
|
||||
func uploadDocumentHandler(service *minio.Service) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// Get form data
|
||||
category := c.PostForm("category")
|
||||
subCategory := c.PostForm("subcategory")
|
||||
tags := c.PostFormArray("tags[]")
|
||||
|
||||
// Get uploaded file
|
||||
file, err := c.FormFile("document")
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "No document provided"})
|
||||
return
|
||||
}
|
||||
|
||||
// Open document
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to open document"})
|
||||
return
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// Upload to hierarchical storage
|
||||
docPath, err := service.UploadDocument(c.Request.Context(), src, file.Filename, minio.HierarchicalUploadOptions{
|
||||
UploadOptions: minio.UploadOptions{
|
||||
ContentType: file.Header.Get("Content-Type"),
|
||||
Metadata: map[string]string{
|
||||
"uploaded_by": "user_id", // Get from context
|
||||
"department": c.PostForm("department"),
|
||||
},
|
||||
},
|
||||
Category: category,
|
||||
SubCategory: subCategory,
|
||||
Tags: tags,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
handleUploadError(err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Document upload failed"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"message": "Document uploaded successfully",
|
||||
"path": docPath,
|
||||
})
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration Examples
|
||||
|
||||
### Development Configuration
|
||||
|
||||
```go
|
||||
config := minio.Config{
|
||||
Endpoint: "localhost:9000",
|
||||
AccessKeyID: "minioadmin",
|
||||
SecretAccessKey: "minioadmin",
|
||||
UseSSL: false,
|
||||
Region: "us-east-1",
|
||||
DefaultBucket: "files",
|
||||
HierarchicalBucket: "documents",
|
||||
MaxFileSize: 50 * 1024 * 1024, // 50MB
|
||||
AllowedTypes: []string{"image/", "application/pdf", "text/"},
|
||||
}
|
||||
```
|
||||
|
||||
### Production Configuration
|
||||
|
||||
```go
|
||||
config := minio.Config{
|
||||
Endpoint: "minio.example.com:443",
|
||||
AccessKeyID: os.Getenv("MINIO_ACCESS_KEY"),
|
||||
SecretAccessKey: os.Getenv("MINIO_SECRET_KEY"),
|
||||
UseSSL: true,
|
||||
Region: "us-west-2",
|
||||
DefaultBucket: "production-files",
|
||||
HierarchicalBucket: "production-documents",
|
||||
MaxFileSize: 100 * 1024 * 1024, // 100MB
|
||||
AllowedTypes: []string{"image/jpeg", "image/png", "application/pdf"},
|
||||
ConnectTimeout: 30 * time.Second,
|
||||
RequestTimeout: 60 * time.Second,
|
||||
}
|
||||
```
|
||||
Reference in New Issue
Block a user