Nima Nakhostin
1a56070556
- Introduced a comprehensive MinIO package for managing object storage, including support for hierarchical document storage and general file management. - Implemented core components such as Config, ConnectionManager, and Service, following Clean Architecture principles. - Added functionality for file uploads, downloads, bucket management, and structured logging, enhancing usability and maintainability. - Included detailed error handling and validation for configuration and operations, ensuring robustness. - Updated documentation with usage examples and configuration guidelines to facilitate integration and understanding of the package's functionality.
461 lines
12 KiB
Markdown
461 lines
12 KiB
Markdown
# MinIO Package Usage Examples
|
|
|
|
## Basic Setup
|
|
|
|
```go
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"path/to/pkg/minio"
|
|
)
|
|
|
|
// Implement the Logger interface
|
|
type SimpleLogger struct{}
|
|
|
|
func (l *SimpleLogger) Info(message string, fields map[string]interface{}) {
|
|
log.Printf("[INFO] %s: %+v", message, fields)
|
|
}
|
|
|
|
func (l *SimpleLogger) Error(message string, fields map[string]interface{}) {
|
|
log.Printf("[ERROR] %s: %+v", message, fields)
|
|
}
|
|
|
|
func (l *SimpleLogger) Debug(message string, fields map[string]interface{}) {
|
|
log.Printf("[DEBUG] %s: %+v", message, fields)
|
|
}
|
|
|
|
func (l *SimpleLogger) Warn(message string, fields map[string]interface{}) {
|
|
log.Printf("[WARN] %s: %+v", message, fields)
|
|
}
|
|
|
|
func main() {
|
|
// Create configuration
|
|
config := minio.Config{
|
|
Endpoint: "localhost:9000",
|
|
AccessKeyID: "minioadmin",
|
|
SecretAccessKey: "minioadmin",
|
|
UseSSL: false,
|
|
DefaultBucket: "files",
|
|
HierarchicalBucket: "documents",
|
|
}
|
|
|
|
// Create logger
|
|
logger := &SimpleLogger{}
|
|
|
|
// Create connection manager
|
|
connManager, err := minio.NewConnectionManager(config, logger)
|
|
if err != nil {
|
|
log.Fatal("Failed to create connection manager:", err)
|
|
}
|
|
|
|
// Connect to MinIO
|
|
if err := connManager.Connect(); err != nil {
|
|
log.Fatal("Failed to connect to MinIO:", err)
|
|
}
|
|
defer connManager.Disconnect(context.Background())
|
|
|
|
// Create service
|
|
service := minio.NewService(connManager, config, logger)
|
|
|
|
// Use the service...
|
|
}
|
|
```
|
|
|
|
## Normal File Storage
|
|
|
|
### Upload an Image
|
|
|
|
```go
|
|
func uploadImage(service *minio.Service) error {
|
|
// Open image file
|
|
file, err := os.Open("profile.jpg")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
// Upload to normal storage
|
|
return service.Upload(context.Background(), "files", "images/profile.jpg", file, minio.UploadOptions{
|
|
ContentType: "image/jpeg",
|
|
Metadata: map[string]string{
|
|
"user_id": "12345",
|
|
"type": "profile_image",
|
|
},
|
|
})
|
|
}
|
|
```
|
|
|
|
### Download and Serve a File
|
|
|
|
```go
|
|
func serveFile(service *minio.Service, w http.ResponseWriter, bucket, objectName string) error {
|
|
// Download file
|
|
reader, err := service.Download(context.Background(), bucket, objectName, minio.DownloadOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer reader.Close()
|
|
|
|
// Get file info for headers
|
|
info, err := service.GetInfo(context.Background(), bucket, objectName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Set response headers
|
|
w.Header().Set("Content-Type", info.ContentType)
|
|
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
|
|
|
// Copy file content to response
|
|
_, err = io.Copy(w, reader)
|
|
return err
|
|
}
|
|
```
|
|
|
|
### Upload with Stream Processing
|
|
|
|
```go
|
|
func uploadWithProgress(service *minio.Service, file io.Reader, filename string) error {
|
|
// Create a progress reader
|
|
progressReader := &ProgressReader{
|
|
Reader: file,
|
|
Total: 0, // Set total size if known
|
|
Reporter: func(bytesRead int64) {
|
|
log.Printf("Uploaded %d bytes", bytesRead)
|
|
},
|
|
}
|
|
|
|
return service.Upload(context.Background(), "files", "uploads/"+filename, progressReader, minio.UploadOptions{
|
|
ContentType: mime.TypeByExtension(filepath.Ext(filename)),
|
|
})
|
|
}
|
|
|
|
type ProgressReader struct {
|
|
Reader io.Reader
|
|
Total int64
|
|
Reporter func(bytesRead int64)
|
|
}
|
|
|
|
func (pr *ProgressReader) Read(p []byte) (int, error) {
|
|
n, err := pr.Reader.Read(p)
|
|
if n > 0 && pr.Reporter != nil {
|
|
pr.Total += int64(n)
|
|
pr.Reporter(pr.Total)
|
|
}
|
|
return n, err
|
|
}
|
|
```
|
|
|
|
## Hierarchical Document Storage
|
|
|
|
### Upload a Contract
|
|
|
|
```go
|
|
func uploadContract(service *minio.Service) error {
|
|
// Open contract file
|
|
file, err := os.Open("contract.pdf")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
// Upload to hierarchical storage
|
|
docPath, err := service.UploadDocument(context.Background(), file, "service_contract.pdf", minio.HierarchicalUploadOptions{
|
|
UploadOptions: minio.UploadOptions{
|
|
ContentType: "application/pdf",
|
|
Metadata: map[string]string{
|
|
"contract_id": "CTR-2024-001",
|
|
"client_id": "CLIENT-123",
|
|
"value": "50000.00",
|
|
"signed": "true",
|
|
"department": "legal",
|
|
},
|
|
},
|
|
Category: "contracts",
|
|
SubCategory: "2024",
|
|
Tags: []string{"legal", "service", "active"},
|
|
})
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Printf("Contract uploaded to: %s", docPath)
|
|
// Output: contracts/2024/legal_service_active/1640995200/service_contract.pdf
|
|
|
|
return nil
|
|
}
|
|
```
|
|
|
|
### List Documents by Category
|
|
|
|
```go
|
|
func listContractsByYear(service *minio.Service, year string) error {
|
|
result, err := service.ListDocuments(context.Background(), "contracts", year, minio.ListOptions{
|
|
MaxKeys: 50,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Printf("Found %d documents in contracts/%s", len(result.Objects), year)
|
|
|
|
for _, doc := range result.Objects {
|
|
log.Printf("- %s (%d bytes)", doc.Name, doc.Size)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
```
|
|
|
|
### Search Documents
|
|
|
|
```go
|
|
func searchLegalDocuments(service *minio.Service) error {
|
|
result, err := service.SearchDocuments(context.Background(), "legal", "contracts", "2024", minio.ListOptions{
|
|
MaxKeys: 20,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Printf("Found %d legal documents", len(result.Objects))
|
|
|
|
return nil
|
|
}
|
|
```
|
|
|
|
## File Operations
|
|
|
|
### Copy Files Between Buckets
|
|
|
|
```go
|
|
func backupFile(service *minio.Service, srcBucket, srcObject, destBucket, destObject string) error {
|
|
return service.Copy(context.Background(), srcBucket, srcObject, destBucket, destObject)
|
|
}
|
|
```
|
|
|
|
### Generate Presigned URLs
|
|
|
|
```go
|
|
func generateDownloadLink(service *minio.Service, bucket, objectName string) (string, error) {
|
|
// Generate URL valid for 1 hour
|
|
return service.GetPresignedURL(context.Background(), bucket, objectName, 3600)
|
|
}
|
|
```
|
|
|
|
### Check File Existence
|
|
|
|
```go
|
|
func fileExists(service *minio.Service, bucket, objectName string) bool {
|
|
exists, err := service.Exists(context.Background(), bucket, objectName)
|
|
if err != nil {
|
|
log.Printf("Error checking file existence: %v", err)
|
|
return false
|
|
}
|
|
return exists
|
|
}
|
|
```
|
|
|
|
## Bucket Management
|
|
|
|
### Create Required Buckets
|
|
|
|
```go
|
|
func ensureBucketsExist(connManager *minio.ConnectionManager) error {
|
|
ctx := context.Background()
|
|
|
|
// Create default bucket
|
|
if err := connManager.CreateBucket(ctx, "files"); err != nil {
|
|
return fmt.Errorf("failed to create files bucket: %w", err)
|
|
}
|
|
|
|
// Create hierarchical bucket
|
|
if err := connManager.CreateBucket(ctx, "documents"); err != nil {
|
|
return fmt.Errorf("failed to create documents bucket: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
```
|
|
|
|
### List All Buckets
|
|
|
|
```go
|
|
func listAllBuckets(connManager *minio.ConnectionManager) error {
|
|
buckets, err := connManager.ListBuckets(context.Background())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Printf("Available buckets:")
|
|
for _, bucket := range buckets {
|
|
log.Printf("- %s", bucket)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
```
|
|
|
|
## Error Handling
|
|
|
|
```go
|
|
func handleUploadError(err error) {
|
|
switch {
|
|
case errors.Is(err, minio.ErrInvalidConfig):
|
|
log.Println("Configuration error:", err)
|
|
case errors.Is(err, minio.ErrNotConnected):
|
|
log.Println("Connection error:", err)
|
|
case errors.Is(err, minio.ErrUploadFailed):
|
|
log.Println("Upload failed:", err)
|
|
case errors.Is(err, minio.ErrFileTooLarge):
|
|
log.Println("File too large:", err)
|
|
case errors.Is(err, minio.ErrInvalidFileType):
|
|
log.Println("Invalid file type:", err)
|
|
default:
|
|
log.Println("Unknown error:", err)
|
|
}
|
|
}
|
|
```
|
|
|
|
## Integration with HTTP Handlers
|
|
|
|
### File Upload Handler
|
|
|
|
```go
|
|
func uploadHandler(service *minio.Service) gin.HandlerFunc {
|
|
return func(c *gin.Context) {
|
|
// Get uploaded file
|
|
file, err := c.FormFile("file")
|
|
if err != nil {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": "No file provided"})
|
|
return
|
|
}
|
|
|
|
// Open uploaded file
|
|
src, err := file.Open()
|
|
if err != nil {
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to open file"})
|
|
return
|
|
}
|
|
defer src.Close()
|
|
|
|
// Generate unique filename
|
|
filename := fmt.Sprintf("%d_%s", time.Now().Unix(), file.Filename)
|
|
objectName := fmt.Sprintf("uploads/%s", filename)
|
|
|
|
// Upload file
|
|
err = service.Upload(c.Request.Context(), "files", objectName, src, minio.UploadOptions{
|
|
ContentType: file.Header.Get("Content-Type"),
|
|
Metadata: map[string]string{
|
|
"original_name": file.Filename,
|
|
"uploaded_by": "user_id", // Get from context
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
handleUploadError(err)
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "Upload failed"})
|
|
return
|
|
}
|
|
|
|
c.JSON(http.StatusOK, gin.H{
|
|
"message": "File uploaded successfully",
|
|
"filename": filename,
|
|
"path": objectName,
|
|
})
|
|
}
|
|
}
|
|
```
|
|
|
|
### Document Upload Handler
|
|
|
|
```go
|
|
func uploadDocumentHandler(service *minio.Service) gin.HandlerFunc {
|
|
return func(c *gin.Context) {
|
|
// Get form data
|
|
category := c.PostForm("category")
|
|
subCategory := c.PostForm("subcategory")
|
|
tags := c.PostFormArray("tags[]")
|
|
|
|
// Get uploaded file
|
|
file, err := c.FormFile("document")
|
|
if err != nil {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": "No document provided"})
|
|
return
|
|
}
|
|
|
|
// Open document
|
|
src, err := file.Open()
|
|
if err != nil {
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to open document"})
|
|
return
|
|
}
|
|
defer src.Close()
|
|
|
|
// Upload to hierarchical storage
|
|
docPath, err := service.UploadDocument(c.Request.Context(), src, file.Filename, minio.HierarchicalUploadOptions{
|
|
UploadOptions: minio.UploadOptions{
|
|
ContentType: file.Header.Get("Content-Type"),
|
|
Metadata: map[string]string{
|
|
"uploaded_by": "user_id", // Get from context
|
|
"department": c.PostForm("department"),
|
|
},
|
|
},
|
|
Category: category,
|
|
SubCategory: subCategory,
|
|
Tags: tags,
|
|
})
|
|
|
|
if err != nil {
|
|
handleUploadError(err)
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "Document upload failed"})
|
|
return
|
|
}
|
|
|
|
c.JSON(http.StatusOK, gin.H{
|
|
"message": "Document uploaded successfully",
|
|
"path": docPath,
|
|
})
|
|
}
|
|
}
|
|
```
|
|
|
|
## Configuration Examples
|
|
|
|
### Development Configuration
|
|
|
|
```go
|
|
config := minio.Config{
|
|
Endpoint: "localhost:9000",
|
|
AccessKeyID: "minioadmin",
|
|
SecretAccessKey: "minioadmin",
|
|
UseSSL: false,
|
|
Region: "us-east-1",
|
|
DefaultBucket: "files",
|
|
HierarchicalBucket: "documents",
|
|
MaxFileSize: 50 * 1024 * 1024, // 50MB
|
|
AllowedTypes: []string{"image/", "application/pdf", "text/"},
|
|
}
|
|
```
|
|
|
|
### Production Configuration
|
|
|
|
```go
|
|
config := minio.Config{
|
|
Endpoint: "minio.example.com:443",
|
|
AccessKeyID: os.Getenv("MINIO_ACCESS_KEY"),
|
|
SecretAccessKey: os.Getenv("MINIO_SECRET_KEY"),
|
|
UseSSL: true,
|
|
Region: "us-west-2",
|
|
DefaultBucket: "production-files",
|
|
HierarchicalBucket: "production-documents",
|
|
MaxFileSize: 100 * 1024 * 1024, // 100MB
|
|
AllowedTypes: []string{"image/jpeg", "image/png", "application/pdf"},
|
|
ConnectTimeout: 30 * time.Second,
|
|
RequestTimeout: 60 * time.Second,
|
|
}
|
|
```
|