diff --git a/JIRA_TASK_RULES.md b/JIRA_TASK_RULES.md index b1c55e8..4569fa3 100644 --- a/JIRA_TASK_RULES.md +++ b/JIRA_TASK_RULES.md @@ -24,7 +24,7 @@ Always set appropriate labels based on task type: ### 4. **Team Assignment by Role** - **Application Developer:** a.ghabeli -- **Panel Developer:** a.jamali, sina_nokhostin +- **Panel Developer:** a.jamali - **Backend Developer:** n.nakhostin - **QA:** a.sheikhmemari - **Designer:** s.fathali @@ -67,4 +67,4 @@ Always set appropriate labels based on task type: --- *This document serves as the standard for all Jira task creation in the TM project.* -*Last updated: September 17, 2025* +*Last updated: October 9, 2025* diff --git a/go.mod b/go.mod index 23b995c..122c2cd 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.24.4 require ( github.com/golang-jwt/jwt/v5 v5.3.0 github.com/labstack/echo/v4 v4.13.4 + github.com/minio/minio-go/v7 v7.0.97 github.com/redis/go-redis/v9 v9.14.0 github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.10.0 @@ -23,16 +24,26 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-openapi/jsonpointer v0.21.1 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/spec v0.21.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/mailru/easyjson v0.9.0 // indirect + github.com/minio/crc64nvme v1.1.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/swaggo/files/v2 v2.0.2 // indirect + github.com/tinylib/msgp v1.3.0 // indirect golang.org/x/mod v0.26.0 // indirect golang.org/x/time v0.11.0 // indirect golang.org/x/tools v0.35.0 // indirect @@ -42,7 +53,7 @@ require ( require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/golang/snappy v1.0.0 // indirect - github.com/klauspost/compress v1.17.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/go.sum b/go.sum index dbec14f..2eaa228 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -28,10 +32,17 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -46,6 +57,14 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= +github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= +github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.14.0 h1:u4tNCjXOyzfgeLN+vAZaW1xUooqWDqVEsZN0U01jfAE= @@ -54,6 +73,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -66,6 +87,8 @@ github.com/swaggo/files/v2 v2.0.2 h1:Bq4tgS/yxLB/3nwOMcul5oLEUKa877Ykgz3CJMVbQKU github.com/swaggo/files/v2 v2.0.2/go.mod h1:TVqetIzZsO9OhHX1Am9sRf9LdrFZqoK49N37KON/jr0= github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= diff --git a/pkg/minio/README.md b/pkg/minio/README.md new file mode 100644 index 0000000..1a309a2 --- /dev/null +++ b/pkg/minio/README.md @@ -0,0 +1,532 @@ +# MinIO Storage Package + +A comprehensive Go package for MinIO object storage with support for hierarchical document storage and general file management. This package provides both normal file storage for images, videos, and other media files, as well as classified hierarchical storage for documents. + +## Features + +✅ **Hierarchical Document Storage** - Organize documents in categorized folders with metadata +✅ **Normal File Storage** - Standard object storage for images, videos, and other files +✅ **File Upload/Download** - Stream-based operations with progress tracking +✅ **Metadata Management** - Rich metadata support for classification and search +✅ **Presigned URLs** - Temporary access URLs for secure file sharing +✅ **Bucket Management** - Create, delete, and manage storage buckets +✅ **File Operations** - Copy, move, delete, and list files +✅ **Connection Management** - Robust connection pooling and health checks +✅ **Structured Logging** - Comprehensive logging with structured fields +✅ **Error Handling** - Detailed error types and context +✅ **Type Safety** - Full Go generics support + +## Installation + +```bash +go get github.com/minio/minio-go/v7 +``` + +## Quick Start + +### 1. Configuration + +```go +import "path/to/pkg/minio" + +// Create configuration +config := minio.Config{ + Endpoint: "localhost:9000", + AccessKeyID: "your-access-key", + SecretAccessKey: "your-secret-key", + UseSSL: false, + Region: "us-east-1", + DefaultBucket: "files", + HierarchicalBucket: "documents", +} + +// Validate configuration +if err := config.Validate(); err != nil { + log.Fatal(err) +} +``` + +### 2. Initialize Service + +```go +// Create logger (implement minio.Logger interface) +logger := &YourLogger{} + +// Create connection manager +connManager, err := minio.NewConnectionManager(config, logger) +if err != nil { + log.Fatal(err) +} + +// Connect to MinIO +if err := connManager.Connect(); err != nil { + log.Fatal(err) +} +defer connManager.Disconnect(context.Background()) + +// Create service +service := minio.NewService(connManager, config, logger) +``` + +### 3. Normal File Storage + +```go +// Upload a file +file, err := os.Open("image.jpg") +if err != nil { + log.Fatal(err) +} +defer file.Close() + +err = service.Upload(context.Background(), "files", "images/profile.jpg", file, minio.UploadOptions{ + ContentType: "image/jpeg", + Metadata: map[string]string{ + "user_id": "123", + "type": "profile", + }, +}) + +// Download a file +reader, err := service.Download(context.Background(), "files", "images/profile.jpg", minio.DownloadOptions{}) +if err != nil { + log.Fatal(err) +} +defer reader.Close() + +// Copy file data to response +io.Copy(w, reader) +``` + +### 4. Hierarchical Document Storage + +```go +// Upload a document with classification +docFile, err := os.Open("contract.pdf") +if err != nil { + log.Fatal(err) +} +defer docFile.Close() + +docPath, err := service.UploadDocument(context.Background(), docFile, "contract.pdf", minio.HierarchicalUploadOptions{ + UploadOptions: minio.UploadOptions{ + ContentType: "application/pdf", + Metadata: map[string]string{ + "title": "Service Contract", + "version": "1.0", + }, + }, + Category: "contracts", + SubCategory: "2024", + Tags: []string{"legal", "service"}, +}) + +fmt.Printf("Document uploaded to: %s\n", docPath) +// Output: contracts/2024/legal_service/1640995200/contract.pdf +``` + +## Storage Types + +### Normal Storage +- **Use Case**: Images, videos, general files +- **Structure**: Flat object storage with custom paths +- **Example**: `images/profile.jpg`, `videos/tutorial.mp4` + +### Hierarchical Storage +- **Use Case**: Documents requiring classification and organization +- **Structure**: Categorized folder structure with metadata +- **Example**: `contracts/2024/legal_service/1640995200/contract.pdf` + +## API Reference + +### File Operations + +#### Upload File +```go +func (s *Service) Upload(ctx context.Context, bucket, objectName string, reader io.Reader, options UploadOptions) error +``` + +#### Upload Hierarchical Document +```go +func (s *Service) UploadHierarchical(ctx context.Context, file io.Reader, filename string, options HierarchicalUploadOptions) (string, error) +``` + +#### Download File +```go +func (s *Service) Download(ctx context.Context, bucket, objectName string, options DownloadOptions) (io.ReadCloser, error) +``` + +#### Delete File +```go +func (s *Service) Delete(ctx context.Context, bucket, objectName string) error +``` + +#### Get File Info +```go +func (s *Service) GetInfo(ctx context.Context, bucket, objectName string) (*FileInfo, error) +``` + +#### List Files +```go +func (s *Service) List(ctx context.Context, bucket string, options ListOptions) (*ListResult, error) +``` + +#### Copy File +```go +func (s *Service) Copy(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) error +``` + +#### Move File +```go +func (s *Service) Move(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) error +``` + +### Document Operations + +#### List Documents +```go +func (s *Service) ListDocuments(ctx context.Context, category, subCategory string, options ListOptions) (*ListResult, error) +``` + +#### Search Documents +```go +func (s *Service) SearchDocuments(ctx context.Context, query string, category, subCategory string, options ListOptions) (*ListResult, error) +``` + +### Bucket Operations + +#### Create Bucket +```go +func (cm *ConnectionManager) CreateBucket(ctx context.Context, bucketName string) error +``` + +#### Delete Bucket +```go +func (cm *ConnectionManager) DeleteBucket(ctx context.Context, bucketName string) error +``` + +#### List Buckets +```go +func (cm *ConnectionManager) ListBuckets(ctx context.Context) ([]string, error) +``` + +## Configuration Options + +```go +type Config struct { + Endpoint string `json:"endpoint"` // MinIO server endpoint + AccessKeyID string `json:"accessKeyID"` // Access key + SecretAccessKey string `json:"secretAccessKey"` // Secret key + UseSSL bool `json:"useSSL"` // Use HTTPS + Region string `json:"region"` // Region + ConnectTimeout time.Duration `json:"connectTimeout"` // Connection timeout + RequestTimeout time.Duration `json:"requestTimeout"` // Request timeout + DefaultBucket string `json:"defaultBucket"` // Default bucket for normal files + HierarchicalBucket string `json:"hierarchicalBucket"` // Bucket for hierarchical storage + PathSeparator string `json:"pathSeparator"` // Path separator (default: "/") + MaxFileSize int64 `json:"maxFileSize"` // Max file size in bytes + AllowedTypes []string `json:"allowedTypes"` // Allowed MIME types + ChunkSize int64 `json:"chunkSize"` // Upload chunk size +} +``` + +## Hierarchical Storage Structure + +Documents are automatically organized in a hierarchical structure: + +``` +{hierarchicalBucket}/ +├── {category}/ +│ ├── {subCategory}/ +│ │ ├── {tags}/ +│ │ │ ├── {timestamp}/ +│ │ │ │ └── {filename} +│ │ │ └── ... +│ │ └── ... +│ └── ... +└── ... +``` + +**Example:** +``` +documents/ +├── contracts/ +│ ├── 2024/ +│ │ ├── legal_service/ +│ │ │ ├── 1640995200/ +│ │ │ │ └── contract.pdf +│ │ │ └── 1640995300/ +│ │ │ └── amendment.pdf +│ │ └── financial/ +│ │ └── 1640995400/ +│ │ └── invoice.pdf +│ └── 2023/ +│ └── legal/ +│ └── 1638360000/ +│ └── old_contract.pdf +└── reports/ + └── monthly/ + └── 1640995500/ + └── sales_report.pdf +``` + +## Metadata Management + +Files can have rich metadata attached: + +```go +metadata := map[string]string{ + "user_id": "12345", + "category": "contracts", + "subcategory": "2024", + "tags": "legal,service", + "version": "1.0", + "title": "Service Contract", + "uploaded_by": "john.doe", + "department": "legal", +} +``` + +## Error Handling + +The package provides specific error types: + +```go +var ( + ErrInvalidConfig = errors.New("invalid configuration") + ErrNotConnected = errors.New("not connected to MinIO") + ErrBucketNotFound = errors.New("bucket not found") + ErrObjectNotFound = errors.New("object not found") + ErrBucketAlreadyExists = errors.New("bucket already exists") + ErrUploadFailed = errors.New("upload failed") + ErrDownloadFailed = errors.New("download failed") + ErrFileTooLarge = errors.New("file too large") + ErrInvalidFileType = errors.New("invalid file type") +) + +// Handle specific errors +file, err := service.GetInfo(ctx, bucket, objectName) +if err != nil { + if errors.Is(err, minio.ErrObjectNotFound) { + // Handle not found + return + } + // Handle other errors +} +``` + +## Presigned URLs + +Generate temporary access URLs: + +```go +// Generate URL valid for 1 hour +url, err := service.GetPresignedURL(context.Background(), "files", "document.pdf", 3600) +if err != nil { + log.Fatal(err) +} + +fmt.Printf("Access URL: %s\n", url) +// Output: https://minio.example.com/files/document.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&... +``` + +## Logger Interface + +Implement structured logging: + +```go +type YourLogger struct{} + +func (l *YourLogger) Info(message string, fields map[string]interface{}) { + log.Printf("[INFO] %s: %+v", message, fields) +} + +func (l *YourLogger) Error(message string, fields map[string]interface{}) { + log.Printf("[ERROR] %s: %+v", message, fields) +} + +func (l *YourLogger) Debug(message string, fields map[string]interface{}) { + log.Printf("[DEBUG] %s: %+v", message, fields) +} + +func (l *YourLogger) Warn(message string, fields map[string]interface{}) { + log.Printf("[WARN] %s: %+v", message, fields) +} +``` + +## Performance Considerations + +### 1. Connection Pooling +Configure appropriate connection pool settings: + +```go +config := minio.Config{ + // ... other settings + ConnectTimeout: 10 * time.Second, + RequestTimeout: 30 * time.Second, +} +``` + +### 2. Chunked Uploads +Large files are automatically chunked for efficient upload: + +```go +config := minio.Config{ + MaxFileSize: 100 * 1024 * 1024, // 100MB + ChunkSize: 5 * 1024 * 1024, // 5MB chunks +} +``` + +### 3. Streaming Operations +Use streaming for large files to minimize memory usage: + +```go +// Streaming upload +file, _ := os.Open("large_file.zip") +defer file.Close() + +err := service.Upload(ctx, bucket, objectName, file, options) + +// Streaming download +reader, _ := service.Download(ctx, bucket, objectName, options) +defer reader.Close() + +io.Copy(destination, reader) +``` + +## Security Best Practices + +### 1. Access Control +- Use IAM policies to restrict access +- Generate presigned URLs for temporary access +- Never expose access keys in client-side code + +### 2. Input Validation +```go +// Validate file type and size +if err := service.validateFileType(contentType); err != nil { + return err +} + +if err := service.validateFileSize(fileSize); err != nil { + return err +} +``` + +### 3. Secure Configuration +- Store credentials securely (environment variables, secret management) +- Use HTTPS in production +- Rotate access keys regularly + +## Testing + +### Unit Testing + +```go +func TestService_Upload(t *testing.T) { + // Create mock connection manager + mockConn := &MockConnectionManager{} + + // Create service + service := minio.NewService(mockConn, config, logger) + + // Test upload + reader := strings.NewReader("test content") + err := service.Upload(context.Background(), "test-bucket", "test-object", reader, minio.UploadOptions{}) + + assert.NoError(t, err) +} +``` + +### Integration Testing + +```go +func TestMinIOIntegration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + // Set up MinIO test server + // ... test implementation +} +``` + +## Examples + +### Image Upload with Metadata +```go +func uploadProfileImage(userID string, imageFile io.Reader) error { + objectName := fmt.Sprintf("profiles/%s/avatar.jpg", userID) + + return service.Upload(context.Background(), "files", objectName, imageFile, minio.UploadOptions{ + ContentType: "image/jpeg", + Metadata: map[string]string{ + "user_id": userID, + "type": "profile", + "uploaded": fmt.Sprintf("%d", time.Now().Unix()), + }, + }) +} +``` + +### Document Management System +```go +func uploadContract(contractFile io.Reader, contractData ContractData) (string, error) { + return service.UploadDocument(context.Background(), contractFile, contractData.Filename, minio.HierarchicalUploadOptions{ + UploadOptions: minio.UploadOptions{ + ContentType: "application/pdf", + Metadata: map[string]string{ + "contract_id": contractData.ID, + "client_id": contractData.ClientID, + "value": fmt.Sprintf("%.2f", contractData.Value), + "signed": fmt.Sprintf("%v", contractData.Signed), + }, + }, + Category: "contracts", + SubCategory: contractData.Year, + Tags: []string{"legal", contractData.Type}, + }) +} + +func listContractsByYear(year string) (*minio.ListResult, error) { + return service.ListDocuments(context.Background(), "contracts", year, minio.ListOptions{ + MaxKeys: 100, + }) +} +``` + +## Migration Guide + +### From Direct MinIO Usage + +```go +// Before +client, err := minio.New(endpoint, &minio.Options{...}) +_, err = client.PutObject(ctx, bucket, objectName, reader, -1, minio.PutObjectOptions{...}) + +// After +service := minio.NewService(connManager, config, logger) +err := service.Upload(ctx, bucket, objectName, reader, options) +``` + +### From Other Storage Solutions + +```go +// From AWS S3 SDK +// Before: s3Client.PutObject(...) + +// After +service := minio.NewService(connManager, config, logger) +err := service.Upload(ctx, bucket, objectName, reader, options) +``` + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Add tests for new functionality +4. Ensure all tests pass +5. Submit a pull request + +## License + +This project is licensed under the MIT License. diff --git a/pkg/minio/USAGE.md b/pkg/minio/USAGE.md new file mode 100644 index 0000000..59f0b96 --- /dev/null +++ b/pkg/minio/USAGE.md @@ -0,0 +1,460 @@ +# MinIO Package Usage Examples + +## Basic Setup + +```go +package main + +import ( + "context" + "log" + "path/to/pkg/minio" +) + +// Implement the Logger interface +type SimpleLogger struct{} + +func (l *SimpleLogger) Info(message string, fields map[string]interface{}) { + log.Printf("[INFO] %s: %+v", message, fields) +} + +func (l *SimpleLogger) Error(message string, fields map[string]interface{}) { + log.Printf("[ERROR] %s: %+v", message, fields) +} + +func (l *SimpleLogger) Debug(message string, fields map[string]interface{}) { + log.Printf("[DEBUG] %s: %+v", message, fields) +} + +func (l *SimpleLogger) Warn(message string, fields map[string]interface{}) { + log.Printf("[WARN] %s: %+v", message, fields) +} + +func main() { + // Create configuration + config := minio.Config{ + Endpoint: "localhost:9000", + AccessKeyID: "minioadmin", + SecretAccessKey: "minioadmin", + UseSSL: false, + DefaultBucket: "files", + HierarchicalBucket: "documents", + } + + // Create logger + logger := &SimpleLogger{} + + // Create connection manager + connManager, err := minio.NewConnectionManager(config, logger) + if err != nil { + log.Fatal("Failed to create connection manager:", err) + } + + // Connect to MinIO + if err := connManager.Connect(); err != nil { + log.Fatal("Failed to connect to MinIO:", err) + } + defer connManager.Disconnect(context.Background()) + + // Create service + service := minio.NewService(connManager, config, logger) + + // Use the service... +} +``` + +## Normal File Storage + +### Upload an Image + +```go +func uploadImage(service *minio.Service) error { + // Open image file + file, err := os.Open("profile.jpg") + if err != nil { + return err + } + defer file.Close() + + // Upload to normal storage + return service.Upload(context.Background(), "files", "images/profile.jpg", file, minio.UploadOptions{ + ContentType: "image/jpeg", + Metadata: map[string]string{ + "user_id": "12345", + "type": "profile_image", + }, + }) +} +``` + +### Download and Serve a File + +```go +func serveFile(service *minio.Service, w http.ResponseWriter, bucket, objectName string) error { + // Download file + reader, err := service.Download(context.Background(), bucket, objectName, minio.DownloadOptions{}) + if err != nil { + return err + } + defer reader.Close() + + // Get file info for headers + info, err := service.GetInfo(context.Background(), bucket, objectName) + if err != nil { + return err + } + + // Set response headers + w.Header().Set("Content-Type", info.ContentType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size)) + + // Copy file content to response + _, err = io.Copy(w, reader) + return err +} +``` + +### Upload with Stream Processing + +```go +func uploadWithProgress(service *minio.Service, file io.Reader, filename string) error { + // Create a progress reader + progressReader := &ProgressReader{ + Reader: file, + Total: 0, // Set total size if known + Reporter: func(bytesRead int64) { + log.Printf("Uploaded %d bytes", bytesRead) + }, + } + + return service.Upload(context.Background(), "files", "uploads/"+filename, progressReader, minio.UploadOptions{ + ContentType: mime.TypeByExtension(filepath.Ext(filename)), + }) +} + +type ProgressReader struct { + Reader io.Reader + Total int64 + Reporter func(bytesRead int64) +} + +func (pr *ProgressReader) Read(p []byte) (int, error) { + n, err := pr.Reader.Read(p) + if n > 0 && pr.Reporter != nil { + pr.Total += int64(n) + pr.Reporter(pr.Total) + } + return n, err +} +``` + +## Hierarchical Document Storage + +### Upload a Contract + +```go +func uploadContract(service *minio.Service) error { + // Open contract file + file, err := os.Open("contract.pdf") + if err != nil { + return err + } + defer file.Close() + + // Upload to hierarchical storage + docPath, err := service.UploadDocument(context.Background(), file, "service_contract.pdf", minio.HierarchicalUploadOptions{ + UploadOptions: minio.UploadOptions{ + ContentType: "application/pdf", + Metadata: map[string]string{ + "contract_id": "CTR-2024-001", + "client_id": "CLIENT-123", + "value": "50000.00", + "signed": "true", + "department": "legal", + }, + }, + Category: "contracts", + SubCategory: "2024", + Tags: []string{"legal", "service", "active"}, + }) + + if err != nil { + return err + } + + log.Printf("Contract uploaded to: %s", docPath) + // Output: contracts/2024/legal_service_active/1640995200/service_contract.pdf + + return nil +} +``` + +### List Documents by Category + +```go +func listContractsByYear(service *minio.Service, year string) error { + result, err := service.ListDocuments(context.Background(), "contracts", year, minio.ListOptions{ + MaxKeys: 50, + }) + if err != nil { + return err + } + + log.Printf("Found %d documents in contracts/%s", len(result.Objects), year) + + for _, doc := range result.Objects { + log.Printf("- %s (%d bytes)", doc.Name, doc.Size) + } + + return nil +} +``` + +### Search Documents + +```go +func searchLegalDocuments(service *minio.Service) error { + result, err := service.SearchDocuments(context.Background(), "legal", "contracts", "2024", minio.ListOptions{ + MaxKeys: 20, + }) + if err != nil { + return err + } + + log.Printf("Found %d legal documents", len(result.Objects)) + + return nil +} +``` + +## File Operations + +### Copy Files Between Buckets + +```go +func backupFile(service *minio.Service, srcBucket, srcObject, destBucket, destObject string) error { + return service.Copy(context.Background(), srcBucket, srcObject, destBucket, destObject) +} +``` + +### Generate Presigned URLs + +```go +func generateDownloadLink(service *minio.Service, bucket, objectName string) (string, error) { + // Generate URL valid for 1 hour + return service.GetPresignedURL(context.Background(), bucket, objectName, 3600) +} +``` + +### Check File Existence + +```go +func fileExists(service *minio.Service, bucket, objectName string) bool { + exists, err := service.Exists(context.Background(), bucket, objectName) + if err != nil { + log.Printf("Error checking file existence: %v", err) + return false + } + return exists +} +``` + +## Bucket Management + +### Create Required Buckets + +```go +func ensureBucketsExist(connManager *minio.ConnectionManager) error { + ctx := context.Background() + + // Create default bucket + if err := connManager.CreateBucket(ctx, "files"); err != nil { + return fmt.Errorf("failed to create files bucket: %w", err) + } + + // Create hierarchical bucket + if err := connManager.CreateBucket(ctx, "documents"); err != nil { + return fmt.Errorf("failed to create documents bucket: %w", err) + } + + return nil +} +``` + +### List All Buckets + +```go +func listAllBuckets(connManager *minio.ConnectionManager) error { + buckets, err := connManager.ListBuckets(context.Background()) + if err != nil { + return err + } + + log.Printf("Available buckets:") + for _, bucket := range buckets { + log.Printf("- %s", bucket) + } + + return nil +} +``` + +## Error Handling + +```go +func handleUploadError(err error) { + switch { + case errors.Is(err, minio.ErrInvalidConfig): + log.Println("Configuration error:", err) + case errors.Is(err, minio.ErrNotConnected): + log.Println("Connection error:", err) + case errors.Is(err, minio.ErrUploadFailed): + log.Println("Upload failed:", err) + case errors.Is(err, minio.ErrFileTooLarge): + log.Println("File too large:", err) + case errors.Is(err, minio.ErrInvalidFileType): + log.Println("Invalid file type:", err) + default: + log.Println("Unknown error:", err) + } +} +``` + +## Integration with HTTP Handlers + +### File Upload Handler + +```go +func uploadHandler(service *minio.Service) gin.HandlerFunc { + return func(c *gin.Context) { + // Get uploaded file + file, err := c.FormFile("file") + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "No file provided"}) + return + } + + // Open uploaded file + src, err := file.Open() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to open file"}) + return + } + defer src.Close() + + // Generate unique filename + filename := fmt.Sprintf("%d_%s", time.Now().Unix(), file.Filename) + objectName := fmt.Sprintf("uploads/%s", filename) + + // Upload file + err = service.Upload(c.Request.Context(), "files", objectName, src, minio.UploadOptions{ + ContentType: file.Header.Get("Content-Type"), + Metadata: map[string]string{ + "original_name": file.Filename, + "uploaded_by": "user_id", // Get from context + }, + }) + + if err != nil { + handleUploadError(err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Upload failed"}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "message": "File uploaded successfully", + "filename": filename, + "path": objectName, + }) + } +} +``` + +### Document Upload Handler + +```go +func uploadDocumentHandler(service *minio.Service) gin.HandlerFunc { + return func(c *gin.Context) { + // Get form data + category := c.PostForm("category") + subCategory := c.PostForm("subcategory") + tags := c.PostFormArray("tags[]") + + // Get uploaded file + file, err := c.FormFile("document") + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "No document provided"}) + return + } + + // Open document + src, err := file.Open() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to open document"}) + return + } + defer src.Close() + + // Upload to hierarchical storage + docPath, err := service.UploadDocument(c.Request.Context(), src, file.Filename, minio.HierarchicalUploadOptions{ + UploadOptions: minio.UploadOptions{ + ContentType: file.Header.Get("Content-Type"), + Metadata: map[string]string{ + "uploaded_by": "user_id", // Get from context + "department": c.PostForm("department"), + }, + }, + Category: category, + SubCategory: subCategory, + Tags: tags, + }) + + if err != nil { + handleUploadError(err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Document upload failed"}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "message": "Document uploaded successfully", + "path": docPath, + }) + } +} +``` + +## Configuration Examples + +### Development Configuration + +```go +config := minio.Config{ + Endpoint: "localhost:9000", + AccessKeyID: "minioadmin", + SecretAccessKey: "minioadmin", + UseSSL: false, + Region: "us-east-1", + DefaultBucket: "files", + HierarchicalBucket: "documents", + MaxFileSize: 50 * 1024 * 1024, // 50MB + AllowedTypes: []string{"image/", "application/pdf", "text/"}, +} +``` + +### Production Configuration + +```go +config := minio.Config{ + Endpoint: "minio.example.com:443", + AccessKeyID: os.Getenv("MINIO_ACCESS_KEY"), + SecretAccessKey: os.Getenv("MINIO_SECRET_KEY"), + UseSSL: true, + Region: "us-west-2", + DefaultBucket: "production-files", + HierarchicalBucket: "production-documents", + MaxFileSize: 100 * 1024 * 1024, // 100MB + AllowedTypes: []string{"image/jpeg", "image/png", "application/pdf"}, + ConnectTimeout: 30 * time.Second, + RequestTimeout: 60 * time.Second, +} +``` diff --git a/pkg/minio/config.go b/pkg/minio/config.go new file mode 100644 index 0000000..24cb379 --- /dev/null +++ b/pkg/minio/config.go @@ -0,0 +1,76 @@ +package minio + +import "time" + +// Config holds MinIO connection configuration +type Config struct { + Endpoint string `json:"endpoint" yaml:"endpoint"` + AccessKeyID string `json:"accessKeyID" yaml:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey" yaml:"secretAccessKey"` + UseSSL bool `json:"useSSL" yaml:"useSSL"` + Region string `json:"region" yaml:"region"` + + // Connection settings + ConnectTimeout time.Duration `json:"connectTimeout" yaml:"connectTimeout"` + RequestTimeout time.Duration `json:"requestTimeout" yaml:"requestTimeout"` + + // Bucket settings + DefaultBucket string `json:"defaultBucket" yaml:"defaultBucket"` + + // Hierarchical storage settings + HierarchicalBucket string `json:"hierarchicalBucket" yaml:"hierarchicalBucket"` + PathSeparator string `json:"pathSeparator" yaml:"pathSeparator"` + + // Upload settings + MaxFileSize int64 `json:"maxFileSize" yaml:"maxFileSize"` // Maximum file size in bytes + AllowedTypes []string `json:"allowedTypes" yaml:"allowedTypes"` // Allowed MIME types + ChunkSize int64 `json:"chunkSize" yaml:"chunkSize"` // Chunk size for multipart uploads +} + +// DefaultConfig returns a default MinIO configuration +func DefaultConfig() Config { + return Config{ + UseSSL: true, + Region: "us-east-1", + ConnectTimeout: 10 * time.Second, + RequestTimeout: 30 * time.Second, + DefaultBucket: "files", + HierarchicalBucket: "documents", + PathSeparator: "/", + MaxFileSize: 100 * 1024 * 1024, // 100MB + AllowedTypes: []string{"*"}, // Allow all types by default + ChunkSize: 5 * 1024 * 1024, // 5MB chunks + } +} + +// Validate validates the MinIO configuration +func (c *Config) Validate() error { + if c.Endpoint == "" { + return NewErrorWithCause(ErrCodeInvalidConfig, "endpoint is required", ErrInvalidConfig) + } + if c.AccessKeyID == "" { + return NewErrorWithCause(ErrCodeInvalidConfig, "accessKeyID is required", ErrInvalidConfig) + } + if c.SecretAccessKey == "" { + return NewErrorWithCause(ErrCodeInvalidConfig, "secretAccessKey is required", ErrInvalidConfig) + } + if c.DefaultBucket == "" { + return NewErrorWithCause(ErrCodeInvalidConfig, "defaultBucket is required", ErrInvalidConfig) + } + if c.HierarchicalBucket == "" { + return NewErrorWithCause(ErrCodeInvalidConfig, "hierarchicalBucket is required", ErrInvalidConfig) + } + if c.PathSeparator == "" { + c.PathSeparator = "/" + } + if c.MaxFileSize <= 0 { + c.MaxFileSize = 100 * 1024 * 1024 // 100MB default + } + if len(c.AllowedTypes) == 0 { + c.AllowedTypes = []string{"*"} + } + if c.ChunkSize <= 0 { + c.ChunkSize = 5 * 1024 * 1024 // 5MB default + } + return nil +} diff --git a/pkg/minio/connection.go b/pkg/minio/connection.go new file mode 100644 index 0000000..b38fa38 --- /dev/null +++ b/pkg/minio/connection.go @@ -0,0 +1,252 @@ +package minio + +import ( + "context" + "net/http" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// ConnectionManager manages MinIO connections and operations +type ConnectionManager struct { + client *minio.Client + config Config + logger Logger +} + +// NewConnectionManager creates a new MinIO connection manager +func NewConnectionManager(config Config, logger Logger) (*ConnectionManager, error) { + if err := config.Validate(); err != nil { + return nil, err + } + + return &ConnectionManager{ + config: config, + logger: logger, + }, nil +} + +// Connect establishes connection to MinIO +func (cm *ConnectionManager) Connect() error { + // Create MinIO client + minioClient, err := minio.New(cm.config.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cm.config.AccessKeyID, cm.config.SecretAccessKey, ""), + Secure: cm.config.UseSSL, + Region: cm.config.Region, + Transport: &http.Transport{ + ResponseHeaderTimeout: cm.config.RequestTimeout, + }, + }) + if err != nil { + cm.logger.Error("Failed to create MinIO client", map[string]interface{}{ + "endpoint": cm.config.Endpoint, + "error": err.Error(), + }) + return NewErrorWithCause(ErrCodeConnectionFailed, "failed to create MinIO client", err) + } + + cm.client = minioClient + + // Test connection by listing buckets + ctx, cancel := context.WithTimeout(context.Background(), cm.config.ConnectTimeout) + defer cancel() + + _, err = cm.client.ListBuckets(ctx) + if err != nil { + cm.logger.Error("Failed to connect to MinIO", map[string]interface{}{ + "endpoint": cm.config.Endpoint, + "error": err.Error(), + }) + return NewErrorWithCause(ErrCodeConnectionFailed, "failed to connect to MinIO", err) + } + + cm.logger.Info("Successfully connected to MinIO", map[string]interface{}{ + "endpoint": cm.config.Endpoint, + "region": cm.config.Region, + }) + + return nil +} + +// Disconnect closes the MinIO connection +func (cm *ConnectionManager) Disconnect(ctx context.Context) error { + if cm.client == nil { + return nil + } + + // MinIO client doesn't have a disconnect method, but we can set it to nil + cm.client = nil + + cm.logger.Info("Disconnected from MinIO", nil) + return nil +} + +// Ping tests the MinIO connection +func (cm *ConnectionManager) Ping(ctx context.Context) error { + if cm.client == nil { + return ErrNotConnected + } + + _, err := cm.client.ListBuckets(ctx) + if err != nil { + cm.logger.Error("MinIO ping failed", map[string]interface{}{ + "error": err.Error(), + }) + return NewErrorWithCause(ErrCodeConnectionFailed, "ping failed", err) + } + + return nil +} + +// IsConnected returns true if connected to MinIO +func (cm *ConnectionManager) IsConnected() bool { + return cm.client != nil +} + +// GetClient returns the underlying MinIO client +func (cm *ConnectionManager) GetClient() interface{} { + return cm.client +} + +// CreateBucket creates a new bucket if it doesn't exist +func (cm *ConnectionManager) CreateBucket(ctx context.Context, bucketName string) error { + if cm.client == nil { + return ErrNotConnected + } + + if bucketName == "" { + return ErrInvalidBucketName + } + + exists, err := cm.client.BucketExists(ctx, bucketName) + if err != nil { + cm.logger.Error("Failed to check bucket existence", map[string]interface{}{ + "bucket": bucketName, + "error": err.Error(), + }) + return NewErrorWithCause(ErrCodeBucketNotFound, "failed to check bucket existence", err) + } + + if exists { + cm.logger.Debug("Bucket already exists", map[string]interface{}{ + "bucket": bucketName, + }) + return nil + } + + err = cm.client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{ + Region: cm.config.Region, + }) + if err != nil { + cm.logger.Error("Failed to create bucket", map[string]interface{}{ + "bucket": bucketName, + "error": err.Error(), + }) + return NewErrorWithCause("BUCKET_CREATE_FAILED", "failed to create bucket", err) + } + + cm.logger.Info("Bucket created successfully", map[string]interface{}{ + "bucket": bucketName, + }) + + return nil +} + +// DeleteBucket deletes a bucket +func (cm *ConnectionManager) DeleteBucket(ctx context.Context, bucketName string) error { + if cm.client == nil { + return ErrNotConnected + } + + if bucketName == "" { + return ErrInvalidBucketName + } + + err := cm.client.RemoveBucket(ctx, bucketName) + if err != nil { + cm.logger.Error("Failed to delete bucket", map[string]interface{}{ + "bucket": bucketName, + "error": err.Error(), + }) + return NewErrorWithCause("BUCKET_DELETE_FAILED", "failed to delete bucket", err) + } + + cm.logger.Info("Bucket deleted successfully", map[string]interface{}{ + "bucket": bucketName, + }) + + return nil +} + +// BucketExists checks if a bucket exists +func (cm *ConnectionManager) BucketExists(ctx context.Context, bucketName string) (bool, error) { + if cm.client == nil { + return false, ErrNotConnected + } + + if bucketName == "" { + return false, ErrInvalidBucketName + } + + exists, err := cm.client.BucketExists(ctx, bucketName) + if err != nil { + cm.logger.Error("Failed to check bucket existence", map[string]interface{}{ + "bucket": bucketName, + "error": err.Error(), + }) + return false, NewErrorWithCause(ErrCodeBucketNotFound, "failed to check bucket existence", err) + } + + return exists, nil +} + +// ListBuckets lists all buckets +func (cm *ConnectionManager) ListBuckets(ctx context.Context) ([]string, error) { + if cm.client == nil { + return nil, ErrNotConnected + } + + buckets, err := cm.client.ListBuckets(ctx) + if err != nil { + cm.logger.Error("Failed to list buckets", map[string]interface{}{ + "error": err.Error(), + }) + return nil, NewErrorWithCause("LIST_BUCKETS_FAILED", "failed to list buckets", err) + } + + bucketNames := make([]string, len(buckets)) + for i, bucket := range buckets { + bucketNames[i] = bucket.Name + } + + cm.logger.Debug("Buckets listed successfully", map[string]interface{}{ + "count": len(bucketNames), + }) + + return bucketNames, nil +} + +// GetBucketInfo gets bucket information +func (cm *ConnectionManager) GetBucketInfo(ctx context.Context, bucketName string) (map[string]interface{}, error) { + if cm.client == nil { + return nil, ErrNotConnected + } + + if bucketName == "" { + return nil, ErrInvalidBucketName + } + + // For now, return basic bucket info + // In a real implementation, you might want to get more detailed info + info := map[string]interface{}{ + "name": bucketName, + "region": cm.config.Region, + } + + cm.logger.Debug("Bucket info retrieved", map[string]interface{}{ + "bucket": bucketName, + }) + + return info, nil +} diff --git a/pkg/minio/errors.go b/pkg/minio/errors.go new file mode 100644 index 0000000..c5943b1 --- /dev/null +++ b/pkg/minio/errors.go @@ -0,0 +1,99 @@ +package minio + +import "errors" + +// Common errors +var ( + ErrInvalidConfig = errors.New("invalid configuration") + ErrNotConnected = errors.New("not connected to MinIO") + ErrBucketNotFound = errors.New("bucket not found") + ErrObjectNotFound = errors.New("object not found") + ErrBucketAlreadyExists = errors.New("bucket already exists") + ErrInvalidBucketName = errors.New("invalid bucket name") + ErrInvalidObjectName = errors.New("invalid object name") + ErrUploadFailed = errors.New("upload failed") + ErrDownloadFailed = errors.New("download failed") + ErrDeleteFailed = errors.New("delete failed") + ErrCopyFailed = errors.New("copy failed") + ErrMoveFailed = errors.New("move failed") + ErrInvalidRange = errors.New("invalid range") + ErrFileTooLarge = errors.New("file too large") + ErrInvalidFileType = errors.New("invalid file type") + ErrInvalidPath = errors.New("invalid path") + ErrAccessDenied = errors.New("access denied") + ErrTimeout = errors.New("operation timeout") + ErrInvalidCredentials = errors.New("invalid credentials") + ErrConnectionFailed = errors.New("connection failed") + ErrPresignFailed = errors.New("presigned URL generation failed") +) + +// Error wraps errors with additional context +type Error struct { + Code string + Message string + Cause error +} + +// Error implements the error interface +func (e *Error) Error() string { + if e.Cause != nil { + return e.Message + ": " + e.Cause.Error() + } + return e.Message +} + +// Unwrap returns the underlying error +func (e *Error) Unwrap() error { + return e.Cause +} + +// Wrap wraps an error with a message +func (e *Error) Wrap(msg string) *Error { + return &Error{ + Code: e.Code, + Message: msg, + Cause: e, + } +} + +// NewError creates a new error with code and message +func NewError(code, message string) *Error { + return &Error{ + Code: code, + Message: message, + } +} + +// NewErrorWithCause creates a new error with code, message and cause +func NewErrorWithCause(code, message string, cause error) *Error { + return &Error{ + Code: code, + Message: message, + Cause: cause, + } +} + +// Error codes +const ( + ErrCodeInvalidConfig = "INVALID_CONFIG" + ErrCodeNotConnected = "NOT_CONNECTED" + ErrCodeBucketNotFound = "BUCKET_NOT_FOUND" + ErrCodeObjectNotFound = "OBJECT_NOT_FOUND" + ErrCodeBucketExists = "BUCKET_EXISTS" + ErrCodeInvalidBucketName = "INVALID_BUCKET_NAME" + ErrCodeInvalidObjectName = "INVALID_OBJECT_NAME" + ErrCodeUploadFailed = "UPLOAD_FAILED" + ErrCodeDownloadFailed = "DOWNLOAD_FAILED" + ErrCodeDeleteFailed = "DELETE_FAILED" + ErrCodeCopyFailed = "COPY_FAILED" + ErrCodeMoveFailed = "MOVE_FAILED" + ErrCodeInvalidRange = "INVALID_RANGE" + ErrCodeFileTooLarge = "FILE_TOO_LARGE" + ErrCodeInvalidFileType = "INVALID_FILE_TYPE" + ErrCodeInvalidPath = "INVALID_PATH" + ErrCodeAccessDenied = "ACCESS_DENIED" + ErrCodeTimeout = "TIMEOUT" + ErrCodeInvalidCredentials = "INVALID_CREDENTIALS" + ErrCodeConnectionFailed = "CONNECTION_FAILED" + ErrCodePresignFailed = "PRESIGN_FAILED" +) diff --git a/pkg/minio/interfaces.go b/pkg/minio/interfaces.go new file mode 100644 index 0000000..de35cdd --- /dev/null +++ b/pkg/minio/interfaces.go @@ -0,0 +1,167 @@ +package minio + +import ( + "context" + "io" +) + +// Logger defines the interface for logging operations +type Logger interface { + Info(message string, fields map[string]interface{}) + Error(message string, fields map[string]interface{}) + Debug(message string, fields map[string]interface{}) + Warn(message string, fields map[string]interface{}) +} + +// FileInfo represents file information +type FileInfo struct { + Name string `json:"name"` + Path string `json:"path"` + Size int64 `json:"size"` + ContentType string `json:"contentType"` + ETag string `json:"etag"` + LastModified int64 `json:"lastModified"` + Metadata map[string]string `json:"metadata"` +} + +// UploadOptions represents upload configuration +type UploadOptions struct { + ContentType string `json:"contentType"` + Metadata map[string]string `json:"metadata"` +} + +// HierarchicalUploadOptions extends UploadOptions for hierarchical storage +type HierarchicalUploadOptions struct { + UploadOptions + Category string `json:"category"` // e.g., "contracts", "invoices", "reports" + SubCategory string `json:"subCategory"` // e.g., "2024", "Q1" + Tags []string `json:"tags"` // Additional classification tags +} + +// DownloadOptions represents download configuration +type DownloadOptions struct { + Range *DownloadRange `json:"range,omitempty"` +} + +// DownloadRange represents a byte range for partial downloads +type DownloadRange struct { + Start int64 `json:"start"` + End int64 `json:"end,omitempty"` // 0 means until end +} + +// ListOptions represents listing configuration +type ListOptions struct { + Prefix string `json:"prefix"` + Delimiter string `json:"delimiter"` + MaxKeys int32 `json:"maxKeys"` + Marker string `json:"marker"` +} + +// ListResult represents the result of a list operation +type ListResult struct { + Objects []FileInfo `json:"objects"` + CommonPrefixes []string `json:"commonPrefixes"` + IsTruncated bool `json:"isTruncated"` + NextMarker string `json:"nextMarker"` +} + +// StorageType represents different storage types +type StorageType string + +const ( + StorageTypeNormal StorageType = "normal" // For general files (images, videos, etc.) + StorageTypeHierarchical StorageType = "hierarchical" // For documents with classification +) + +// FileManager defines the interface for file operations +type FileManager interface { + // Upload uploads a file to normal storage + Upload(ctx context.Context, bucket, objectName string, reader io.Reader, options UploadOptions) error + + // UploadHierarchical uploads a file to hierarchical storage with classification + UploadHierarchical(ctx context.Context, file io.Reader, filename string, options HierarchicalUploadOptions) (string, error) + + // Download downloads a file + Download(ctx context.Context, bucket, objectName string, options DownloadOptions) (io.ReadCloser, error) + + // Delete deletes a file + Delete(ctx context.Context, bucket, objectName string) error + + // GetInfo gets file information + GetInfo(ctx context.Context, bucket, objectName string) (*FileInfo, error) + + // List lists files with optional prefix + List(ctx context.Context, bucket string, options ListOptions) (*ListResult, error) + + // Copy copies a file within the same bucket or to another bucket + Copy(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) error + + // Move moves a file (copy + delete) + Move(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) error + + // Exists checks if a file exists + Exists(ctx context.Context, bucket, objectName string) (bool, error) + + // GetPresignedURL generates a presigned URL for temporary access + GetPresignedURL(ctx context.Context, bucket, objectName string, expirySeconds int64) (string, error) +} + +// HierarchicalStorage defines the interface for hierarchical document storage +type HierarchicalStorage interface { + // UploadDocument uploads a document with hierarchical classification + UploadDocument(ctx context.Context, file io.Reader, filename string, options HierarchicalUploadOptions) (string, error) + + // ListDocuments lists documents in a category/subcategory + ListDocuments(ctx context.Context, category, subCategory string, options ListOptions) (*ListResult, error) + + // GetDocumentPath generates the hierarchical path for a document + GetDocumentPath(category, subCategory, filename string, tags []string) string + + // SearchDocuments searches documents by tags or metadata + SearchDocuments(ctx context.Context, query string, category, subCategory string, options ListOptions) (*ListResult, error) +} + +// BucketManager defines the interface for bucket operations +type BucketManager interface { + // CreateBucket creates a new bucket + CreateBucket(ctx context.Context, bucketName string) error + + // DeleteBucket deletes a bucket + DeleteBucket(ctx context.Context, bucketName string) error + + // BucketExists checks if a bucket exists + BucketExists(ctx context.Context, bucketName string) (bool, error) + + // ListBuckets lists all buckets + ListBuckets(ctx context.Context) ([]string, error) + + // GetBucketInfo gets bucket information + GetBucketInfo(ctx context.Context, bucketName string) (map[string]interface{}, error) +} + +// ConnectionManagerInterface defines the interface for MinIO connection management +type ConnectionManagerInterface interface { + // Connect establishes connection to MinIO + Connect() error + + // Disconnect closes the connection + Disconnect(ctx context.Context) error + + // Ping tests the connection + Ping(ctx context.Context) error + + // IsConnected returns true if connected + IsConnected() bool + + // GetClient returns the underlying MinIO client + GetClient() interface{} + + // CreateBucket creates a new bucket + CreateBucket(ctx context.Context, bucketName string) error + + // BucketExists checks if a bucket exists + BucketExists(ctx context.Context, bucketName string) (bool, error) + + // ListBuckets lists all buckets + ListBuckets(ctx context.Context) ([]string, error) +} diff --git a/pkg/minio/service.go b/pkg/minio/service.go new file mode 100644 index 0000000..7e26406 --- /dev/null +++ b/pkg/minio/service.go @@ -0,0 +1,501 @@ +package minio + +import ( + "context" + "fmt" + "io" + "mime" + "net/url" + "path/filepath" + "strings" + "time" + + "github.com/minio/minio-go/v7" +) + +// Service provides MinIO file storage operations +type Service struct { + connMgr *ConnectionManager + config Config + logger Logger +} + +// NewService creates a new MinIO service +func NewService(connMgr *ConnectionManager, config Config, logger Logger) *Service { + return &Service{ + connMgr: connMgr, + config: config, + logger: logger, + } +} + +// Upload uploads a file to the specified bucket +func (s *Service) Upload(ctx context.Context, bucket, objectName string, reader io.Reader, options UploadOptions) error { + if !s.connMgr.IsConnected() { + return ErrNotConnected + } + + client := s.connMgr.GetClient().(*minio.Client) + + // Set default content type if not provided + contentType := options.ContentType + if contentType == "" { + contentType = mime.TypeByExtension(filepath.Ext(objectName)) + if contentType == "" { + contentType = "application/octet-stream" + } + } + + // Upload the file + _, err := client.PutObject(ctx, bucket, objectName, reader, -1, minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: options.Metadata, + }) + if err != nil { + s.logger.Error("Failed to upload file", map[string]interface{}{ + "bucket": bucket, + "objectName": objectName, + "error": err.Error(), + }) + return NewErrorWithCause(ErrCodeUploadFailed, "failed to upload file", err) + } + + s.logger.Info("File uploaded successfully", map[string]interface{}{ + "bucket": bucket, + "objectName": objectName, + "contentType": contentType, + }) + + return nil +} + +// UploadHierarchical uploads a file to hierarchical storage with classification +func (s *Service) UploadHierarchical(ctx context.Context, file io.Reader, filename string, options HierarchicalUploadOptions) (string, error) { + if !s.connMgr.IsConnected() { + return "", ErrNotConnected + } + + // Generate hierarchical path + objectPath := s.GetDocumentPath(options.Category, options.SubCategory, filename, options.Tags) + + // Ensure bucket exists + err := s.connMgr.CreateBucket(ctx, s.config.HierarchicalBucket) + if err != nil { + return "", fmt.Errorf("failed to create hierarchical bucket: %w", err) + } + + // Set metadata for classification + metadata := options.Metadata + if metadata == nil { + metadata = make(map[string]string) + } + metadata["category"] = options.Category + metadata["subcategory"] = options.SubCategory + if len(options.Tags) > 0 { + metadata["tags"] = strings.Join(options.Tags, ",") + } + metadata["original_filename"] = filename + metadata["uploaded_at"] = fmt.Sprintf("%d", time.Now().Unix()) + + uploadOptions := UploadOptions{ + ContentType: options.ContentType, + Metadata: metadata, + } + + err = s.Upload(ctx, s.config.HierarchicalBucket, objectPath, file, uploadOptions) + if err != nil { + return "", err + } + + s.logger.Info("Hierarchical file uploaded successfully", map[string]interface{}{ + "bucket": s.config.HierarchicalBucket, + "objectPath": objectPath, + "category": options.Category, + "subCategory": options.SubCategory, + }) + + return objectPath, nil +} + +// Download downloads a file from the specified bucket +func (s *Service) Download(ctx context.Context, bucket, objectName string, options DownloadOptions) (io.ReadCloser, error) { + if !s.connMgr.IsConnected() { + return nil, ErrNotConnected + } + + client := s.connMgr.GetClient().(*minio.Client) + + getOptions := minio.GetObjectOptions{} + if options.Range != nil { + getOptions.SetRange(options.Range.Start, options.Range.End) + } + + object, err := client.GetObject(ctx, bucket, objectName, getOptions) + if err != nil { + s.logger.Error("Failed to download file", map[string]interface{}{ + "bucket": bucket, + "objectName": objectName, + "error": err.Error(), + }) + return nil, NewErrorWithCause(ErrCodeDownloadFailed, "failed to download file", err) + } + + s.logger.Debug("File download initiated", map[string]interface{}{ + "bucket": bucket, + "objectName": objectName, + }) + + return object, nil +} + +// Delete deletes a file from the specified bucket +func (s *Service) Delete(ctx context.Context, bucket, objectName string) error { + if !s.connMgr.IsConnected() { + return ErrNotConnected + } + + client := s.connMgr.GetClient().(*minio.Client) + + err := client.RemoveObject(ctx, bucket, objectName, minio.RemoveObjectOptions{}) + if err != nil { + s.logger.Error("Failed to delete file", map[string]interface{}{ + "bucket": bucket, + "objectName": objectName, + "error": err.Error(), + }) + return NewErrorWithCause(ErrCodeDeleteFailed, "failed to delete file", err) + } + + s.logger.Info("File deleted successfully", map[string]interface{}{ + "bucket": bucket, + "objectName": objectName, + }) + + return nil +} + +// GetInfo gets information about a file +func (s *Service) GetInfo(ctx context.Context, bucket, objectName string) (*FileInfo, error) { + if !s.connMgr.IsConnected() { + return nil, ErrNotConnected + } + + client := s.connMgr.GetClient().(*minio.Client) + + stat, err := client.StatObject(ctx, bucket, objectName, minio.StatObjectOptions{}) + if err != nil { + s.logger.Error("Failed to get file info", map[string]interface{}{ + "bucket": bucket, + "objectName": objectName, + "error": err.Error(), + }) + return nil, NewErrorWithCause(ErrCodeObjectNotFound, "failed to get file info", err) + } + + fileInfo := &FileInfo{ + Name: filepath.Base(objectName), + Path: objectName, + Size: stat.Size, + ContentType: stat.ContentType, + ETag: stat.ETag, + LastModified: stat.LastModified.Unix(), + Metadata: stat.UserMetadata, + } + + return fileInfo, nil +} + +// List lists files in a bucket with optional prefix +func (s *Service) List(ctx context.Context, bucket string, options ListOptions) (*ListResult, error) { + if !s.connMgr.IsConnected() { + return nil, ErrNotConnected + } + + client := s.connMgr.GetClient().(*minio.Client) + + listOptions := minio.ListObjectsOptions{ + Prefix: options.Prefix, + Recursive: false, + } + + if options.Delimiter != "" { + listOptions = minio.ListObjectsOptions{ + Prefix: options.Prefix, + Recursive: false, + UseV1: true, + } + } + + if options.MaxKeys > 0 { + listOptions.MaxKeys = int(options.MaxKeys) + } + + objectCh := client.ListObjects(ctx, bucket, listOptions) + + var objects []FileInfo + var commonPrefixes []string + + for object := range objectCh { + if object.Err != nil { + s.logger.Error("Error listing objects", map[string]interface{}{ + "bucket": bucket, + "prefix": options.Prefix, + "error": object.Err.Error(), + }) + return nil, NewErrorWithCause("LIST_FAILED", "failed to list objects", object.Err) + } + + if strings.HasSuffix(object.Key, "/") { + // This is a "directory" (common prefix) + commonPrefixes = append(commonPrefixes, object.Key) + } else { + // This is a file + fileInfo := FileInfo{ + Name: filepath.Base(object.Key), + Path: object.Key, + Size: object.Size, + ContentType: "", // Would need to get from StatObject if needed + ETag: object.ETag, + LastModified: object.LastModified.Unix(), + Metadata: nil, + } + objects = append(objects, fileInfo) + } + } + + result := &ListResult{ + Objects: objects, + CommonPrefixes: commonPrefixes, + IsTruncated: false, // MinIO Go client doesn't provide this info directly + NextMarker: "", + } + + s.logger.Debug("Objects listed successfully", map[string]interface{}{ + "bucket": bucket, + "prefix": options.Prefix, + "objects_count": len(objects), + "prefixes_count": len(commonPrefixes), + }) + + return result, nil +} + +// Copy copies a file within the same bucket or to another bucket +func (s *Service) Copy(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) error { + if !s.connMgr.IsConnected() { + return ErrNotConnected + } + + client := s.connMgr.GetClient().(*minio.Client) + + src := minio.CopySrcOptions{ + Bucket: srcBucket, + Object: srcObject, + } + dst := minio.CopyDestOptions{ + Bucket: destBucket, + Object: destObject, + } + + _, err := client.CopyObject(ctx, dst, src) + if err != nil { + s.logger.Error("Failed to copy file", map[string]interface{}{ + "srcBucket": srcBucket, + "srcObject": srcObject, + "destBucket": destBucket, + "destObject": destObject, + "error": err.Error(), + }) + return NewErrorWithCause(ErrCodeCopyFailed, "failed to copy file", err) + } + + s.logger.Info("File copied successfully", map[string]interface{}{ + "srcBucket": srcBucket, + "srcObject": srcObject, + "destBucket": destBucket, + "destObject": destObject, + }) + + return nil +} + +// Move moves a file (copy + delete) +func (s *Service) Move(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) error { + err := s.Copy(ctx, srcBucket, srcObject, destBucket, destObject) + if err != nil { + return err + } + + return s.Delete(ctx, srcBucket, srcObject) +} + +// Exists checks if a file exists +func (s *Service) Exists(ctx context.Context, bucket, objectName string) (bool, error) { + _, err := s.GetInfo(ctx, bucket, objectName) + if err != nil { + if strings.Contains(err.Error(), "NoSuchKey") || strings.Contains(err.Error(), "not found") { + return false, nil + } + return false, err + } + return true, nil +} + +// GetPresignedURL generates a presigned URL for temporary access +func (s *Service) GetPresignedURL(ctx context.Context, bucket, objectName string, expirySeconds int64) (string, error) { + if !s.connMgr.IsConnected() { + return "", ErrNotConnected + } + + client := s.connMgr.GetClient().(*minio.Client) + + reqParams := make(url.Values) + presignedURL, err := client.PresignedGetObject(ctx, bucket, objectName, time.Duration(expirySeconds)*time.Second, reqParams) + if err != nil { + s.logger.Error("Failed to generate presigned URL", map[string]interface{}{ + "bucket": bucket, + "objectName": objectName, + "expiry": expirySeconds, + "error": err.Error(), + }) + return "", NewErrorWithCause(ErrCodePresignFailed, "failed to generate presigned URL", err) + } + + return presignedURL.String(), nil +} + +// UploadDocument uploads a document with hierarchical classification +func (s *Service) UploadDocument(ctx context.Context, file io.Reader, filename string, options HierarchicalUploadOptions) (string, error) { + return s.UploadHierarchical(ctx, file, filename, options) +} + +// ListDocuments lists documents in a category/subcategory +func (s *Service) ListDocuments(ctx context.Context, category, subCategory string, options ListOptions) (*ListResult, error) { + // Build prefix for hierarchical listing + prefix := s.buildCategoryPrefix(category, subCategory) + if prefix != "" && !strings.HasSuffix(prefix, s.config.PathSeparator) { + prefix += s.config.PathSeparator + } + + listOptions := ListOptions{ + Prefix: prefix, + Delimiter: s.config.PathSeparator, + MaxKeys: options.MaxKeys, + Marker: options.Marker, + } + + return s.List(ctx, s.config.HierarchicalBucket, listOptions) +} + +// GetDocumentPath generates the hierarchical path for a document +func (s *Service) GetDocumentPath(category, subCategory, filename string, tags []string) string { + var pathParts []string + + if category != "" { + pathParts = append(pathParts, category) + } + + if subCategory != "" { + pathParts = append(pathParts, subCategory) + } + + if len(tags) > 0 { + // Create a subdirectory based on tags + tagDir := strings.Join(tags, "_") + if tagDir != "" { + pathParts = append(pathParts, tagDir) + } + } + + // Add timestamp for uniqueness + timestamp := fmt.Sprintf("%d", time.Now().Unix()) + pathParts = append(pathParts, timestamp) + + // Add filename + pathParts = append(pathParts, filename) + + return strings.Join(pathParts, s.config.PathSeparator) +} + +// SearchDocuments searches documents by tags or metadata (basic implementation) +func (s *Service) SearchDocuments(ctx context.Context, query string, category, subCategory string, options ListOptions) (*ListResult, error) { + // For now, do a prefix-based search + // In a real implementation, you might want to use MinIO's search capabilities or index metadata + prefix := s.buildCategoryPrefix(category, subCategory) + + listOptions := ListOptions{ + Prefix: prefix, + Delimiter: s.config.PathSeparator, + MaxKeys: options.MaxKeys, + Marker: options.Marker, + } + + result, err := s.List(ctx, s.config.HierarchicalBucket, listOptions) + if err != nil { + return nil, err + } + + // Basic filtering based on query (could be enhanced) + if query != "" { + var filteredObjects []FileInfo + for _, obj := range result.Objects { + // Check if query matches filename or metadata + if strings.Contains(strings.ToLower(obj.Name), strings.ToLower(query)) { + filteredObjects = append(filteredObjects, obj) + } else { + // Check metadata + for _, value := range obj.Metadata { + if strings.Contains(strings.ToLower(value), strings.ToLower(query)) { + filteredObjects = append(filteredObjects, obj) + break + } + } + } + } + result.Objects = filteredObjects + } + + return result, nil +} + +// buildCategoryPrefix builds the prefix for category-based searches +func (s *Service) buildCategoryPrefix(category, subCategory string) string { + var parts []string + + if category != "" { + parts = append(parts, category) + } + + if subCategory != "" { + parts = append(parts, subCategory) + } + + if len(parts) == 0 { + return "" + } + + return strings.Join(parts, s.config.PathSeparator) +} + +// validateFileType validates if the file type is allowed +func (s *Service) validateFileType(contentType string) error { + if len(s.config.AllowedTypes) == 1 && s.config.AllowedTypes[0] == "*" { + return nil // Allow all types + } + + for _, allowedType := range s.config.AllowedTypes { + if allowedType == contentType || strings.HasPrefix(contentType, allowedType) { + return nil + } + } + + return NewError(ErrCodeInvalidFileType, fmt.Sprintf("file type %s is not allowed", contentType)) +} + +// validateFileSize validates if the file size is within limits +func (s *Service) validateFileSize(size int64) error { + if size > s.config.MaxFileSize { + return NewError(ErrCodeFileTooLarge, fmt.Sprintf("file size %d exceeds maximum allowed size %d", size, s.config.MaxFileSize)) + } + return nil +}