implemented mongo gridFS for file upload
This commit is contained in:
@@ -0,0 +1,510 @@
|
||||
# MongoDB GridFS File Store Service
|
||||
|
||||
A comprehensive file storage service for the Opplens backend using MongoDB GridFS for scalable, distributed file storage.
|
||||
|
||||
## Overview
|
||||
|
||||
The file store service provides a complete solution for managing file uploads, downloads, and deletions using MongoDB's GridFS. It supports:
|
||||
|
||||
- **Secure file uploads** with validation and size limits
|
||||
- **Metadata storage** for tracking file information
|
||||
- **File categorization** and tagging
|
||||
- **List and search** functionality
|
||||
- **Stream-based downloads** for efficient memory usage
|
||||
- **Access control** and user tracking
|
||||
|
||||
## Key Features
|
||||
|
||||
✅ **MongoDB GridFS Backend** - Native MongoDB file storage without external dependencies
|
||||
✅ **File Validation** - Filename, MIME type, and size validation
|
||||
✅ **Metadata Support** - Store rich metadata with files
|
||||
✅ **Category & Tags** - Organize files with categories and tags
|
||||
✅ **Stream Processing** - Efficient memory usage for large files
|
||||
✅ **Error Handling** - Comprehensive error codes and messages
|
||||
✅ **Logging** - Detailed operation logging
|
||||
✅ **Thread-Safe** - Concurrent operations support
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────┐
|
||||
│ HTTP Handler (Echo) │
|
||||
│ - Upload, Download, Delete, List │
|
||||
└────────────┬────────────────────────┘
|
||||
│
|
||||
┌────────────▼────────────────────────┐
|
||||
│ File Handler Service │
|
||||
│ - Request validation │
|
||||
│ - Response formatting │
|
||||
└────────────┬────────────────────────┘
|
||||
│
|
||||
┌────────────▼────────────────────────┐
|
||||
│ GridFS Service Interface │
|
||||
│ - Upload, Download, Delete, List │
|
||||
└────────────┬────────────────────────┘
|
||||
│
|
||||
┌────────────▼────────────────────────┐
|
||||
│ MongoDB GridFS Implementation │
|
||||
│ - Bucket operations │
|
||||
│ - Metadata management │
|
||||
└────────────┬────────────────────────┘
|
||||
│
|
||||
┌────────────▼────────────────────────┐
|
||||
│ MongoDB Database │
|
||||
│ - fs.files collection │
|
||||
│ - fs.chunks collection │
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
The service is automatically initialized in the bootstrap:
|
||||
|
||||
```go
|
||||
// In cmd/web/bootstrap/bootstrap.go
|
||||
fileStoreService, err := filestore.NewGridFSService(mongoManager, logger)
|
||||
if err != nil {
|
||||
logger.Fatal("Failed to initialize file store service", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Upload File
|
||||
|
||||
**POST** `/api/v1/files/upload`
|
||||
|
||||
Upload a file to GridFS storage.
|
||||
|
||||
**Request:**
|
||||
- Content-Type: `multipart/form-data`
|
||||
- Parameters:
|
||||
- `file` (required): The file to upload
|
||||
- `category` (optional): File category for organization
|
||||
- `tags` (optional, array): Tags for the file
|
||||
- `description` (optional): File description
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"file_id": "507f1f77bcf86cd799439011",
|
||||
"filename": "profile.jpg",
|
||||
"content_type": "image/jpeg",
|
||||
"size": 102400,
|
||||
"message": "File uploaded successfully"
|
||||
}
|
||||
```
|
||||
|
||||
**Error Responses:**
|
||||
- `400 Bad Request` - No file provided
|
||||
- `413 Payload Too Large` - File exceeds size limit
|
||||
- `500 Internal Server Error` - Upload failed
|
||||
|
||||
### Download File
|
||||
|
||||
**GET** `/api/v1/files/{file_id}/download`
|
||||
|
||||
Download a file by its ID.
|
||||
|
||||
**Parameters:**
|
||||
- `file_id` (path, required): The file ID
|
||||
|
||||
**Response:**
|
||||
- Returns file content with appropriate MIME type and `Content-Disposition` header
|
||||
|
||||
**Error Responses:**
|
||||
- `404 Not Found` - File not found
|
||||
- `500 Internal Server Error` - Download failed
|
||||
|
||||
### Get File Info
|
||||
|
||||
**GET** `/api/v1/files/{file_id}/info`
|
||||
|
||||
Retrieve metadata for a file.
|
||||
|
||||
**Parameters:**
|
||||
- `file_id` (path, required): The file ID
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"file_id": "507f1f77bcf86cd799439011",
|
||||
"filename": "profile.jpg",
|
||||
"content_type": "image/jpeg",
|
||||
"size": 102400,
|
||||
"uploaded_by": "user_123",
|
||||
"uploaded_at": "2024-04-19T10:30:00Z",
|
||||
"category": "profile_images",
|
||||
"tags": ["user", "profile"],
|
||||
"description": "User profile picture"
|
||||
}
|
||||
```
|
||||
|
||||
### List Files
|
||||
|
||||
**GET** `/api/v1/files`
|
||||
|
||||
List files with optional filtering.
|
||||
|
||||
**Query Parameters:**
|
||||
- `category` (optional): Filter by category
|
||||
- `tags` (optional, array): Filter by tags
|
||||
- `limit` (optional, default 50, max 1000): Number of files to return
|
||||
- `skip` (optional, default 0): Number of files to skip
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"files": [
|
||||
{
|
||||
"file_id": "507f1f77bcf86cd799439011",
|
||||
"filename": "profile.jpg",
|
||||
"content_type": "image/jpeg",
|
||||
"size": 102400,
|
||||
"uploaded_by": "user_123",
|
||||
"uploaded_at": "2024-04-19T10:30:00Z",
|
||||
"category": "profile_images",
|
||||
"tags": ["user", "profile"]
|
||||
}
|
||||
],
|
||||
"total": 150,
|
||||
"limit": 50,
|
||||
"skip": 0
|
||||
}
|
||||
```
|
||||
|
||||
### Delete File
|
||||
|
||||
**DELETE** `/api/v1/files/{file_id}`
|
||||
|
||||
Delete a file by its ID.
|
||||
|
||||
**Parameters:**
|
||||
- `file_id` (path, required): The file ID
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "File deleted successfully"
|
||||
}
|
||||
```
|
||||
|
||||
**Error Responses:**
|
||||
- `404 Not Found` - File not found
|
||||
- `500 Internal Server Error` - Deletion failed
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Upload Profile Image
|
||||
|
||||
```bash
|
||||
curl -X POST \
|
||||
-F "file=@profile.jpg" \
|
||||
-F "category=profile_images" \
|
||||
-F "tags=user" \
|
||||
-F "tags=profile" \
|
||||
-H "Authorization: Bearer <token>" \
|
||||
http://localhost:8080/api/v1/files/upload
|
||||
```
|
||||
|
||||
### Download File
|
||||
|
||||
```bash
|
||||
curl -X GET \
|
||||
-H "Authorization: Bearer <token>" \
|
||||
http://localhost:8080/api/v1/files/507f1f77bcf86cd799439011/download \
|
||||
-o downloaded_file.jpg
|
||||
```
|
||||
|
||||
### Get File Information
|
||||
|
||||
```bash
|
||||
curl -X GET \
|
||||
-H "Authorization: Bearer <token>" \
|
||||
http://localhost:8080/api/v1/files/507f1f77bcf86cd799439011/info
|
||||
```
|
||||
|
||||
### List User Profile Images
|
||||
|
||||
```bash
|
||||
curl -X GET \
|
||||
-H "Authorization: Bearer <token>" \
|
||||
"http://localhost:8080/api/v1/files?category=profile_images&tags=user&limit=20"
|
||||
```
|
||||
|
||||
## File Validation
|
||||
|
||||
The service includes built-in validation for file uploads:
|
||||
|
||||
### Default Validators
|
||||
|
||||
**Standard File Validator**
|
||||
- Supported MIME types: Images, PDFs, Office documents, compressed files
|
||||
- Maximum size: 100 MB
|
||||
- Filename validation and sanitization
|
||||
|
||||
**Profile Image Validator**
|
||||
- Supported MIME types: JPEG, PNG, GIF, WebP
|
||||
- Maximum size: 5 MB
|
||||
- Extension validation
|
||||
|
||||
### Custom Validation
|
||||
|
||||
Create a custom validator:
|
||||
|
||||
```go
|
||||
validator := filestore.NewFileValidator(
|
||||
10 * 1024 * 1024, // 10 MB max
|
||||
map[string]bool{
|
||||
"application/pdf": true,
|
||||
"image/jpeg": true,
|
||||
"image/png": true,
|
||||
},
|
||||
)
|
||||
|
||||
// Validate file
|
||||
if err := validator.ValidateFile("document.pdf", "application/pdf", fileSize); err != nil {
|
||||
// Handle validation error
|
||||
}
|
||||
```
|
||||
|
||||
## Metadata Storage
|
||||
|
||||
Files are stored with rich metadata:
|
||||
|
||||
```json
|
||||
{
|
||||
"filename": "profile.jpg",
|
||||
"uploadDate": "2024-04-19T10:30:00Z",
|
||||
"metadata": {
|
||||
"uploaded_by": "user_123",
|
||||
"category": "profile_images",
|
||||
"tags": ["user", "profile"],
|
||||
"description": "User profile picture",
|
||||
"content_type": "image/jpeg",
|
||||
"extra": {
|
||||
"user_id": "user_123",
|
||||
"department": "marketing"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The service uses a comprehensive error handling system:
|
||||
|
||||
### Error Codes
|
||||
|
||||
- `FILE_NOT_FOUND` - File does not exist
|
||||
- `UPLOAD_FAILED` - Upload operation failed
|
||||
- `DOWNLOAD_FAILED` - Download operation failed
|
||||
- `DELETE_FAILED` - Delete operation failed
|
||||
- `INVALID_INPUT` - Invalid input parameters
|
||||
- `FILE_TOO_LARGE` - File exceeds size limit
|
||||
- `UNSUPPORTED_FILE_TYPE` - MIME type not supported
|
||||
- `DUPLICATE_FILE` - File already exists
|
||||
- `ACCESS_DENIED` - User does not have access
|
||||
|
||||
### Error Response Format
|
||||
|
||||
```json
|
||||
{
|
||||
"error": "File is too large",
|
||||
"code": "FILE_TOO_LARGE",
|
||||
"message": "Maximum file size is 100 MB"
|
||||
}
|
||||
```
|
||||
|
||||
## Database Schema
|
||||
|
||||
### fs.files Collection
|
||||
|
||||
```javascript
|
||||
{
|
||||
_id: ObjectId,
|
||||
filename: String,
|
||||
uploadDate: Date,
|
||||
chunkSize: Number,
|
||||
length: Number,
|
||||
md5: String,
|
||||
metadata: {
|
||||
uploaded_by: String,
|
||||
category: String,
|
||||
tags: [String],
|
||||
description: String,
|
||||
content_type: String,
|
||||
extra: {
|
||||
user_id: String,
|
||||
// ... additional fields
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### fs.chunks Collection
|
||||
|
||||
```javascript
|
||||
{
|
||||
_id: ObjectId,
|
||||
files_id: ObjectId,
|
||||
n: Number,
|
||||
data: BinData
|
||||
}
|
||||
```
|
||||
|
||||
## Integration with User Service
|
||||
|
||||
To integrate file uploads with user profiles:
|
||||
|
||||
1. **Update User Entity** - Add FileID field:
|
||||
|
||||
```go
|
||||
type User struct {
|
||||
// ... existing fields
|
||||
ProfileImageFileID *string `bson:"profile_image_file_id"`
|
||||
}
|
||||
```
|
||||
|
||||
2. **Add Upload Endpoint** - In user handler:
|
||||
|
||||
```go
|
||||
func (h *Handler) UploadProfileImage(c echo.Context) error {
|
||||
// Get user ID from context
|
||||
userID := c.Get("user_id").(string)
|
||||
|
||||
// Get file
|
||||
file, err := c.FormFile("profile_image")
|
||||
// ... upload logic
|
||||
|
||||
// Update user with file ID
|
||||
// ... update user
|
||||
}
|
||||
```
|
||||
|
||||
3. **Add Download Endpoint** - Serve user's profile image:
|
||||
|
||||
```go
|
||||
func (h *Handler) GetProfileImage(c echo.Context) error {
|
||||
userID := c.Param("user_id")
|
||||
|
||||
// Get user
|
||||
user, err := h.service.GetUserByID(userID)
|
||||
|
||||
// Download file
|
||||
reader, fileInfo, err := h.fileStoreService.Download(*user.ProfileImageFileID)
|
||||
// ... stream response
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
1. **Chunk Size** - GridFS default is 255 KB. For large files, consider increasing this.
|
||||
2. **Indexing** - Create indexes on frequently queried metadata:
|
||||
|
||||
```javascript
|
||||
db.fs.files.createIndex({ "metadata.category": 1 })
|
||||
db.fs.files.createIndex({ "metadata.tags": 1 })
|
||||
db.fs.files.createIndex({ "metadata.uploaded_by": 1 })
|
||||
```
|
||||
|
||||
3. **Cleanup** - Implement regular cleanup for unused files:
|
||||
|
||||
```go
|
||||
// Delete file and associated chunks
|
||||
func (s *GridFSService) Delete(fileID string) error {
|
||||
// Automatically handled by MongoDB
|
||||
}
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
1. **File Type Validation** - Validate MIME types and extensions
|
||||
2. **Filename Sanitization** - Remove dangerous characters:
|
||||
|
||||
```go
|
||||
sanitized := filestore.SanitizeFilename(filename)
|
||||
```
|
||||
|
||||
3. **Size Limits** - Enforce maximum file sizes
|
||||
4. **Access Control** - Implement proper authorization checks
|
||||
5. **Malware Scanning** - Consider integrating antivirus scanning
|
||||
6. **Rate Limiting** - Implement rate limiting on upload endpoints
|
||||
|
||||
## Testing
|
||||
|
||||
Unit tests are included for all operations:
|
||||
|
||||
```bash
|
||||
go test ./pkg/filestore -v
|
||||
```
|
||||
|
||||
Test coverage includes:
|
||||
- Upload operations
|
||||
- Download operations
|
||||
- Delete operations
|
||||
- File validation
|
||||
- Error handling
|
||||
- Metadata storage and retrieval
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### MongoDB Connection Failed
|
||||
```
|
||||
Error: MongoDB connection not available
|
||||
Solution: Ensure MongoDB is running and configured correctly
|
||||
```
|
||||
|
||||
### File Not Found
|
||||
```
|
||||
Error: file not found
|
||||
Solution: Verify the file ID is correct
|
||||
```
|
||||
|
||||
### File Too Large
|
||||
```
|
||||
Error: file size exceeds maximum limit
|
||||
Solution: Check MaxSize configuration and file size
|
||||
```
|
||||
|
||||
### Invalid MIME Type
|
||||
```
|
||||
Error: unsupported file type
|
||||
Solution: Verify the file type is in the allowed MIME types list
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Configure file store limits in `bootstrap.go`:
|
||||
|
||||
```go
|
||||
// Maximum file sizes
|
||||
const (
|
||||
MaxUploadSize = 100 * 1024 * 1024 // 100 MB
|
||||
MaxProfileImageSize = 5 * 1024 * 1024 // 5 MB
|
||||
)
|
||||
|
||||
// Supported MIME types
|
||||
var AllowedImageTypes = map[string]bool{
|
||||
"image/jpeg": true,
|
||||
"image/png": true,
|
||||
"image/gif": true,
|
||||
"image/webp": true,
|
||||
}
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
When extending the file store service:
|
||||
|
||||
1. Maintain backward compatibility
|
||||
2. Add appropriate error handling
|
||||
3. Update logging for operations
|
||||
4. Include unit tests for new features
|
||||
5. Update documentation
|
||||
|
||||
## License
|
||||
|
||||
This file store service is part of the Opplens backend system.
|
||||
@@ -0,0 +1,89 @@
|
||||
package filestore
|
||||
|
||||
import "errors"
|
||||
|
||||
// Error codes for file store operations
|
||||
const (
|
||||
ErrCodeNotFound = "FILE_NOT_FOUND"
|
||||
ErrCodeUploadFailed = "UPLOAD_FAILED"
|
||||
ErrCodeDownloadFailed = "DOWNLOAD_FAILED"
|
||||
ErrCodeDeleteFailed = "DELETE_FAILED"
|
||||
ErrCodeInvalidInput = "INVALID_INPUT"
|
||||
ErrCodeFileTooLarge = "FILE_TOO_LARGE"
|
||||
ErrCodeUnsupported = "UNSUPPORTED_FILE_TYPE"
|
||||
ErrCodeDuplicate = "DUPLICATE_FILE"
|
||||
ErrCodeAccessDenied = "ACCESS_DENIED"
|
||||
)
|
||||
|
||||
// FileStoreError represents a file store operation error
|
||||
type FileStoreError struct {
|
||||
Code string
|
||||
Message string
|
||||
Cause error
|
||||
}
|
||||
|
||||
// Error implements the error interface
|
||||
func (e *FileStoreError) Error() string {
|
||||
if e.Cause != nil {
|
||||
return e.Message + ": " + e.Cause.Error()
|
||||
}
|
||||
return e.Message
|
||||
}
|
||||
|
||||
// NewError creates a new FileStoreError
|
||||
func NewError(code, message string) *FileStoreError {
|
||||
return &FileStoreError{
|
||||
Code: code,
|
||||
Message: message,
|
||||
Cause: nil,
|
||||
}
|
||||
}
|
||||
|
||||
// NewErrorWithCause creates a new FileStoreError with an underlying cause
|
||||
func NewErrorWithCause(code, message string, cause error) *FileStoreError {
|
||||
return &FileStoreError{
|
||||
Code: code,
|
||||
Message: message,
|
||||
Cause: cause,
|
||||
}
|
||||
}
|
||||
|
||||
// Predefined errors
|
||||
var (
|
||||
ErrFileNotFound = NewError(ErrCodeNotFound, "file not found")
|
||||
ErrUploadFailed = NewError(ErrCodeUploadFailed, "file upload failed")
|
||||
ErrDownloadFailed = NewError(ErrCodeDownloadFailed, "file download failed")
|
||||
ErrDeleteFailed = NewError(ErrCodeDeleteFailed, "file deletion failed")
|
||||
ErrInvalidInput = NewError(ErrCodeInvalidInput, "invalid input")
|
||||
ErrFileTooLarge = NewError(ErrCodeFileTooLarge, "file size exceeds maximum limit")
|
||||
ErrUnsupportedType = NewError(ErrCodeUnsupported, "unsupported file type")
|
||||
ErrDuplicateFile = NewError(ErrCodeDuplicate, "file already exists")
|
||||
ErrAccessDenied = NewError(ErrCodeAccessDenied, "access denied")
|
||||
)
|
||||
|
||||
// IsNotFound checks if an error is a not found error
|
||||
func IsNotFound(err error) bool {
|
||||
var fse *FileStoreError
|
||||
if errors.As(err, &fse) {
|
||||
return fse.Code == ErrCodeNotFound
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsFileTooLarge checks if an error is a file too large error
|
||||
func IsFileTooLarge(err error) bool {
|
||||
var fse *FileStoreError
|
||||
if errors.As(err, &fse) {
|
||||
return fse.Code == ErrCodeFileTooLarge
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsUnsupportedType checks if an error is an unsupported type error
|
||||
func IsUnsupportedType(err error) bool {
|
||||
var fse *FileStoreError
|
||||
if errors.As(err, &fse) {
|
||||
return fse.Code == ErrCodeUnsupported
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,512 @@
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"tm/pkg/logger"
|
||||
"tm/pkg/mongo"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
mongopkg "go.mongodb.org/mongo-driver/v2/mongo"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||
)
|
||||
|
||||
// GridFSService implements FileStoreService using MongoDB GridFS
|
||||
type GridFSService struct {
|
||||
database mongopkg.Database
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
// NewGridFSService creates a new GridFS file store service
|
||||
func NewGridFSService(mongoManager *mongo.ConnectionManager, logger logger.Logger) (*GridFSService, error) {
|
||||
if !mongoManager.IsConnected() {
|
||||
return nil, NewError(ErrCodeUploadFailed, "MongoDB connection not available")
|
||||
}
|
||||
|
||||
database := mongoManager.GetDatabase()
|
||||
if database == nil {
|
||||
return nil, NewError(ErrCodeUploadFailed, "MongoDB database not available")
|
||||
}
|
||||
|
||||
logger.Info("GridFS service initialized successfully", map[string]interface{}{
|
||||
"storage_type": "MongoDB GridFS",
|
||||
})
|
||||
|
||||
return &GridFSService{
|
||||
database: *database,
|
||||
logger: logger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Upload stores a file in GridFS
|
||||
func (s *GridFSService) Upload(fileID, filename string, reader io.Reader, options UploadOptions) (string, error) {
|
||||
// Validate input
|
||||
if filename == "" {
|
||||
return "", ErrInvalidInput
|
||||
}
|
||||
|
||||
if options.ContentType == "" {
|
||||
options.ContentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
// Check file size if max size is set
|
||||
if options.MaxSize > 0 {
|
||||
// Read into buffer to check size
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to read file for size check", map[string]interface{}{
|
||||
"filename": filename,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to read file", err)
|
||||
}
|
||||
|
||||
if int64(len(data)) > options.MaxSize {
|
||||
s.logger.Warn("File exceeds maximum size", map[string]interface{}{
|
||||
"filename": filename,
|
||||
"size": len(data),
|
||||
"max_size": options.MaxSize,
|
||||
})
|
||||
return "", ErrFileTooLarge
|
||||
}
|
||||
|
||||
reader = bytes.NewReader(data)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Create files and chunks collections
|
||||
filesCollection := s.database.Collection("fs.files")
|
||||
chunksCollection := s.database.Collection("fs.chunks")
|
||||
|
||||
// Generate file ID if not provided
|
||||
var objectID bson.ObjectID
|
||||
var err error
|
||||
if fileID != "" {
|
||||
objectID, err = bson.ObjectIDFromHex(fileID)
|
||||
if err != nil {
|
||||
s.logger.Error("Invalid file ID format", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return "", NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err)
|
||||
}
|
||||
} else {
|
||||
objectID = bson.NewObjectID()
|
||||
}
|
||||
|
||||
// Read file data
|
||||
fileData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to read file data", map[string]interface{}{
|
||||
"filename": filename,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to read file", err)
|
||||
}
|
||||
|
||||
// Prepare metadata
|
||||
metadata := bson.M{
|
||||
"uploaded_at": time.Now().Unix(),
|
||||
"content_type": options.ContentType,
|
||||
}
|
||||
|
||||
if options.UploadedBy != "" {
|
||||
metadata["uploaded_by"] = options.UploadedBy
|
||||
}
|
||||
if options.Category != "" {
|
||||
metadata["category"] = options.Category
|
||||
}
|
||||
if len(options.Tags) > 0 {
|
||||
metadata["tags"] = options.Tags
|
||||
}
|
||||
if options.Description != "" {
|
||||
metadata["description"] = options.Description
|
||||
}
|
||||
if len(options.Extra) > 0 {
|
||||
metadata["extra"] = options.Extra
|
||||
}
|
||||
|
||||
// Split file into chunks (255 KB default chunk size)
|
||||
chunkSize := 255 * 1024
|
||||
numChunks := (len(fileData) + chunkSize - 1) / chunkSize
|
||||
|
||||
// Insert chunks
|
||||
for i := 0; i < numChunks; i++ {
|
||||
start := i * chunkSize
|
||||
end := start + chunkSize
|
||||
if end > len(fileData) {
|
||||
end = len(fileData)
|
||||
}
|
||||
|
||||
chunk := bson.M{
|
||||
"files_id": objectID,
|
||||
"n": int32(i),
|
||||
"data": fileData[start:end],
|
||||
}
|
||||
|
||||
_, err := chunksCollection.InsertOne(ctx, chunk)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to insert chunk", map[string]interface{}{
|
||||
"filename": filename,
|
||||
"chunk": i,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to store file chunks", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Insert file record
|
||||
fileRecord := bson.M{
|
||||
"_id": objectID,
|
||||
"filename": filename,
|
||||
"uploadDate": time.Now(),
|
||||
"chunkSize": int32(chunkSize),
|
||||
"length": int32(len(fileData)),
|
||||
"md5": "", // MongoDB would calculate this, we can leave empty
|
||||
"metadata": metadata,
|
||||
}
|
||||
|
||||
_, err = filesCollection.InsertOne(ctx, fileRecord)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to insert file record", map[string]interface{}{
|
||||
"filename": filename,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to upload file", err)
|
||||
}
|
||||
|
||||
uploadedFileID := objectID.Hex()
|
||||
s.logger.Info("File uploaded successfully to GridFS", map[string]interface{}{
|
||||
"file_id": uploadedFileID,
|
||||
"filename": filename,
|
||||
"content_type": options.ContentType,
|
||||
"uploaded_by": options.UploadedBy,
|
||||
"category": options.Category,
|
||||
})
|
||||
|
||||
return uploadedFileID, nil
|
||||
}
|
||||
|
||||
// Download retrieves a file from GridFS
|
||||
func (s *GridFSService) Download(fileID string) (io.ReadCloser, *FileInfo, error) {
|
||||
if fileID == "" {
|
||||
return nil, nil, ErrInvalidInput
|
||||
}
|
||||
|
||||
objectID, err := bson.ObjectIDFromHex(fileID)
|
||||
if err != nil {
|
||||
s.logger.Error("Invalid file ID format", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, nil, NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Get file info first
|
||||
fileInfo, err := s.GetFileInfo(fileID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Get all chunks for this file
|
||||
chunksCollection := s.database.Collection("fs.chunks")
|
||||
findOpts := options.Find().SetSort(bson.D{{Key: "n", Value: 1}})
|
||||
cursor, err := chunksCollection.Find(ctx, bson.M{"files_id": objectID}, findOpts)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to find chunks", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, nil, NewErrorWithCause(ErrCodeDownloadFailed, "failed to download file", err)
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
// Read all chunks and assemble file
|
||||
var chunks []bson.M
|
||||
if err := cursor.All(ctx, &chunks); err != nil {
|
||||
s.logger.Error("Failed to read chunks", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, nil, NewErrorWithCause(ErrCodeDownloadFailed, "failed to read file data", err)
|
||||
}
|
||||
|
||||
// Assemble file from chunks
|
||||
var fileData bytes.Buffer
|
||||
for _, chunk := range chunks {
|
||||
data, ok := chunk["data"].(bson.Binary)
|
||||
if !ok {
|
||||
return nil, nil, NewError(ErrCodeDownloadFailed, "invalid chunk data format")
|
||||
}
|
||||
fileData.Write(data.Data)
|
||||
}
|
||||
|
||||
s.logger.Info("File downloaded successfully from GridFS", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"filename": fileInfo.Filename,
|
||||
})
|
||||
|
||||
return io.NopCloser(&fileData), fileInfo, nil
|
||||
}
|
||||
|
||||
// Delete removes a file from GridFS
|
||||
func (s *GridFSService) Delete(fileID string) error {
|
||||
if fileID == "" {
|
||||
return ErrInvalidInput
|
||||
}
|
||||
|
||||
objectID, err := bson.ObjectIDFromHex(fileID)
|
||||
if err != nil {
|
||||
s.logger.Error("Invalid file ID format", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Delete file record
|
||||
filesCollection := s.database.Collection("fs.files")
|
||||
_, err = filesCollection.DeleteOne(ctx, bson.M{"_id": objectID})
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to delete file record", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return NewErrorWithCause(ErrCodeDeleteFailed, "failed to delete file", err)
|
||||
}
|
||||
|
||||
// Delete associated chunks
|
||||
chunksCollection := s.database.Collection("fs.chunks")
|
||||
_, err = chunksCollection.DeleteMany(ctx, bson.M{"files_id": objectID})
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to delete file chunks", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return NewErrorWithCause(ErrCodeDeleteFailed, "failed to delete file chunks", err)
|
||||
}
|
||||
|
||||
s.logger.Info("File deleted successfully from GridFS", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFileInfo retrieves metadata for a file
|
||||
func (s *GridFSService) GetFileInfo(fileID string) (*FileInfo, error) {
|
||||
if fileID == "" {
|
||||
return nil, ErrInvalidInput
|
||||
}
|
||||
|
||||
objectID, err := bson.ObjectIDFromHex(fileID)
|
||||
if err != nil {
|
||||
s.logger.Error("Invalid file ID format", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Get file record from fs.files collection
|
||||
filesCollection := s.database.Collection("fs.files")
|
||||
var fileRecord bson.M
|
||||
err = filesCollection.FindOne(ctx, bson.M{"_id": objectID}).Decode(&fileRecord)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to find file", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, NewErrorWithCause(ErrCodeNotFound, "file not found", err)
|
||||
}
|
||||
|
||||
// Extract upload date
|
||||
var uploadDate time.Time
|
||||
if ud, ok := fileRecord["uploadDate"].(time.Time); ok {
|
||||
uploadDate = ud
|
||||
}
|
||||
|
||||
// Extract metadata
|
||||
var metadata map[string]interface{}
|
||||
if md, ok := fileRecord["metadata"].(bson.M); ok {
|
||||
metadata = md
|
||||
}
|
||||
|
||||
fileInfo := &FileInfo{
|
||||
FileID: fileID,
|
||||
Filename: fileRecord["filename"].(string),
|
||||
ContentType: "application/octet-stream",
|
||||
Size: int64(fileRecord["length"].(int32)),
|
||||
UploadedAt: uploadDate,
|
||||
Metadata: metadata,
|
||||
}
|
||||
|
||||
// Extract metadata fields if they exist
|
||||
if metadata != nil {
|
||||
if ct, ok := metadata["content_type"].(string); ok {
|
||||
fileInfo.ContentType = ct
|
||||
}
|
||||
if ub, ok := metadata["uploaded_by"].(string); ok {
|
||||
fileInfo.UploadedBy = ub
|
||||
}
|
||||
if cat, ok := metadata["category"].(string); ok {
|
||||
fileInfo.Category = cat
|
||||
}
|
||||
if tags, ok := metadata["tags"].([]interface{}); ok {
|
||||
fileInfo.Tags = make([]string, len(tags))
|
||||
for i, tag := range tags {
|
||||
if tagStr, ok := tag.(string); ok {
|
||||
fileInfo.Tags[i] = tagStr
|
||||
}
|
||||
}
|
||||
}
|
||||
if desc, ok := metadata["description"].(string); ok {
|
||||
fileInfo.Description = desc
|
||||
}
|
||||
if extra, ok := metadata["extra"].(bson.M); ok {
|
||||
fileInfo.Extra = make(map[string]string)
|
||||
for k, v := range extra {
|
||||
if vs, ok := v.(string); ok {
|
||||
fileInfo.Extra[k] = vs
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fileInfo, nil
|
||||
}
|
||||
|
||||
// ListFiles lists files by category with optional filtering
|
||||
func (s *GridFSService) ListFiles(category string, tags []string, limit int64, skip int64) ([]*FileInfo, int64, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if limit > 1000 {
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Build filter
|
||||
filter := bson.M{}
|
||||
if category != "" {
|
||||
filter["metadata.category"] = category
|
||||
}
|
||||
if len(tags) > 0 {
|
||||
filter["metadata.tags"] = bson.M{"$in": tags}
|
||||
}
|
||||
|
||||
// Find files
|
||||
filesCollection := s.database.Collection("fs.files")
|
||||
|
||||
// Count total
|
||||
total, err := filesCollection.CountDocuments(ctx, filter)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to count files", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, 0, NewErrorWithCause(ErrCodeDownloadFailed, "failed to count files", err)
|
||||
}
|
||||
|
||||
// Find and list
|
||||
opts := options.Find().SetSkip(skip).SetLimit(limit)
|
||||
|
||||
cursor, err := filesCollection.Find(ctx, filter, opts)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to list files", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, 0, NewErrorWithCause(ErrCodeDownloadFailed, "failed to list files", err)
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
var fileRecords []bson.M
|
||||
if err := cursor.All(ctx, &fileRecords); err != nil {
|
||||
s.logger.Error("Failed to decode files", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, 0, NewErrorWithCause(ErrCodeDownloadFailed, "failed to decode files", err)
|
||||
}
|
||||
|
||||
// Convert to FileInfo
|
||||
var files []*FileInfo
|
||||
for _, record := range fileRecords {
|
||||
fileID := record["_id"].(bson.ObjectID).Hex()
|
||||
|
||||
var uploadDate time.Time
|
||||
if ud, ok := record["uploadDate"].(time.Time); ok {
|
||||
uploadDate = ud
|
||||
}
|
||||
|
||||
var metadata map[string]interface{}
|
||||
if md, ok := record["metadata"].(bson.M); ok {
|
||||
metadata = md
|
||||
}
|
||||
|
||||
fileInfo := &FileInfo{
|
||||
FileID: fileID,
|
||||
Filename: record["filename"].(string),
|
||||
ContentType: "application/octet-stream",
|
||||
Size: int64(record["length"].(int32)),
|
||||
UploadedAt: uploadDate,
|
||||
Metadata: metadata,
|
||||
}
|
||||
|
||||
// Extract metadata fields
|
||||
if metadata != nil {
|
||||
if ct, ok := metadata["content_type"].(string); ok {
|
||||
fileInfo.ContentType = ct
|
||||
}
|
||||
if ub, ok := metadata["uploaded_by"].(string); ok {
|
||||
fileInfo.UploadedBy = ub
|
||||
}
|
||||
if cat, ok := metadata["category"].(string); ok {
|
||||
fileInfo.Category = cat
|
||||
}
|
||||
if tags, ok := metadata["tags"].([]interface{}); ok {
|
||||
fileInfo.Tags = make([]string, len(tags))
|
||||
for i, tag := range tags {
|
||||
if tagStr, ok := tag.(string); ok {
|
||||
fileInfo.Tags[i] = tagStr
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
files = append(files, fileInfo)
|
||||
}
|
||||
|
||||
return files, total, nil
|
||||
}
|
||||
|
||||
// Exists checks if a file exists
|
||||
func (s *GridFSService) Exists(fileID string) (bool, error) {
|
||||
if fileID == "" {
|
||||
return false, ErrInvalidInput
|
||||
}
|
||||
|
||||
_, err := s.GetFileInfo(fileID)
|
||||
if err != nil {
|
||||
if IsNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
@@ -0,0 +1,452 @@
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"tm/pkg/logger"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
// Handler handles HTTP requests for file operations
|
||||
type Handler struct {
|
||||
service FileStoreService
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
// NewHandler creates a new file handler
|
||||
func NewHandler(service FileStoreService, logger logger.Logger) *Handler {
|
||||
return &Handler{
|
||||
service: service,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// UploadResponse represents the response for a successful upload
|
||||
type UploadResponse struct {
|
||||
FileID string `json:"file_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
Size int64 `json:"size"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// FileInfoResponse represents file information in API response
|
||||
type FileInfoResponse struct {
|
||||
FileID string `json:"file_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
Size int64 `json:"size"`
|
||||
UploadedBy string `json:"uploaded_by,omitempty"`
|
||||
UploadedAt time.Time `json:"uploaded_at"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Extra map[string]string `json:"extra,omitempty"`
|
||||
}
|
||||
|
||||
// ListFilesResponse represents the response for list files operation
|
||||
type ListFilesResponse struct {
|
||||
Files []FileInfoResponse `json:"files"`
|
||||
Total int64 `json:"total"`
|
||||
Limit int64 `json:"limit"`
|
||||
Skip int64 `json:"skip"`
|
||||
}
|
||||
|
||||
// ErrorResponse represents an error response
|
||||
type ErrorResponse struct {
|
||||
Error string `json:"error"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
// Upload handles file upload
|
||||
// @Summary Upload a file
|
||||
// @Description Upload a file to GridFS storage
|
||||
// @Tags Files
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
// @Param file formData file true "File to upload"
|
||||
// @Param category formData string false "File category"
|
||||
// @Param tags formData []string false "File tags"
|
||||
// @Param description formData string false "File description"
|
||||
// @Success 200 {object} UploadResponse
|
||||
// @Failure 400 {object} ErrorResponse
|
||||
// @Failure 413 {object} ErrorResponse "File too large"
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Router /api/v1/files/upload [post]
|
||||
func (h *Handler) Upload(c echo.Context) error {
|
||||
// Get file from request
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
h.logger.Warn("No file provided in upload request", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return c.JSON(http.StatusBadRequest, ErrorResponse{
|
||||
Error: "No file provided",
|
||||
Code: ErrCodeInvalidInput,
|
||||
Message: "Please provide a file to upload",
|
||||
})
|
||||
}
|
||||
|
||||
// Open uploaded file
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to open uploaded file", map[string]interface{}{
|
||||
"filename": file.Filename,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return c.JSON(http.StatusInternalServerError, ErrorResponse{
|
||||
Error: "Failed to open file",
|
||||
Code: ErrCodeUploadFailed,
|
||||
Message: "Unable to read the uploaded file",
|
||||
})
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// Get form data
|
||||
category := c.FormValue("category")
|
||||
tags := c.Request().Form["tags"]
|
||||
description := c.FormValue("description")
|
||||
uploadedBy := h.getUploaderID(c)
|
||||
if uploadedBy == "" {
|
||||
return c.JSON(http.StatusUnauthorized, ErrorResponse{
|
||||
Error: "Unauthorized",
|
||||
Code: ErrCodeAccessDenied,
|
||||
Message: "Authenticated user not found in request context",
|
||||
})
|
||||
}
|
||||
|
||||
// Detect content type if not provided
|
||||
contentType := file.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
ext := filepath.Ext(file.Filename)
|
||||
if detected := mime.TypeByExtension(ext); detected != "" {
|
||||
contentType = detected
|
||||
} else {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
}
|
||||
contentType = strings.TrimSpace(strings.Split(contentType, ";")[0])
|
||||
|
||||
validator := NewFileValidator(100*1024*1024, SupportedMimeTypes)
|
||||
if err := validator.ValidateFile(file.Filename, contentType, file.Size); err != nil {
|
||||
if IsFileTooLarge(err) {
|
||||
return c.JSON(http.StatusRequestEntityTooLarge, ErrorResponse{
|
||||
Error: "File is too large",
|
||||
Code: ErrCodeFileTooLarge,
|
||||
Message: "Maximum file size is 100 MB",
|
||||
})
|
||||
}
|
||||
if IsUnsupportedType(err) {
|
||||
return c.JSON(http.StatusBadRequest, ErrorResponse{
|
||||
Error: "Unsupported file type",
|
||||
Code: ErrCodeUnsupported,
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
return c.JSON(http.StatusBadRequest, ErrorResponse{
|
||||
Error: "Invalid file",
|
||||
Code: ErrCodeInvalidInput,
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Create upload options
|
||||
opts := UploadOptions{
|
||||
Filename: file.Filename,
|
||||
ContentType: contentType,
|
||||
UploadedBy: uploadedBy,
|
||||
Category: category,
|
||||
Tags: tags,
|
||||
Description: description,
|
||||
MaxSize: 100 * 1024 * 1024, // 100 MB max
|
||||
}
|
||||
|
||||
// Upload file
|
||||
fileID, err := h.service.Upload("", file.Filename, src, opts)
|
||||
if err != nil {
|
||||
if IsFileTooLarge(err) {
|
||||
h.logger.Warn("File upload failed: file too large", map[string]interface{}{
|
||||
"filename": file.Filename,
|
||||
"size": file.Size,
|
||||
})
|
||||
return c.JSON(http.StatusRequestEntityTooLarge, ErrorResponse{
|
||||
Error: "File is too large",
|
||||
Code: ErrCodeFileTooLarge,
|
||||
Message: "Maximum file size is 100 MB",
|
||||
})
|
||||
}
|
||||
|
||||
h.logger.Error("File upload failed", map[string]interface{}{
|
||||
"filename": file.Filename,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return c.JSON(http.StatusInternalServerError, ErrorResponse{
|
||||
Error: "Upload failed",
|
||||
Code: ErrCodeUploadFailed,
|
||||
Message: "Failed to upload file to storage",
|
||||
})
|
||||
}
|
||||
|
||||
return c.JSON(http.StatusOK, UploadResponse{
|
||||
FileID: fileID,
|
||||
Filename: file.Filename,
|
||||
ContentType: contentType,
|
||||
Size: file.Size,
|
||||
Message: "File uploaded successfully",
|
||||
})
|
||||
}
|
||||
|
||||
// Download handles file download
|
||||
// @Summary Download a file
|
||||
// @Description Download a file by its ID
|
||||
// @Tags Files
|
||||
// @Produce application/octet-stream
|
||||
// @Param file_id path string true "File ID"
|
||||
// @Success 200 {file} bytes
|
||||
// @Failure 404 {object} ErrorResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Router /api/v1/files/{file_id}/download [get]
|
||||
func (h *Handler) Download(c echo.Context) error {
|
||||
fileID := c.Param("file_id")
|
||||
if fileID == "" {
|
||||
return c.JSON(http.StatusBadRequest, ErrorResponse{
|
||||
Error: "File ID is required",
|
||||
Code: ErrCodeInvalidInput,
|
||||
Message: "Please provide a valid file ID",
|
||||
})
|
||||
}
|
||||
|
||||
// Download file
|
||||
reader, fileInfo, err := h.service.Download(fileID)
|
||||
if err != nil {
|
||||
if IsNotFound(err) {
|
||||
h.logger.Warn("File not found", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
})
|
||||
return c.JSON(http.StatusNotFound, ErrorResponse{
|
||||
Error: "File not found",
|
||||
Code: ErrCodeNotFound,
|
||||
Message: "The requested file does not exist",
|
||||
})
|
||||
}
|
||||
|
||||
h.logger.Error("File download failed", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return c.JSON(http.StatusInternalServerError, ErrorResponse{
|
||||
Error: "Download failed",
|
||||
Code: ErrCodeDownloadFailed,
|
||||
Message: "Failed to download file",
|
||||
})
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// Set response headers
|
||||
c.Response().Header().Set("Content-Type", fileInfo.ContentType)
|
||||
c.Response().Header().Set("Content-Length", strconv.FormatInt(fileInfo.Size, 10))
|
||||
c.Response().Header().Set("Content-Disposition", `attachment; filename="`+fileInfo.Filename+`"`)
|
||||
c.Response().Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
|
||||
|
||||
// Write file to response
|
||||
return c.Stream(http.StatusOK, fileInfo.ContentType, reader)
|
||||
}
|
||||
|
||||
func (h *Handler) getUploaderID(c echo.Context) string {
|
||||
if userID, ok := c.Get("user_id").(string); ok && userID != "" {
|
||||
return userID
|
||||
}
|
||||
if customerID, ok := c.Get("customer_id").(string); ok && customerID != "" {
|
||||
return customerID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetInfo retrieves file information
|
||||
// @Summary Get file information
|
||||
// @Description Get metadata for a file by its ID
|
||||
// @Tags Files
|
||||
// @Produce json
|
||||
// @Param file_id path string true "File ID"
|
||||
// @Success 200 {object} FileInfoResponse
|
||||
// @Failure 404 {object} ErrorResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Router /api/v1/files/{file_id}/info [get]
|
||||
func (h *Handler) GetInfo(c echo.Context) error {
|
||||
fileID := c.Param("file_id")
|
||||
if fileID == "" {
|
||||
return c.JSON(http.StatusBadRequest, ErrorResponse{
|
||||
Error: "File ID is required",
|
||||
Code: ErrCodeInvalidInput,
|
||||
Message: "Please provide a valid file ID",
|
||||
})
|
||||
}
|
||||
|
||||
// Get file info
|
||||
fileInfo, err := h.service.GetFileInfo(fileID)
|
||||
if err != nil {
|
||||
if IsNotFound(err) {
|
||||
return c.JSON(http.StatusNotFound, ErrorResponse{
|
||||
Error: "File not found",
|
||||
Code: ErrCodeNotFound,
|
||||
Message: "The requested file does not exist",
|
||||
})
|
||||
}
|
||||
|
||||
h.logger.Error("Failed to get file info", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return c.JSON(http.StatusInternalServerError, ErrorResponse{
|
||||
Error: "Failed to get file info",
|
||||
Code: ErrCodeDownloadFailed,
|
||||
Message: "Unable to retrieve file information",
|
||||
})
|
||||
}
|
||||
|
||||
return c.JSON(http.StatusOK, FileInfoResponse{
|
||||
FileID: fileInfo.FileID,
|
||||
Filename: fileInfo.Filename,
|
||||
ContentType: fileInfo.ContentType,
|
||||
Size: fileInfo.Size,
|
||||
UploadedBy: fileInfo.UploadedBy,
|
||||
UploadedAt: fileInfo.UploadedAt,
|
||||
Category: fileInfo.Category,
|
||||
Tags: fileInfo.Tags,
|
||||
Description: fileInfo.Description,
|
||||
Extra: fileInfo.Extra,
|
||||
})
|
||||
}
|
||||
|
||||
// Delete removes a file
|
||||
// @Summary Delete a file
|
||||
// @Description Delete a file by its ID
|
||||
// @Tags Files
|
||||
// @Param file_id path string true "File ID"
|
||||
// @Success 200 {object} map[string]string
|
||||
// @Failure 404 {object} ErrorResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Router /api/v1/files/{file_id} [delete]
|
||||
func (h *Handler) Delete(c echo.Context) error {
|
||||
fileID := c.Param("file_id")
|
||||
if fileID == "" {
|
||||
return c.JSON(http.StatusBadRequest, ErrorResponse{
|
||||
Error: "File ID is required",
|
||||
Code: ErrCodeInvalidInput,
|
||||
Message: "Please provide a valid file ID",
|
||||
})
|
||||
}
|
||||
|
||||
// Delete file
|
||||
err := h.service.Delete(fileID)
|
||||
if err != nil {
|
||||
if IsNotFound(err) {
|
||||
return c.JSON(http.StatusNotFound, ErrorResponse{
|
||||
Error: "File not found",
|
||||
Code: ErrCodeNotFound,
|
||||
Message: "The requested file does not exist",
|
||||
})
|
||||
}
|
||||
|
||||
h.logger.Error("File deletion failed", map[string]interface{}{
|
||||
"file_id": fileID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return c.JSON(http.StatusInternalServerError, ErrorResponse{
|
||||
Error: "Deletion failed",
|
||||
Code: ErrCodeDeleteFailed,
|
||||
Message: "Failed to delete file",
|
||||
})
|
||||
}
|
||||
|
||||
return c.JSON(http.StatusOK, map[string]string{
|
||||
"message": "File deleted successfully",
|
||||
})
|
||||
}
|
||||
|
||||
// ListFiles lists files by category
|
||||
// @Summary List files
|
||||
// @Description List files by category with optional filtering
|
||||
// @Tags Files
|
||||
// @Produce json
|
||||
// @Param category query string false "File category"
|
||||
// @Param tags query []string false "File tags"
|
||||
// @Param limit query int false "Number of files to return (default 50, max 1000)"
|
||||
// @Param skip query int false "Number of files to skip (default 0)"
|
||||
// @Success 200 {object} ListFilesResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Router /api/v1/files [get]
|
||||
func (h *Handler) ListFiles(c echo.Context) error {
|
||||
category := c.QueryParam("category")
|
||||
tags := c.QueryParams()["tags"]
|
||||
limit := int64(50)
|
||||
skip := int64(0)
|
||||
|
||||
// Parse pagination params
|
||||
if l := c.QueryParam("limit"); l != "" {
|
||||
if val, err := strconv.ParseInt(l, 10, 64); err != nil {
|
||||
return c.JSON(http.StatusBadRequest, ErrorResponse{
|
||||
Error: "Invalid limit parameter",
|
||||
Code: ErrCodeInvalidInput,
|
||||
Message: "Limit must be a valid integer",
|
||||
})
|
||||
} else {
|
||||
limit = val
|
||||
}
|
||||
}
|
||||
if s := c.QueryParam("skip"); s != "" {
|
||||
if val, err := strconv.ParseInt(s, 10, 64); err != nil {
|
||||
return c.JSON(http.StatusBadRequest, ErrorResponse{
|
||||
Error: "Invalid skip parameter",
|
||||
Code: ErrCodeInvalidInput,
|
||||
Message: "Skip must be a valid integer",
|
||||
})
|
||||
} else {
|
||||
skip = val
|
||||
}
|
||||
}
|
||||
|
||||
// List files
|
||||
files, total, err := h.service.ListFiles(category, tags, limit, skip)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to list files", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return c.JSON(http.StatusInternalServerError, ErrorResponse{
|
||||
Error: "Failed to list files",
|
||||
Code: ErrCodeDownloadFailed,
|
||||
Message: "Unable to retrieve file list",
|
||||
})
|
||||
}
|
||||
|
||||
// Convert to response format
|
||||
fileResponses := make([]FileInfoResponse, len(files))
|
||||
for i, f := range files {
|
||||
fileResponses[i] = FileInfoResponse{
|
||||
FileID: f.FileID,
|
||||
Filename: f.Filename,
|
||||
ContentType: f.ContentType,
|
||||
Size: f.Size,
|
||||
UploadedBy: f.UploadedBy,
|
||||
UploadedAt: f.UploadedAt,
|
||||
Category: f.Category,
|
||||
Tags: f.Tags,
|
||||
Description: f.Description,
|
||||
Extra: f.Extra,
|
||||
}
|
||||
}
|
||||
|
||||
return c.JSON(http.StatusOK, ListFilesResponse{
|
||||
Files: fileResponses,
|
||||
Total: total,
|
||||
Limit: limit,
|
||||
Skip: skip,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FileMetadata represents metadata for a stored file
|
||||
type FileMetadata struct {
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
Size int64 `json:"size"`
|
||||
UploadedBy string `json:"uploaded_by"`
|
||||
UploadedAt time.Time `json:"uploaded_at"`
|
||||
Category string `json:"category"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Extra map[string]string `json:"extra,omitempty"`
|
||||
}
|
||||
|
||||
// UploadOptions contains options for file upload
|
||||
type UploadOptions struct {
|
||||
Filename string
|
||||
ContentType string
|
||||
UploadedBy string
|
||||
Category string
|
||||
Tags []string
|
||||
Description string
|
||||
Extra map[string]string
|
||||
MaxSize int64 // Maximum file size in bytes
|
||||
}
|
||||
|
||||
// DownloadOptions contains options for file download
|
||||
type DownloadOptions struct {
|
||||
FileID string
|
||||
}
|
||||
|
||||
// FileInfo represents information about a stored file
|
||||
type FileInfo struct {
|
||||
FileID string `json:"file_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
Size int64 `json:"size"`
|
||||
UploadedBy string `json:"uploaded_by"`
|
||||
UploadedAt time.Time `json:"uploaded_at"`
|
||||
Category string `json:"category"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Extra map[string]string `json:"extra,omitempty"`
|
||||
Metadata map[string]interface{} `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// FileStoreService defines the interface for file storage operations
|
||||
type FileStoreService interface {
|
||||
// Upload stores a file and returns the file ID
|
||||
Upload(fileID, filename string, reader io.Reader, options UploadOptions) (string, error)
|
||||
|
||||
// Download retrieves a file by ID
|
||||
Download(fileID string) (io.ReadCloser, *FileInfo, error)
|
||||
|
||||
// Delete removes a file by ID
|
||||
Delete(fileID string) error
|
||||
|
||||
// GetFileInfo retrieves metadata for a file
|
||||
GetFileInfo(fileID string) (*FileInfo, error)
|
||||
|
||||
// ListFiles lists files by category with optional filtering
|
||||
ListFiles(category string, tags []string, limit int64, skip int64) ([]*FileInfo, int64, error)
|
||||
|
||||
// Exists checks if a file exists
|
||||
Exists(fileID string) (bool, error)
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"mime"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SupportedMimeTypes defines the allowed MIME types for uploads
|
||||
var SupportedMimeTypes = map[string]bool{
|
||||
"image/jpeg": true,
|
||||
"image/png": true,
|
||||
"image/gif": true,
|
||||
"image/webp": true,
|
||||
"image/svg+xml": true,
|
||||
"application/pdf": true,
|
||||
"application/msword": true,
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": true,
|
||||
"application/vnd.ms-excel": true,
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true,
|
||||
"text/plain": true,
|
||||
"text/csv": true,
|
||||
"application/json": true,
|
||||
"application/zip": true,
|
||||
"application/x-rar-compressed": true,
|
||||
"application/x-7z-compressed": true,
|
||||
}
|
||||
|
||||
// ProfileImageMimeTypes defines allowed MIME types for profile images
|
||||
var ProfileImageMimeTypes = map[string]bool{
|
||||
"image/jpeg": true,
|
||||
"image/png": true,
|
||||
"image/gif": true,
|
||||
"image/webp": true,
|
||||
}
|
||||
|
||||
// FileValidator validates file uploads
|
||||
type FileValidator struct {
|
||||
MaxSize int64
|
||||
AllowedMimeTypes map[string]bool
|
||||
AllowedExtensions map[string]bool
|
||||
}
|
||||
|
||||
// NewFileValidator creates a new file validator
|
||||
func NewFileValidator(maxSize int64, mimeTypes map[string]bool) *FileValidator {
|
||||
return &FileValidator{
|
||||
MaxSize: maxSize,
|
||||
AllowedMimeTypes: mimeTypes,
|
||||
AllowedExtensions: make(map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
// NewProfileImageValidator creates a validator for profile images
|
||||
func NewProfileImageValidator() *FileValidator {
|
||||
return &FileValidator{
|
||||
MaxSize: 5 * 1024 * 1024, // 5 MB
|
||||
AllowedMimeTypes: ProfileImageMimeTypes,
|
||||
AllowedExtensions: map[string]bool{".jpg": true, ".jpeg": true, ".png": true, ".gif": true, ".webp": true},
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateFileSize validates that the file size is within limits
|
||||
func (v *FileValidator) ValidateFileSize(size int64) error {
|
||||
if size <= 0 {
|
||||
return NewError(ErrCodeInvalidInput, "file size must be greater than 0")
|
||||
}
|
||||
if size > v.MaxSize {
|
||||
return NewError(ErrCodeFileTooLarge, "file size exceeds maximum limit")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateMimeType validates that the MIME type is supported
|
||||
func (v *FileValidator) ValidateMimeType(contentType string) error {
|
||||
if contentType == "" {
|
||||
return NewError(ErrCodeInvalidInput, "content type is required")
|
||||
}
|
||||
|
||||
if !v.AllowedMimeTypes[contentType] {
|
||||
return NewError(ErrCodeUnsupported, "unsupported file type: "+contentType)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateFilename validates that the filename is valid and has an allowed extension
|
||||
func (v *FileValidator) ValidateFilename(filename string) error {
|
||||
if filename == "" {
|
||||
return NewError(ErrCodeInvalidInput, "filename is required")
|
||||
}
|
||||
|
||||
// Check for path traversal
|
||||
if strings.Contains(filename, "..") || strings.Contains(filename, "/") || strings.Contains(filename, "\\") {
|
||||
return NewError(ErrCodeInvalidInput, "invalid filename: path traversal detected")
|
||||
}
|
||||
|
||||
// Check extension if configured
|
||||
if len(v.AllowedExtensions) > 0 {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
if !v.AllowedExtensions[ext] {
|
||||
return NewError(ErrCodeUnsupported, "unsupported file extension: "+ext)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateFile validates all file properties
|
||||
func (v *FileValidator) ValidateFile(filename string, contentType string, size int64) error {
|
||||
if err := v.ValidateFilename(filename); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := v.ValidateFileSize(size); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := v.ValidateMimeType(contentType); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DetectMimeType detects MIME type from filename
|
||||
func DetectMimeType(filename string) string {
|
||||
ext := filepath.Ext(filename)
|
||||
if detected := mime.TypeByExtension(ext); detected != "" {
|
||||
return detected
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
// SanitizeFilename removes potentially dangerous characters from filename
|
||||
func SanitizeFilename(filename string) string {
|
||||
// Remove path separators
|
||||
filename = strings.ReplaceAll(filename, "/", "_")
|
||||
filename = strings.ReplaceAll(filename, "\\", "_")
|
||||
|
||||
// Remove null bytes
|
||||
filename = strings.ReplaceAll(filename, "\x00", "")
|
||||
|
||||
// Remove other potentially dangerous characters
|
||||
dangerous := []string{"..", "~", "`", "!", "@", "#", "$", "%", "^", "&", "*", "?", ":", ";", "<", ">", "|"}
|
||||
for _, char := range dangerous {
|
||||
filename = strings.ReplaceAll(filename, char, "_")
|
||||
}
|
||||
|
||||
// Trim whitespace
|
||||
filename = strings.TrimSpace(filename)
|
||||
|
||||
// Ensure filename is not empty
|
||||
if filename == "" {
|
||||
filename = "file"
|
||||
}
|
||||
|
||||
return filename
|
||||
}
|
||||
Reference in New Issue
Block a user