m.nazemi
513 lines
13 KiB
Go
513 lines
13 KiB
Go
package filestore
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"io"
|
|
"time"
|
|
|
|
"tm/pkg/logger"
|
|
"tm/pkg/mongo"
|
|
|
|
"go.mongodb.org/mongo-driver/v2/bson"
|
|
mongopkg "go.mongodb.org/mongo-driver/v2/mongo"
|
|
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
|
)
|
|
|
|
// GridFSService implements FileStoreService using MongoDB GridFS
|
|
type GridFSService struct {
|
|
database mongopkg.Database
|
|
logger logger.Logger
|
|
}
|
|
|
|
// NewGridFSService creates a new GridFS file store service
|
|
func NewGridFSService(mongoManager *mongo.ConnectionManager, logger logger.Logger) (*GridFSService, error) {
|
|
if !mongoManager.IsConnected() {
|
|
return nil, NewError(ErrCodeUploadFailed, "MongoDB connection not available")
|
|
}
|
|
|
|
database := mongoManager.GetDatabase()
|
|
if database == nil {
|
|
return nil, NewError(ErrCodeUploadFailed, "MongoDB database not available")
|
|
}
|
|
|
|
logger.Info("GridFS service initialized successfully", map[string]interface{}{
|
|
"storage_type": "MongoDB GridFS",
|
|
})
|
|
|
|
return &GridFSService{
|
|
database: *database,
|
|
logger: logger,
|
|
}, nil
|
|
}
|
|
|
|
// Upload stores a file in GridFS
|
|
func (s *GridFSService) Upload(fileID, filename string, reader io.Reader, options UploadOptions) (string, error) {
|
|
// Validate input
|
|
if filename == "" {
|
|
return "", ErrInvalidInput
|
|
}
|
|
|
|
if options.ContentType == "" {
|
|
options.ContentType = "application/octet-stream"
|
|
}
|
|
|
|
// Check file size if max size is set
|
|
if options.MaxSize > 0 {
|
|
// Read into buffer to check size
|
|
data, err := io.ReadAll(reader)
|
|
if err != nil {
|
|
s.logger.Error("Failed to read file for size check", map[string]interface{}{
|
|
"filename": filename,
|
|
"error": err.Error(),
|
|
})
|
|
return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to read file", err)
|
|
}
|
|
|
|
if int64(len(data)) > options.MaxSize {
|
|
s.logger.Warn("File exceeds maximum size", map[string]interface{}{
|
|
"filename": filename,
|
|
"size": len(data),
|
|
"max_size": options.MaxSize,
|
|
})
|
|
return "", ErrFileTooLarge
|
|
}
|
|
|
|
reader = bytes.NewReader(data)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
// Create files and chunks collections
|
|
filesCollection := s.database.Collection("fs.files")
|
|
chunksCollection := s.database.Collection("fs.chunks")
|
|
|
|
// Generate file ID if not provided
|
|
var objectID bson.ObjectID
|
|
var err error
|
|
if fileID != "" {
|
|
objectID, err = bson.ObjectIDFromHex(fileID)
|
|
if err != nil {
|
|
s.logger.Error("Invalid file ID format", map[string]interface{}{
|
|
"file_id": fileID,
|
|
"error": err.Error(),
|
|
})
|
|
return "", NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err)
|
|
}
|
|
} else {
|
|
objectID = bson.NewObjectID()
|
|
}
|
|
|
|
// Read file data
|
|
fileData, err := io.ReadAll(reader)
|
|
if err != nil {
|
|
s.logger.Error("Failed to read file data", map[string]interface{}{
|
|
"filename": filename,
|
|
"error": err.Error(),
|
|
})
|
|
return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to read file", err)
|
|
}
|
|
|
|
// Prepare metadata
|
|
metadata := bson.M{
|
|
"uploaded_at": time.Now().Unix(),
|
|
"content_type": options.ContentType,
|
|
}
|
|
|
|
if options.UploadedBy != "" {
|
|
metadata["uploaded_by"] = options.UploadedBy
|
|
}
|
|
if options.Category != "" {
|
|
metadata["category"] = options.Category
|
|
}
|
|
if len(options.Tags) > 0 {
|
|
metadata["tags"] = options.Tags
|
|
}
|
|
if options.Description != "" {
|
|
metadata["description"] = options.Description
|
|
}
|
|
if len(options.Extra) > 0 {
|
|
metadata["extra"] = options.Extra
|
|
}
|
|
|
|
// Split file into chunks (255 KB default chunk size)
|
|
chunkSize := 255 * 1024
|
|
numChunks := (len(fileData) + chunkSize - 1) / chunkSize
|
|
|
|
// Insert chunks
|
|
for i := 0; i < numChunks; i++ {
|
|
start := i * chunkSize
|
|
end := start + chunkSize
|
|
if end > len(fileData) {
|
|
end = len(fileData)
|
|
}
|
|
|
|
chunk := bson.M{
|
|
"files_id": objectID,
|
|
"n": int32(i),
|
|
"data": fileData[start:end],
|
|
}
|
|
|
|
_, err := chunksCollection.InsertOne(ctx, chunk)
|
|
if err != nil {
|
|
s.logger.Error("Failed to insert chunk", map[string]interface{}{
|
|
"filename": filename,
|
|
"chunk": i,
|
|
"error": err.Error(),
|
|
})
|
|
return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to store file chunks", err)
|
|
}
|
|
}
|
|
|
|
// Insert file record
|
|
fileRecord := bson.M{
|
|
"_id": objectID,
|
|
"filename": filename,
|
|
"uploadDate": time.Now(),
|
|
"chunkSize": int32(chunkSize),
|
|
"length": int32(len(fileData)),
|
|
"md5": "", // MongoDB would calculate this, we can leave empty
|
|
"metadata": metadata,
|
|
}
|
|
|
|
_, err = filesCollection.InsertOne(ctx, fileRecord)
|
|
if err != nil {
|
|
s.logger.Error("Failed to insert file record", map[string]interface{}{
|
|
"filename": filename,
|
|
"error": err.Error(),
|
|
})
|
|
return "", NewErrorWithCause(ErrCodeUploadFailed, "failed to upload file", err)
|
|
}
|
|
|
|
uploadedFileID := objectID.Hex()
|
|
s.logger.Info("File uploaded successfully to GridFS", map[string]interface{}{
|
|
"file_id": uploadedFileID,
|
|
"filename": filename,
|
|
"content_type": options.ContentType,
|
|
"uploaded_by": options.UploadedBy,
|
|
"category": options.Category,
|
|
})
|
|
|
|
return uploadedFileID, nil
|
|
}
|
|
|
|
// Download retrieves a file from GridFS
|
|
func (s *GridFSService) Download(fileID string) (io.ReadCloser, *FileInfo, error) {
|
|
if fileID == "" {
|
|
return nil, nil, ErrInvalidInput
|
|
}
|
|
|
|
objectID, err := bson.ObjectIDFromHex(fileID)
|
|
if err != nil {
|
|
s.logger.Error("Invalid file ID format", map[string]interface{}{
|
|
"file_id": fileID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, nil, NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
// Get file info first
|
|
fileInfo, err := s.GetFileInfo(fileID)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
// Get all chunks for this file
|
|
chunksCollection := s.database.Collection("fs.chunks")
|
|
findOpts := options.Find().SetSort(bson.D{{Key: "n", Value: 1}})
|
|
cursor, err := chunksCollection.Find(ctx, bson.M{"files_id": objectID}, findOpts)
|
|
if err != nil {
|
|
s.logger.Error("Failed to find chunks", map[string]interface{}{
|
|
"file_id": fileID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, nil, NewErrorWithCause(ErrCodeDownloadFailed, "failed to download file", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
// Read all chunks and assemble file
|
|
var chunks []bson.M
|
|
if err := cursor.All(ctx, &chunks); err != nil {
|
|
s.logger.Error("Failed to read chunks", map[string]interface{}{
|
|
"file_id": fileID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, nil, NewErrorWithCause(ErrCodeDownloadFailed, "failed to read file data", err)
|
|
}
|
|
|
|
// Assemble file from chunks
|
|
var fileData bytes.Buffer
|
|
for _, chunk := range chunks {
|
|
data, ok := chunk["data"].(bson.Binary)
|
|
if !ok {
|
|
return nil, nil, NewError(ErrCodeDownloadFailed, "invalid chunk data format")
|
|
}
|
|
fileData.Write(data.Data)
|
|
}
|
|
|
|
s.logger.Info("File downloaded successfully from GridFS", map[string]interface{}{
|
|
"file_id": fileID,
|
|
"filename": fileInfo.Filename,
|
|
})
|
|
|
|
return io.NopCloser(&fileData), fileInfo, nil
|
|
}
|
|
|
|
// Delete removes a file from GridFS
|
|
func (s *GridFSService) Delete(fileID string) error {
|
|
if fileID == "" {
|
|
return ErrInvalidInput
|
|
}
|
|
|
|
objectID, err := bson.ObjectIDFromHex(fileID)
|
|
if err != nil {
|
|
s.logger.Error("Invalid file ID format", map[string]interface{}{
|
|
"file_id": fileID,
|
|
"error": err.Error(),
|
|
})
|
|
return NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
// Delete file record
|
|
filesCollection := s.database.Collection("fs.files")
|
|
_, err = filesCollection.DeleteOne(ctx, bson.M{"_id": objectID})
|
|
if err != nil {
|
|
s.logger.Error("Failed to delete file record", map[string]interface{}{
|
|
"file_id": fileID,
|
|
"error": err.Error(),
|
|
})
|
|
return NewErrorWithCause(ErrCodeDeleteFailed, "failed to delete file", err)
|
|
}
|
|
|
|
// Delete associated chunks
|
|
chunksCollection := s.database.Collection("fs.chunks")
|
|
_, err = chunksCollection.DeleteMany(ctx, bson.M{"files_id": objectID})
|
|
if err != nil {
|
|
s.logger.Error("Failed to delete file chunks", map[string]interface{}{
|
|
"file_id": fileID,
|
|
"error": err.Error(),
|
|
})
|
|
return NewErrorWithCause(ErrCodeDeleteFailed, "failed to delete file chunks", err)
|
|
}
|
|
|
|
s.logger.Info("File deleted successfully from GridFS", map[string]interface{}{
|
|
"file_id": fileID,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetFileInfo retrieves metadata for a file
|
|
func (s *GridFSService) GetFileInfo(fileID string) (*FileInfo, error) {
|
|
if fileID == "" {
|
|
return nil, ErrInvalidInput
|
|
}
|
|
|
|
objectID, err := bson.ObjectIDFromHex(fileID)
|
|
if err != nil {
|
|
s.logger.Error("Invalid file ID format", map[string]interface{}{
|
|
"file_id": fileID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, NewErrorWithCause(ErrCodeInvalidInput, "invalid file ID format", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
// Get file record from fs.files collection
|
|
filesCollection := s.database.Collection("fs.files")
|
|
var fileRecord bson.M
|
|
err = filesCollection.FindOne(ctx, bson.M{"_id": objectID}).Decode(&fileRecord)
|
|
if err != nil {
|
|
s.logger.Error("Failed to find file", map[string]interface{}{
|
|
"file_id": fileID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, NewErrorWithCause(ErrCodeNotFound, "file not found", err)
|
|
}
|
|
|
|
// Extract upload date
|
|
var uploadDate time.Time
|
|
if ud, ok := fileRecord["uploadDate"].(time.Time); ok {
|
|
uploadDate = ud
|
|
}
|
|
|
|
// Extract metadata
|
|
var metadata map[string]interface{}
|
|
if md, ok := fileRecord["metadata"].(bson.M); ok {
|
|
metadata = md
|
|
}
|
|
|
|
fileInfo := &FileInfo{
|
|
FileID: fileID,
|
|
Filename: fileRecord["filename"].(string),
|
|
ContentType: "application/octet-stream",
|
|
Size: int64(fileRecord["length"].(int32)),
|
|
UploadedAt: uploadDate,
|
|
Metadata: metadata,
|
|
}
|
|
|
|
// Extract metadata fields if they exist
|
|
if metadata != nil {
|
|
if ct, ok := metadata["content_type"].(string); ok {
|
|
fileInfo.ContentType = ct
|
|
}
|
|
if ub, ok := metadata["uploaded_by"].(string); ok {
|
|
fileInfo.UploadedBy = ub
|
|
}
|
|
if cat, ok := metadata["category"].(string); ok {
|
|
fileInfo.Category = cat
|
|
}
|
|
if tags, ok := metadata["tags"].([]interface{}); ok {
|
|
fileInfo.Tags = make([]string, len(tags))
|
|
for i, tag := range tags {
|
|
if tagStr, ok := tag.(string); ok {
|
|
fileInfo.Tags[i] = tagStr
|
|
}
|
|
}
|
|
}
|
|
if desc, ok := metadata["description"].(string); ok {
|
|
fileInfo.Description = desc
|
|
}
|
|
if extra, ok := metadata["extra"].(bson.M); ok {
|
|
fileInfo.Extra = make(map[string]string)
|
|
for k, v := range extra {
|
|
if vs, ok := v.(string); ok {
|
|
fileInfo.Extra[k] = vs
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return fileInfo, nil
|
|
}
|
|
|
|
// ListFiles lists files by category with optional filtering
|
|
func (s *GridFSService) ListFiles(category string, tags []string, limit int64, skip int64) ([]*FileInfo, int64, error) {
|
|
if limit <= 0 {
|
|
limit = 50
|
|
}
|
|
if limit > 1000 {
|
|
limit = 1000
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
// Build filter
|
|
filter := bson.M{}
|
|
if category != "" {
|
|
filter["metadata.category"] = category
|
|
}
|
|
if len(tags) > 0 {
|
|
filter["metadata.tags"] = bson.M{"$in": tags}
|
|
}
|
|
|
|
// Find files
|
|
filesCollection := s.database.Collection("fs.files")
|
|
|
|
// Count total
|
|
total, err := filesCollection.CountDocuments(ctx, filter)
|
|
if err != nil {
|
|
s.logger.Error("Failed to count files", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, 0, NewErrorWithCause(ErrCodeDownloadFailed, "failed to count files", err)
|
|
}
|
|
|
|
// Find and list
|
|
opts := options.Find().SetSkip(skip).SetLimit(limit)
|
|
|
|
cursor, err := filesCollection.Find(ctx, filter, opts)
|
|
if err != nil {
|
|
s.logger.Error("Failed to list files", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, 0, NewErrorWithCause(ErrCodeDownloadFailed, "failed to list files", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
var fileRecords []bson.M
|
|
if err := cursor.All(ctx, &fileRecords); err != nil {
|
|
s.logger.Error("Failed to decode files", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, 0, NewErrorWithCause(ErrCodeDownloadFailed, "failed to decode files", err)
|
|
}
|
|
|
|
// Convert to FileInfo
|
|
var files []*FileInfo
|
|
for _, record := range fileRecords {
|
|
fileID := record["_id"].(bson.ObjectID).Hex()
|
|
|
|
var uploadDate time.Time
|
|
if ud, ok := record["uploadDate"].(time.Time); ok {
|
|
uploadDate = ud
|
|
}
|
|
|
|
var metadata map[string]interface{}
|
|
if md, ok := record["metadata"].(bson.M); ok {
|
|
metadata = md
|
|
}
|
|
|
|
fileInfo := &FileInfo{
|
|
FileID: fileID,
|
|
Filename: record["filename"].(string),
|
|
ContentType: "application/octet-stream",
|
|
Size: int64(record["length"].(int32)),
|
|
UploadedAt: uploadDate,
|
|
Metadata: metadata,
|
|
}
|
|
|
|
// Extract metadata fields
|
|
if metadata != nil {
|
|
if ct, ok := metadata["content_type"].(string); ok {
|
|
fileInfo.ContentType = ct
|
|
}
|
|
if ub, ok := metadata["uploaded_by"].(string); ok {
|
|
fileInfo.UploadedBy = ub
|
|
}
|
|
if cat, ok := metadata["category"].(string); ok {
|
|
fileInfo.Category = cat
|
|
}
|
|
if tags, ok := metadata["tags"].([]interface{}); ok {
|
|
fileInfo.Tags = make([]string, len(tags))
|
|
for i, tag := range tags {
|
|
if tagStr, ok := tag.(string); ok {
|
|
fileInfo.Tags[i] = tagStr
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
files = append(files, fileInfo)
|
|
}
|
|
|
|
return files, total, nil
|
|
}
|
|
|
|
// Exists checks if a file exists
|
|
func (s *GridFSService) Exists(fileID string) (bool, error) {
|
|
if fileID == "" {
|
|
return false, ErrInvalidInput
|
|
}
|
|
|
|
_, err := s.GetFileInfo(fileID)
|
|
if err != nil {
|
|
if IsNotFound(err) {
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
|
|
return true, nil
|
|
}
|