bagas/filekeeper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add recursive deletion for S3 objects to enable folder deletion

Browse Source
This commit is contained in:
Bagas Aulia Rezki
2024-10-28 14:31:57 +07:00
parent 25d62bfd14
commit 7b55f19dbb
22 changed files with 309 additions and 437 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/app.go
View File

@ -14,13 +14,12 @@ type App struct {
http.Server
Database types.Database
Cache types.CachingServer
Service types.Services
Storage types.Storage
Logger *logger.AggregatedLogger
Mail *email.SmtpServer
}
func NewClientServer(addr string, handler http.Handler, logger logger.AggregatedLogger, database types.Database, cache types.CachingServer, storage types.Storage, service types.Services, mail email.SmtpServer) App {
func NewClientServer(addr string, handler http.Handler, logger logger.AggregatedLogger, database types.Database, cache types.CachingServer, storage types.Storage, mail email.SmtpServer) App {
return App{
Server: http.Server{
Addr: addr,
@ -30,7 +29,6 @@ func NewClientServer(addr string, handler http.Handler, logger logger.Aggregated
Logger: &logger,
Database: database,
Cache: cache,
Service: service,
Mail: &mail,
}
}

246
cache/cache.go vendored
View File

@ -2,8 +2,13 @@ package cache
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/types"
"github.com/fossyy/filekeeper/types/models"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"time"
)
@ -13,6 +18,13 @@ type RedisServer struct {
database types.Database
}
const (
UserCacheKey = "UserCache:%s"
UserFilesCacheKey = "UserFilesCache:%s"
FileCacheKey = "FileCache:%s"
FileChunkCacheKey = "FileChunkCache:%s"
)
func NewRedisServer(host, port, password string, db types.Database) types.CachingServer {
client := redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("%s:%s", host, port),
@ -66,3 +78,237 @@ func (r *RedisServer) GetKeys(ctx context.Context, pattern string) ([]string, er
}
return keys, nil
}
func (r *RedisServer) GetUser(ctx context.Context, email string) (*models.User, error) {
cacheKey := fmt.Sprintf(UserCacheKey, email)
userJSON, err := r.GetCache(ctx, cacheKey)
if err != nil {
if errors.Is(err, redis.Nil) {
userData, err := app.Server.Database.GetUser(email)
if err != nil {
return nil, err
}
user := &models.User{
UserID: userData.UserID,
Username: userData.Username,
Email: userData.Email,
Password: userData.Password,
Totp: userData.Totp,
}
newUserJSON, _ := json.Marshal(user)
err = r.SetCache(ctx, cacheKey, newUserJSON, time.Hour*12)
if err != nil {
return nil, err
}
return user, nil
}
return nil, err
}
var user models.User
err = json.Unmarshal([]byte(userJSON), &user)
if err != nil {
return nil, err
}
return &user, nil
}
func (r *RedisServer) RemoveUserCache(ctx context.Context, email string) error {
cacheKey := fmt.Sprintf(UserCacheKey, email)
return r.DeleteCache(ctx, cacheKey)
}
func (r *RedisServer) GetFile(ctx context.Context, id string) (*models.File, error) {
cacheKey := fmt.Sprintf(FileCacheKey, id)
fileJSON, err := r.GetCache(ctx, cacheKey)
if err != nil {
if errors.Is(err, redis.Nil) {
fileData, err := app.Server.Database.GetFile(id)
if err != nil {
return nil, err
}
newFileJSON, _ := json.Marshal(fileData)
err = r.SetCache(ctx, cacheKey, newFileJSON, time.Hour*24)
if err != nil {
return nil, err
}
return fileData, nil
}
return nil, err
}
var file models.File
err = json.Unmarshal([]byte(fileJSON), &file)
if err != nil {
return nil, err
}
return &file, nil
}
func (r *RedisServer) RemoveFileCache(ctx context.Context, id string) error {
cacheKey := fmt.Sprintf(FileCacheKey, id)
return r.DeleteCache(ctx, cacheKey)
}
func (r *RedisServer) GetUserFiles(ctx context.Context, ownerID uuid.UUID) ([]*models.File, error) {
cacheKey := fmt.Sprintf(UserFilesCacheKey, ownerID.String())
filesJSON, err := r.GetCache(ctx, cacheKey)
if err != nil {
if errors.Is(err, redis.Nil) {
files, err := app.Server.Database.GetFiles(ownerID.String(), "", types.All)
if err != nil {
return nil, err
}
filesJSON, err := json.Marshal(files)
if err != nil {
return nil, err
}
err = r.SetCache(ctx, cacheKey, filesJSON, time.Hour*6)
if err != nil {
return nil, err
}
return files, nil
}
return nil, err
}
var files []*models.File
err = json.Unmarshal([]byte(filesJSON), &files)
if err != nil {
return nil, err
}
return files, nil
}
func (r *RedisServer) RemoveUserFilesCache(ctx context.Context, ownerID uuid.UUID) error {
cacheKey := fmt.Sprintf(UserFilesCacheKey, ownerID.String())
return r.DeleteCache(ctx, cacheKey)
}
func (r *RedisServer) CalculateUserStorageUsage(ctx context.Context, ownerID string) (uint64, error) {
files, err := app.Server.Database.GetFiles(ownerID, "", types.All)
if err != nil {
return 0, err
}
var total uint64
for _, file := range files {
total += file.Size
}
return total, nil
}
func (r *RedisServer) GetFileChunks(ctx context.Context, fileID, ownerID uuid.UUID, totalChunk uint64) (*types.FileState, error) {
cacheKey := fmt.Sprintf(FileChunkCacheKey, fileID.String())
fileJSON, err := r.GetCache(ctx, cacheKey)
if err != nil {
if errors.Is(err, redis.Nil) {
prefix := fmt.Sprintf("%s/%s/chunk_", ownerID.String(), fileID.String())
existingChunks, err := app.Server.Storage.ListObjects(ctx, prefix)
if err != nil {
return nil, err
}
fileState := types.FileState{
Done: len(existingChunks) == int(totalChunk),
Chunk: make(map[string]bool),
}
for i := 0; i < int(totalChunk); i++ {
fileState.Chunk[fmt.Sprintf("chunk_%d", i)] = false
}
for _, chunkFile := range existingChunks {
var chunkIndex int
fmt.Sscanf(chunkFile, "chunk_%d", &chunkIndex)
fileState.Chunk[fmt.Sprintf("chunk_%d", chunkIndex)] = true
}
newChunkCacheJSON, err := json.Marshal(fileState)
if err != nil {
return nil, err
}
err = r.SetCache(ctx, cacheKey, newChunkCacheJSON, time.Minute*30)
if err != nil {
return nil, err
}
return &fileState, nil
}
return nil, err
}
var existingCache types.FileState
err = json.Unmarshal([]byte(fileJSON), &existingCache)
if err != nil {
return nil, err
}
return &existingCache, nil
}
func (r *RedisServer) UpdateFileChunk(ctx context.Context, fileID, ownerID uuid.UUID, chunk string, totalChunk uint64) error {
chunks, err := r.GetFileChunks(ctx, fileID, ownerID, totalChunk)
if err != nil {
return err
}
chunks.Chunk[fmt.Sprintf("chunk_%s", chunk)] = true
chunks.Done = true
for i := 0; i < int(totalChunk); i++ {
if !chunks.Chunk[fmt.Sprintf("chunk_%d", i)] {
chunks.Done = false
break
}
}
updatedChunkCacheJSON, err := json.Marshal(chunks)
if err != nil {
return err
}
cacheKey := fmt.Sprintf(FileChunkCacheKey, fileID.String())
err = r.SetCache(ctx, cacheKey, updatedChunkCacheJSON, time.Minute*30)
if err != nil {
return err
}
return nil
}
func (r *RedisServer) GetFileDetail(ctx context.Context, fileID uuid.UUID) (*types.FileData, error) {
fileData, err := r.GetFile(ctx, fileID.String())
if err != nil {
return nil, err
}
chunks, err := r.GetFileChunks(ctx, fileData.ID, fileData.OwnerID, fileData.TotalChunk)
if err != nil {
return nil, err
}
return &types.FileData{
ID: fileData.ID,
OwnerID: fileData.OwnerID,
Name: fileData.Name,
Size: fileData.Size,
TotalChunk: fileData.TotalChunk,
StartHash: fileData.StartHash,
EndHash: fileData.EndHash,
Downloaded: fileData.Downloaded,
IsPrivate: fileData.IsPrivate,
Type: fileData.Type,
Done: chunks.Done,
Chunk: chunks.Chunk,
}, nil
}

2
handler/auth/forgotPassword/forgotPassword.go
View File

@ -49,7 +49,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
emailForm := r.Form.Get("email")
user, err := app.Server.Service.GetUser(r.Context(), emailForm)
user, err := app.Server.Cache.GetUser(r.Context(), emailForm)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
component := forgotPasswordView.Main("Filekeeper - Forgot Password Page", types.Message{

2
handler/auth/forgotPassword/verify/verify.go
View File

@ -116,7 +116,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
err = app.Server.Service.RemoveUserCache(r.Context(), userData.User.Email)
err = app.Server.Cache.RemoveUserCache(r.Context(), userData.User.Email)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

2
handler/auth/google/callback/callback.go
View File

@ -135,7 +135,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
return
}
user, err := app.Server.Service.GetUser(r.Context(), oauthUser.Email)
user, err := app.Server.Cache.GetUser(r.Context(), oauthUser.Email)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)

2
handler/auth/signin/signin.go
View File

@ -73,7 +73,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
}
email := r.Form.Get("email")
password := r.Form.Get("password")
userData, err := app.Server.Service.GetUser(r.Context(), email)
userData, err := app.Server.Cache.GetUser(r.Context(), email)
if err != nil {
component := signinView.Main("Filekeeper - Sign in Page", types.Message{
Code: 0,

4
handler/file/delete/delete.go
View File

@ -31,14 +31,14 @@ func DELETE(w http.ResponseWriter, r *http.Request) {
return
}
err = app.Server.Service.RemoveUserFilesCache(r.Context(), userSession.UserID)
err = app.Server.Cache.RemoveUserFilesCache(r.Context(), userSession.UserID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
err = app.Server.Storage.Delete(r.Context(), fmt.Sprintf("%s/%s", file.OwnerID.String(), file.ID.String()))
err = app.Server.Storage.DeleteRecursive(r.Context(), fmt.Sprintf("%s/%s", file.OwnerID.String(), file.ID.String()))
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

6
handler/file/file.go
View File

@ -12,7 +12,7 @@ import (
func GET(w http.ResponseWriter, r *http.Request) {
userSession := r.Context().Value("user").(types.User)
files, err := app.Server.Service.GetUserFiles(r.Context(), userSession.UserID)
files, err := app.Server.Cache.GetUserFiles(r.Context(), userSession.UserID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
@ -21,7 +21,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
var filesData []types.FileData
for _, file := range files {
userFile, err := app.Server.Service.GetFileDetail(r.Context(), file.ID)
userFile, err := app.Server.Cache.GetFileDetail(r.Context(), file.ID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
@ -38,7 +38,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
return
}
usage, err := app.Server.Service.CalculateUserStorageUsage(r.Context(), userSession.UserID.String())
usage, err := app.Server.Cache.CalculateUserStorageUsage(r.Context(), userSession.UserID.String())
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

2
handler/file/query/query.go
View File

@ -31,7 +31,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
var filesData []types.FileData
for _, file := range files {
userFile, err := app.Server.Service.GetFileDetail(r.Context(), file.ID)
userFile, err := app.Server.Cache.GetFileDetail(r.Context(), file.ID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

6
handler/file/rename/rename.go
View File

@ -36,21 +36,21 @@ func PATCH(w http.ResponseWriter, r *http.Request) {
return
}
err = app.Server.Service.RemoveFileCache(r.Context(), fileID)
err = app.Server.Cache.RemoveFileCache(r.Context(), fileID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
err = app.Server.Service.RemoveUserFilesCache(r.Context(), userSession.UserID)
err = app.Server.Cache.RemoveUserFilesCache(r.Context(), userSession.UserID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
userFile, err := app.Server.Service.GetFileDetail(r.Context(), newFile.ID)
userFile, err := app.Server.Cache.GetFileDetail(r.Context(), newFile.ID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

4
handler/file/table/table.go
View File

@ -9,7 +9,7 @@ import (
func GET(w http.ResponseWriter, r *http.Request) {
userSession := r.Context().Value("user").(types.User)
files, err := app.Server.Service.GetUserFiles(r.Context(), userSession.UserID)
files, err := app.Server.Cache.GetUserFiles(r.Context(), userSession.UserID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
@ -18,7 +18,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
var filesData []types.FileData
for _, file := range files {
userFile, err := app.Server.Service.GetFileDetail(r.Context(), file.ID)
userFile, err := app.Server.Cache.GetFileDetail(r.Context(), file.ID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

4
handler/file/upload/upload.go
View File

@ -16,7 +16,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
file, err := app.Server.Service.GetFile(r.Context(), fileID)
file, err := app.Server.Cache.GetFile(r.Context(), fileID)
if err != nil {
app.Server.Logger.Error("error getting upload info: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
@ -52,7 +52,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
app.Server.Logger.Error("error copying byte to file dst: " + err.Error())
return
}
err = app.Server.Service.UpdateFileChunk(r.Context(), file.ID, file.OwnerID, rawIndex, file.TotalChunk)
err = app.Server.Cache.UpdateFileChunk(r.Context(), file.ID, file.OwnerID, rawIndex, file.TotalChunk)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

6
handler/file/visibility/visibility.go
View File

@ -29,21 +29,21 @@ func PUT(w http.ResponseWriter, r *http.Request) {
return
}
err = app.Server.Service.RemoveFileCache(r.Context(), fileID)
err = app.Server.Cache.RemoveFileCache(r.Context(), fileID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
err = app.Server.Service.RemoveUserFilesCache(r.Context(), userSession.UserID)
err = app.Server.Cache.RemoveUserFilesCache(r.Context(), userSession.UserID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
userFile, err := app.Server.Service.GetFileDetail(r.Context(), file.ID)
userFile, err := app.Server.Cache.GetFileDetail(r.Context(), file.ID)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

4
handler/user/ResetPassword/ResetPassword.go
View File

@ -18,7 +18,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
userSession := r.Context().Value("user").(types.User)
currentPassword := r.Form.Get("currentPassword")
password := r.Form.Get("password")
user, err := app.Server.Service.GetUser(r.Context(), userSession.Email)
user, err := app.Server.Cache.GetUser(r.Context(), userSession.Email)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
@ -51,7 +51,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
err = app.Server.Service.RemoveUserCache(r.Context(), userSession.Email)
err = app.Server.Cache.RemoveUserCache(r.Context(), userSession.Email)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

2
handler/user/totp/setup.go
View File

@ -87,7 +87,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
app.Server.Logger.Error(err.Error())
return
}
err := app.Server.Service.RemoveUserCache(r.Context(), userSession.Email)
err := app.Server.Cache.RemoveUserCache(r.Context(), userSession.Email)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

133
handler/user/totp/setup.go~
View File

@ -1,133 +0,0 @@
package userHandlerTotpSetup
import (
"bytes"
"encoding/base64"
"fmt"
"github.com/a-h/templ"
"github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/view/client/user/totp"
"image/png"
"net/http"
"time"
"github.com/fossyy/filekeeper/types"
"github.com/skip2/go-qrcode"
"github.com/xlzd/gotp"
)
func generateQRCode(uri string) (string, error) {
qr, err := qrcode.New(uri, qrcode.Medium)
if err != nil {
return "", fmt.Errorf("failed to generate QR code: %w", err)
}
var buffer bytes.Buffer
if err := png.Encode(&buffer, qr.Image(256)); err != nil {
return "", fmt.Errorf("failed to encode QR code to PNG: %w", err)
}
return base64.StdEncoding.EncodeToString(buffer.Bytes()), nil
}
func GET(w http.ResponseWriter, r *http.Request) {
secret := gotp.RandomSecret(16)
userSession := r.Context().Value("user").(types.User)
totp := gotp.NewDefaultTOTP(secret)
uri := totp.ProvisioningUri(userSession.Email, "filekeeper")
base64Str, err := generateQRCode(uri)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
var component templ.Component
if r.Header.Get("hx-request") == "true" {
component = userTotpSetupView.MainContent("Filekeeper - 2FA Setup Page", base64Str, secret, userSession, types.Message{
Code: 3,
Message: "",
})
} else {
component = userTotpSetupView.Main("Filekeeper - 2FA Setup Page", base64Str, secret, userSession, types.Message{
Code: 3,
Message: "",
})
}
if err := component.Render(r.Context(), w); err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
}
func POST(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
code := r.Form.Get("totp")
secret := r.Form.Get("secret")
totp := gotp.NewDefaultTOTP(secret)
userSession := r.Context().Value("user").(types.User)
uri := totp.ProvisioningUri(userSession.Email, "filekeeper")
base64Str, err := generateQRCode(uri)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
var component templ.Component
if totp.Verify(code, time.Now().Unix()) {
if err := app.Server.Database.InitializeTotp(userSession.Email, secret); err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
err := app.Server.Service.DeleteUser(r.Context(), userSession.Email)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
if r.Header.Get("hx-request") == "true" {
component = userTotpSetupView.MainContent("Filekeeper - 2FA Setup Page", base64Str, secret, userSession, types.Message{
Code: 1,
Message: "Your TOTP setup is complete! Your account is now more secure.",
})
} else {
component = userTotpSetupView.Main("Filekeeper - 2FA Setup Page", base64Str, secret, userSession, types.Message{
Code: 1,
Message: "Your TOTP setup is complete! Your account is now more secure.",
})
}
if err := component.Render(r.Context(), w); err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
return
} else {
if r.Header.Get("hx-request") == "true" {
component = userTotpSetupView.MainContent("Filekeeper - 2FA Setup Page", base64Str, secret, userSession, types.Message{
Code: 0,
Message: "The code you entered is incorrect. Please double-check the code and try again.",
})
} else {
component = userTotpSetupView.Main("Filekeeper - 2FA Setup Page", base64Str, secret, userSession, types.Message{
Code: 0,
Message: "The code you entered is incorrect. Please double-check the code and try again.",
})
}
if err := component.Render(r.Context(), w); err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
return
}
}

2
handler/user/user.go
View File

@ -31,7 +31,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
return
}
usage, err := app.Server.Service.CalculateUserStorageUsage(r.Context(), userSession.UserID.String())
usage, err := app.Server.Cache.CalculateUserStorageUsage(r.Context(), userSession.UserID.String())
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

6
handler/websocket/websocket.go
View File

@ -126,14 +126,14 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
continue
}
err = app.Server.Service.RemoveUserFilesCache(context.Background(), userSession.UserID)
err = app.Server.Cache.RemoveUserFilesCache(context.Background(), userSession.UserID)
if err != nil {
app.Server.Logger.Error(err.Error())
sendErrorResponse(conn, action.Action, "Error Creating File")
return
}
userFile, err := app.Server.Service.GetFileDetail(context.Background(), fileID)
userFile, err := app.Server.Cache.GetFileDetail(context.Background(), fileID)
if err != nil {
app.Server.Logger.Error(err.Error())
sendErrorResponse(conn, action.Action, "Unknown error")
@ -152,7 +152,7 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
sendErrorResponse(conn, action.Action, "File Is Different")
continue
}
userFile, err := app.Server.Service.GetFileDetail(context.Background(), file.ID)
userFile, err := app.Server.Cache.GetFileDetail(context.Background(), file.ID)
if err != nil {
app.Server.Logger.Error(err.Error())
sendErrorResponse(conn, action.Action, "Unknown error")

4
main.go
View File

@ -13,7 +13,6 @@ import (
"github.com/fossyy/filekeeper/middleware"
"github.com/fossyy/filekeeper/routes/admin"
"github.com/fossyy/filekeeper/routes/client"
"github.com/fossyy/filekeeper/service"
"github.com/fossyy/filekeeper/utils"
)
@ -32,7 +31,6 @@ func main() {
database := db.NewPostgresDB(dbUser, dbPass, dbHost, dbPort, dbName, db.DisableSSL)
cacheServer := cache.NewRedisServer(redisHost, redisPort, redisPassword, database)
services := service.NewService(database, cacheServer)
smtpPort, _ := strconv.Atoi(utils.Getenv("SMTP_PORT"))
mailServer := email.NewSmtpServer(utils.Getenv("SMTP_HOST"), smtpPort, utils.Getenv("SMTP_USER"), utils.Getenv("SMTP_PASSWORD"))
@ -44,7 +42,7 @@ func main() {
secretKey := utils.Getenv("S3_SECRET_KEY")
S3 := storage.NewS3(bucket, region, endpoint, accessKey, secretKey)
app.Server = app.NewClientServer(clientAddr, middleware.Handler(client.SetupRoutes()), *logger.Logger(), database, cacheServer, S3, services, mailServer)
app.Server = app.NewClientServer(clientAddr, middleware.Handler(client.SetupRoutes()), *logger.Logger(), database, cacheServer, S3, mailServer)
app.Admin = app.NewAdminServer(adminAddr, middleware.Handler(admin.SetupRoutes()), database)
go func() {

268
service/service.go
View File

@ -1,268 +0,0 @@
package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/types"
"github.com/fossyy/filekeeper/types/models"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
)
const (
UserCacheKey = "UserCache:%s"
UserFilesCacheKey = "UserFilesCache:%s"
FileCacheKey = "FileCache:%s"
FileChunkCacheKey = "FileChunkCache:%s"
)
type Service struct {
db types.Database
cache types.CachingServer
}
func NewService(db types.Database, cache types.CachingServer) *Service {
return &Service{
db: db,
cache: cache,
}
}
func (r *Service) GetUser(ctx context.Context, email string) (*models.User, error) {
cacheKey := fmt.Sprintf(UserCacheKey, email)
userJSON, err := r.cache.GetCache(ctx, cacheKey)
if err != nil {
if errors.Is(err, redis.Nil) {
userData, err := r.db.GetUser(email)
if err != nil {
return nil, err
}
user := &models.User{
UserID: userData.UserID,
Username: userData.Username,
Email: userData.Email,
Password: userData.Password,
Totp: userData.Totp,
}
newUserJSON, _ := json.Marshal(user)
err = r.cache.SetCache(ctx, cacheKey, newUserJSON, time.Hour*12)
if err != nil {
return nil, err
}
return user, nil
}
return nil, err
}
var user models.User
err = json.Unmarshal([]byte(userJSON), &user)
if err != nil {
return nil, err
}
return &user, nil
}
func (r *Service) RemoveUserCache(ctx context.Context, email string) error {
cacheKey := fmt.Sprintf(UserCacheKey, email)
return r.cache.DeleteCache(ctx, cacheKey)
}
func (r *Service) GetFile(ctx context.Context, id string) (*models.File, error) {
cacheKey := fmt.Sprintf(FileCacheKey, id)
fileJSON, err := r.cache.GetCache(ctx, cacheKey)
if err != nil {
if errors.Is(err, redis.Nil) {
fileData, err := r.db.GetFile(id)
if err != nil {
return nil, err
}
newFileJSON, _ := json.Marshal(fileData)
err = r.cache.SetCache(ctx, cacheKey, newFileJSON, time.Hour*24)
if err != nil {
return nil, err
}
return fileData, nil
}
return nil, err
}
var file models.File
err = json.Unmarshal([]byte(fileJSON), &file)
if err != nil {
return nil, err
}
return &file, nil
}
func (r *Service) RemoveFileCache(ctx context.Context, id string) error {
cacheKey := fmt.Sprintf(FileCacheKey, id)
return r.cache.DeleteCache(ctx, cacheKey)
}
func (r *Service) GetUserFiles(ctx context.Context, ownerID uuid.UUID) ([]*models.File, error) {
cacheKey := fmt.Sprintf(UserFilesCacheKey, ownerID.String())
filesJSON, err := r.cache.GetCache(ctx, cacheKey)
if err != nil {
if errors.Is(err, redis.Nil) {
files, err := r.db.GetFiles(ownerID.String(), "", types.All)
if err != nil {
return nil, err
}
filesJSON, err := json.Marshal(files)
if err != nil {
return nil, err
}
err = r.cache.SetCache(ctx, cacheKey, filesJSON, time.Hour*6)
if err != nil {
return nil, err
}
return files, nil
}
return nil, err
}
var files []*models.File
err = json.Unmarshal([]byte(filesJSON), &files)
if err != nil {
return nil, err
}
return files, nil
}
func (r *Service) RemoveUserFilesCache(ctx context.Context, ownerID uuid.UUID) error {
cacheKey := fmt.Sprintf(UserFilesCacheKey, ownerID.String())
return r.cache.DeleteCache(ctx, cacheKey)
}
func (r *Service) CalculateUserStorageUsage(ctx context.Context, ownerID string) (uint64, error) {
files, err := r.db.GetFiles(ownerID, "", types.All)
if err != nil {
return 0, err
}
var total uint64
for _, file := range files {
total += file.Size
}
return total, nil
}
func (r *Service) GetFileChunks(ctx context.Context, fileID, ownerID uuid.UUID, totalChunk uint64) (*types.FileState, error) {
cacheKey := fmt.Sprintf(FileChunkCacheKey, fileID.String())
fileJSON, err := r.cache.GetCache(ctx, cacheKey)
if err != nil {
if errors.Is(err, redis.Nil) {
prefix := fmt.Sprintf("%s/%s/chunk_", ownerID.String(), fileID.String())
existingChunks, err := app.Server.Storage.ListObjects(ctx, prefix)
if err != nil {
return nil, err
}
fileState := types.FileState{
Done: len(existingChunks) == int(totalChunk),
Chunk: make(map[string]bool),
}
for i := 0; i < int(totalChunk); i++ {
fileState.Chunk[fmt.Sprintf("chunk_%d", i)] = false
}
for _, chunkFile := range existingChunks {
var chunkIndex int
fmt.Sscanf(chunkFile, "chunk_%d", &chunkIndex)
fileState.Chunk[fmt.Sprintf("chunk_%d", chunkIndex)] = true
}
newChunkCacheJSON, err := json.Marshal(fileState)
if err != nil {
return nil, err
}
err = r.cache.SetCache(ctx, cacheKey, newChunkCacheJSON, time.Minute*30)
if err != nil {
return nil, err
}
return &fileState, nil
}
return nil, err
}
var existingCache types.FileState
err = json.Unmarshal([]byte(fileJSON), &existingCache)
if err != nil {
return nil, err
}
return &existingCache, nil
}
func (r *Service) UpdateFileChunk(ctx context.Context, fileID, ownerID uuid.UUID, chunk string, totalChunk uint64) error {
chunks, err := r.GetFileChunks(ctx, fileID, ownerID, totalChunk)
if err != nil {
return err
}
chunks.Chunk[fmt.Sprintf("chunk_%s", chunk)] = true
chunks.Done = true
for i := 0; i < int(totalChunk); i++ {
if !chunks.Chunk[fmt.Sprintf("chunk_%d", i)] {
chunks.Done = false
break
}
}
updatedChunkCacheJSON, err := json.Marshal(chunks)
if err != nil {
return err
}
cacheKey := fmt.Sprintf(FileChunkCacheKey, fileID.String())
err = r.cache.SetCache(ctx, cacheKey, updatedChunkCacheJSON, time.Minute*30)
if err != nil {
return err
}
return nil
}
func (r *Service) GetFileDetail(ctx context.Context, fileID uuid.UUID) (*types.FileData, error) {
fileData, err := r.GetFile(ctx, fileID.String())
if err != nil {
return nil, err
}
chunks, err := r.GetFileChunks(ctx, fileData.ID, fileData.OwnerID, fileData.TotalChunk)
if err != nil {
return nil, err
}
return &types.FileData{
ID: fileData.ID,
OwnerID: fileData.OwnerID,
Name: fileData.Name,
Size: fileData.Size,
TotalChunk: fileData.TotalChunk,
StartHash: fileData.StartHash,
EndHash: fileData.EndHash,
Downloaded: fileData.Downloaded,
IsPrivate: fileData.IsPrivate,
Type: fileData.Type,
Done: chunks.Done,
Chunk: chunks.Chunk,
}, nil
}

33
storage/storage.go
View File

@ -7,7 +7,9 @@ import (
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"io"
"path"
"path/filepath"
"sync"
)
type S3 struct {
@ -54,6 +56,37 @@ func (storage *S3) Add(ctx context.Context, key string, data []byte) error {
return nil
}
func (storage *S3) DeleteRecursive(ctx context.Context, key string) error {
if key[len(key)-1] != '/' {
key += "/"
}
objects, err := storage.ListObjects(ctx, key)
if err != nil {
return err
}
var wg sync.WaitGroup
var deleteErr error
var mu sync.Mutex
for _, object := range objects {
wg.Add(1)
go func(object string) {
defer wg.Done()
err := storage.Delete(ctx, path.Join(key, object))
if err != nil {
mu.Lock()
deleteErr = fmt.Errorf("failed to delete object %s: %w", object, err)
mu.Unlock()
}
}(object)
}
wg.Wait()
return deleteErr
}
func (storage *S3) Delete(ctx context.Context, key string) error {
err := storage.Client.RemoveObject(ctx, storage.Bucket, key, minio.RemoveObjectOptions{})
if err != nil {

4
types/types.go
View File

@ -88,9 +88,6 @@ type CachingServer interface {
SetCache(ctx context.Context, key string, value interface{}, expiration time.Duration) error
DeleteCache(ctx context.Context, key string) error
GetKeys(ctx context.Context, pattern string) ([]string, error)
}
type Services interface {
GetUser(ctx context.Context, email string) (*models.User, error)
RemoveUserCache(ctx context.Context, email string) error
GetFile(ctx context.Context, id string) (*models.File, error)
@ -106,6 +103,7 @@ type Services interface {
type Storage interface {
Get(ctx context.Context, key string) ([]byte, error)
Add(ctx context.Context, key string, data []byte) error
DeleteRecursive(ctx context.Context, key string) error
Delete(ctx context.Context, key string) error
ListObjects(ctx context.Context, prefix string) ([]string, error)
}