bagas/filekeeper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement Redis caching for S3 list object calls

Browse Source
This commit is contained in:
bagas
2024-09-23 10:25:10 +07:00
parent 5656c5a616
commit 9452d48bbd
14 changed files with 279 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
handler/file/file.go
View File

@ -8,7 +8,6 @@ import (
"github.com/fossyy/filekeeper/utils"
fileView "github.com/fossyy/filekeeper/view/client/file"
"net/http"
"strconv"
)
func GET(w http.ResponseWriter, r *http.Request) {
@ -22,26 +21,14 @@ func GET(w http.ResponseWriter, r *http.Request) {
var filesData []types.FileData
for _, file := range files {
prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
userFile, err := app.Server.Service.GetUserFile(r.Context(), file.Name, file.OwnerID.String())
if err != nil {
app.Server.Logger.Error(err.Error())
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
missingChunk := len(existingChunks) != int(file.TotalChunk)
filesData = append(filesData, types.FileData{
ID: file.ID.String(),
Name: file.Name,
Size: utils.ConvertFileSize(file.Size),
IsPrivate: file.IsPrivate,
Type: file.Type,
Done: !missingChunk,
Downloaded: strconv.FormatUint(file.Downloaded, 10),
})
filesData = append(filesData, *userFile)
}
allowance, err := app.Server.Database.GetAllowance(userSession.UserID)
@ -50,7 +37,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
app.Server.Logger.Error(err.Error())
return
}
usage, err := app.Server.Service.GetUserStorageUsage(userSession.UserID.String())
usage, err := app.Server.Service.GetUserStorageUsage(r.Context(), userSession.UserID.String())
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

19
handler/file/query/query.go
View File

@ -1,13 +1,10 @@
package queryHandler
import (
"fmt"
"github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/types"
"github.com/fossyy/filekeeper/utils"
fileView "github.com/fossyy/filekeeper/view/client/file"
"net/http"
"strconv"
)
func GET(w http.ResponseWriter, r *http.Request) {
@ -34,26 +31,14 @@ func GET(w http.ResponseWriter, r *http.Request) {
var filesData []types.FileData
for _, file := range files {
prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
userFile, err := app.Server.Service.GetUserFile(r.Context(), file.Name, file.OwnerID.String())
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
missingChunk := len(existingChunks) != int(file.TotalChunk)
filesData = append(filesData, types.FileData{
ID: file.ID.String(),
Name: file.Name,
Size: utils.ConvertFileSize(file.Size),
IsPrivate: file.IsPrivate,
Type: file.Type,
Done: !missingChunk,
Downloaded: strconv.FormatUint(file.Downloaded, 10),
})
filesData = append(filesData, *userFile)
}
if r.Header.Get("hx-request") == "true" {

21
handler/file/rename/rename.go
View File

@ -1,13 +1,10 @@
package renameFileHandler
import (
"fmt"
"github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/types"
"github.com/fossyy/filekeeper/utils"
fileView "github.com/fossyy/filekeeper/view/client/file"
"net/http"
"strconv"
)
func PATCH(w http.ResponseWriter, r *http.Request) {
@ -39,28 +36,14 @@ func PATCH(w http.ResponseWriter, r *http.Request) {
return
}
prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
userFile, err := app.Server.Service.GetUserFile(r.Context(), newFile.Name, newFile.OwnerID.String())
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
missingChunk := len(existingChunks) != int(file.TotalChunk)
fileData := types.FileData{
ID: newFile.ID.String(),
Name: newFile.Name,
Size: utils.ConvertFileSize(newFile.Size),
IsPrivate: newFile.IsPrivate,
Type: newFile.Type,
Done: !missingChunk,
Downloaded: strconv.FormatUint(newFile.Downloaded, 10),
}
component := fileView.JustFile(fileData)
component := fileView.JustFile(*userFile)
err = component.Render(r.Context(), w)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)

4
handler/file/upload/upload.go
View File

@ -16,7 +16,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
file, err := app.Server.Service.GetFile(fileID)
file, err := app.Server.Service.GetFile(r.Context(), fileID)
if err != nil {
app.Server.Logger.Error("error getting upload info: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
@ -52,7 +52,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
app.Server.Logger.Error("error copying byte to file dst: " + err.Error())
return
}
app.Server.Service.UpdateFileChunk(r.Context(), file.ID, file.OwnerID, rawIndex, file.TotalChunk)
w.WriteHeader(http.StatusAccepted)
return
}

20
handler/file/visibility/visibility.go
View File

@ -1,13 +1,10 @@
package visibilityHandler
import (
"fmt"
"github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/types"
"github.com/fossyy/filekeeper/utils"
fileView "github.com/fossyy/filekeeper/view/client/file"
"net/http"
"strconv"
)
func PUT(w http.ResponseWriter, r *http.Request) {
@ -32,26 +29,13 @@ func PUT(w http.ResponseWriter, r *http.Request) {
return
}
prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
userFile, err := app.Server.Service.GetUserFile(r.Context(), file.Name, file.OwnerID.String())
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
missingChunk := len(existingChunks) != int(file.TotalChunk)
fileData := types.FileData{
ID: file.ID.String(),
Name: file.Name,
Size: utils.ConvertFileSize(file.Size),
IsPrivate: !file.IsPrivate,
Type: file.Type,
Done: !missingChunk,
Downloaded: strconv.FormatUint(file.Downloaded, 10),
}
component := fileView.JustFile(fileData)
component := fileView.JustFile(*userFile)
err = component.Render(r.Context(), w)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)

2
handler/forgotPassword/verify/verify.go
View File

@ -122,7 +122,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
err = app.Server.Service.DeleteUser(userData.User.Email)
err = app.Server.Service.DeleteUser(r.Context(), userData.User.Email)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

2
handler/user/ResetPassword/ResetPassword.go
View File

@ -51,7 +51,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
err = app.Server.Service.DeleteUser(userSession.Email)
err = app.Server.Service.DeleteUser(r.Context(), userSession.Email)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

2
handler/user/totp/setup.go
View File

@ -87,7 +87,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
app.Server.Logger.Error(err.Error())
return
}
err := app.Server.Service.DeleteUser(userSession.Email)
err := app.Server.Service.DeleteUser(r.Context(), userSession.Email)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())

67
handler/user/user.go
View File

@ -76,7 +76,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
return
}
usage, err := app.Server.Service.GetUserStorageUsage(userSession.UserID.String())
usage, err := app.Server.Service.GetUserStorageUsage(r.Context(), userSession.UserID.String())
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
@ -178,8 +178,9 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
fileID := uuid.New()
newFile := models.File{
ID: uuid.New(),
ID: fileID,
OwnerID: userSession.UserID,
Name: uploadNewFile.Name,
Size: uploadNewFile.Size,
@ -189,42 +190,24 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
TotalChunk: uploadNewFile.Chunk,
Downloaded: 0,
}
err := app.Server.Database.CreateFile(&newFile)
if err != nil {
sendErrorResponse(conn, action.Action, "Error Creating File")
continue
}
fileData := &types.FileWithDetail{
ID: newFile.ID,
OwnerID: newFile.OwnerID,
Name: newFile.Name,
Size: newFile.Size,
Downloaded: newFile.Downloaded,
Done: false,
}
fileData.Chunk = make(map[string]bool)
prefix := fmt.Sprintf("%s/%s/chunk_", userSession.UserID.String(), newFile.ID.String())
existingChunks, err := app.Server.Storage.ListObjects(context.TODO(), prefix)
userFile, err := app.Server.Service.GetUserFile(context.Background(), uploadNewFile.Name, userSession.UserID.String())
if err != nil {
app.Server.Logger.Error(err.Error())
sendErrorResponse(conn, action.Action, "Unknown error")
continue
} else {
for i := 0; i < int(newFile.TotalChunk); i++ {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = false
}
for _, chunkFile := range existingChunks {
var chunkIndex int
fmt.Sscanf(chunkFile, "chunk_%d", &chunkIndex)
fileData.Chunk[fmt.Sprintf("chunk_%d", chunkIndex)] = true
}
}
sendSuccessResponseWithID(conn, action.Action, fileData, uploadNewFile.RequestID)
sendSuccessResponseWithID(conn, action.Action, userFile, uploadNewFile.RequestID)
continue
} else {
app.Server.Logger.Error(err.Error())
sendErrorResponse(conn, action.Action, "Unknown error")
continue
}
@ -233,40 +216,14 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
sendErrorResponse(conn, action.Action, "File Is Different")
continue
}
fileData := &types.FileWithDetail{
ID: file.ID,
OwnerID: file.OwnerID,
Name: file.Name,
Size: file.Size,
Downloaded: file.Downloaded,
Chunk: make(map[string]bool),
Done: true,
}
prefix := fmt.Sprintf("%s/%s/chunk_", userSession.UserID.String(), file.ID.String())
existingChunks, err := app.Server.Storage.ListObjects(context.TODO(), prefix)
userFile, err := app.Server.Service.GetUserFile(context.Background(), file.Name, userSession.UserID.String())
if err != nil {
app.Server.Logger.Error(err.Error())
fileData.Done = false
} else {
for i := 0; i < int(file.TotalChunk); i++ {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = false
}
for _, chunkFile := range existingChunks {
var chunkIndex int
fmt.Sscanf(chunkFile, "chunk_%d", &chunkIndex)
fileData.Chunk[fmt.Sprintf("chunk_%d", chunkIndex)] = true
}
for i := 0; i < int(file.TotalChunk); i++ {
if !fileData.Chunk[fmt.Sprintf("chunk_%d", i)] {
fileData.Done = false
break
}
}
sendErrorResponse(conn, action.Action, "Unknown error")
continue
}
sendSuccessResponseWithID(conn, action.Action, fileData, uploadNewFile.RequestID)
sendSuccessResponseWithID(conn, action.Action, userFile, uploadNewFile.RequestID)
continue
case Ping:
sendSuccessResponse(conn, action.Action, map[string]string{"message": "received"})