bagas/filekeeper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Integrate S3 storage solution into the project

Browse Source
This commit is contained in:
bagas
2024-09-20 16:40:04 +07:00
parent 5b1222e024
commit 21a0a92f22
15 changed files with 318 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
handler/file/delete/delete.go
View File

@ -1,11 +1,10 @@
package deleteHandler
import (
"fmt"
"github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/types"
"net/http"
"os"
"path/filepath"
)
func DELETE(w http.ResponseWriter, r *http.Request) {
@ -30,11 +29,7 @@ func DELETE(w http.ResponseWriter, r *http.Request) {
return
}
uploadDir := "uploads"
currentDir, _ := os.Getwd()
basePath := filepath.Join(currentDir, uploadDir)
fileFolder := filepath.Join(basePath, file.OwnerID.String(), file.ID.String())
err = os.RemoveAll(fileFolder)
err = app.Server.Storage.Delete(r.Context(), fmt.Sprintf("%s/%s", file.OwnerID.String(), file.ID.String()))
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return

103
handler/file/download/download.go
View File

@ -1,14 +1,12 @@
package downloadHandler
import (
"context"
"fmt"
"github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/session"
"github.com/fossyy/filekeeper/types/models"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
)
@ -33,16 +31,16 @@ func GET(w http.ResponseWriter, r *http.Request) {
}
}
uploadDir := "uploads"
currentDir, _ := os.Getwd()
basePath := filepath.Join(currentDir, uploadDir)
saveFolder := filepath.Join(basePath, file.OwnerID.String(), file.ID.String())
if filepath.Dir(saveFolder) != filepath.Join(basePath, file.OwnerID.String()) {
http.Error(w, "Invalid Path", http.StatusInternalServerError)
app.Server.Logger.Error("invalid path")
return
}
//uploadDir := "uploads"
//currentDir, _ := os.Getwd()
//basePath := filepath.Join(currentDir, uploadDir)
//saveFolder := filepath.Join(basePath, file.OwnerID.String(), file.ID.String())
//
//if filepath.Dir(saveFolder) != filepath.Join(basePath, file.OwnerID.String()) {
// http.Error(w, "Invalid Path", http.StatusInternalServerError)
// app.Server.Logger.Error("invalid path")
// return
//}
rangeHeader := r.Header.Get("Range")
if rangeHeader != "" {
@ -69,7 +67,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, file.Size))
w.Header().Set("Content-Length", fmt.Sprintf("%d", end-start+1))
w.WriteHeader(http.StatusPartialContent)
sendFileChunk(w, saveFolder, file, start, end)
sendFileChunk(w, file, start, end)
return
}
}
@ -79,11 +77,11 @@ func GET(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Accept-Ranges", "bytes")
w.Header().Set("Content-Length", fmt.Sprintf("%d", file.Size))
sendFileChunk(w, saveFolder, file, 0, int64(file.Size-1))
sendFileChunk(w, file, 0, int64(file.Size-1))
return
}
func sendFileChunk(w http.ResponseWriter, saveFolder string, file *models.File, start, end int64) {
func sendFileChunk(w http.ResponseWriter, file *models.File, start, end int64) {
chunkSize := int64(2 * 1024 * 1024)
startChunk := start / chunkSize
@ -93,64 +91,39 @@ func sendFileChunk(w http.ResponseWriter, saveFolder string, file *models.File,
endOffset := end % chunkSize
for i := startChunk; i <= endChunk; i++ {
chunkPath := filepath.Join(saveFolder, fmt.Sprintf("chunk_%d", i))
chunkFile, err := os.Open(chunkPath)
chunkKey := fmt.Sprintf("%s/%s/chunk_%d", file.OwnerID.String(), file.ID.String(), i)
chunkData, err := app.Server.Storage.Get(context.TODO(), chunkKey)
if err != nil {
http.Error(w, fmt.Sprintf("Error opening chunk: %v", err), http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
defer chunkFile.Close()
var chunkStart, chunkEnd int64
if i == startChunk {
chunkStart = startOffset
} else {
chunkStart = 0
}
if i == endChunk {
chunkEnd = endOffset
} else {
chunkEnd = chunkSize - 1
}
_, err = chunkFile.Seek(chunkStart, io.SeekStart)
if err != nil {
http.Error(w, fmt.Sprintf("Error seeking chunk: %v", err), http.StatusInternalServerError)
http.Error(w, fmt.Sprintf("Error retrieving chunk: %v", err), http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
buffer := make([]byte, 2048)
toSend := chunkEnd - chunkStart + 1
for toSend > 0 {
n, err := chunkFile.Read(buffer)
if err != nil && err != io.EOF {
http.Error(w, fmt.Sprintf("Error reading chunk: %v", err), http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
if n == 0 {
break
}
if int64(n) > toSend {
n = int(toSend)
}
_, err = w.Write(buffer[:n])
var dataToSend []byte
if i == startChunk && i == endChunk {
dataToSend = chunkData[startOffset : endOffset+1]
} else if i == startChunk {
dataToSend = chunkData[startOffset:]
} else if i == endChunk {
dataToSend = chunkData[:endOffset+1]
} else {
dataToSend = chunkData
}
_, err = w.Write(dataToSend)
if err != nil {
http.Error(w, fmt.Sprintf("Error writing chunk: %v", err), http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
if i == int64(file.TotalChunk)-1 {
err := app.Server.Database.IncrementDownloadCount(file.ID.String())
if err != nil {
http.Error(w, fmt.Sprintf("Error writing chunk: %v", err), http.StatusInternalServerError)
http.Error(w, fmt.Sprintf("Error updating download count: %v", err), http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
toSend -= int64(n)
if i == int64(file.TotalChunk)-1 && toSend == 0 {
err := app.Server.Database.IncrementDownloadCount(file.ID.String())
if err != nil {
http.Error(w, fmt.Sprintf("Error writing chunk: %v", err), http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return
}
}
}
}
}

13
handler/file/file.go
View File

@ -8,7 +8,6 @@ import (
"github.com/fossyy/filekeeper/utils"
fileView "github.com/fossyy/filekeeper/view/client/file"
"net/http"
"path/filepath"
"strconv"
)
@ -23,12 +22,16 @@ func GET(w http.ResponseWriter, r *http.Request) {
var filesData []types.FileData
for _, file := range files {
saveFolder := filepath.Join("uploads", userSession.UserID.String(), file.ID.String())
prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
pattern := fmt.Sprintf("%s/chunk_*", saveFolder)
chunkFiles, err := filepath.Glob(pattern)
existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
if err != nil {
app.Server.Logger.Error(err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
missingChunk := err != nil || len(chunkFiles) != int(file.TotalChunk)
missingChunk := len(existingChunks) != int(file.TotalChunk)
filesData = append(filesData, types.FileData{
ID: file.ID.String(),

16
handler/file/query/query.go
View File

@ -7,7 +7,6 @@ import (
"github.com/fossyy/filekeeper/utils"
fileView "github.com/fossyy/filekeeper/view/client/file"
"net/http"
"path/filepath"
"strconv"
)
@ -16,6 +15,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query().Get("q")
status := r.URL.Query().Get("status")
var fileStatus types.FileStatus
if status == "private" {
fileStatus = types.Private
} else if status == "public" {
@ -23,6 +23,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
} else {
fileStatus = types.All
}
files, err := app.Server.Database.GetFiles(userSession.UserID.String(), query, fileStatus)
if err != nil {
app.Server.Logger.Error(err.Error())
@ -33,12 +34,16 @@ func GET(w http.ResponseWriter, r *http.Request) {
var filesData []types.FileData
for _, file := range files {
saveFolder := filepath.Join("uploads", userSession.UserID.String(), file.ID.String())
prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
pattern := fmt.Sprintf("%s/chunk_*", saveFolder)
chunkFiles, err := filepath.Glob(pattern)
existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
if err != nil {
app.Server.Logger.Error(err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
missingChunk := err != nil || len(chunkFiles) != int(file.TotalChunk)
missingChunk := len(existingChunks) != int(file.TotalChunk)
filesData = append(filesData, types.FileData{
ID: file.ID.String(),
@ -62,5 +67,4 @@ func GET(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(http.StatusForbidden)
return
}

14
handler/file/rename/rename.go
View File

@ -7,7 +7,6 @@ import (
"github.com/fossyy/filekeeper/utils"
fileView "github.com/fossyy/filekeeper/view/client/file"
"net/http"
"path/filepath"
"strconv"
)
@ -38,11 +37,16 @@ func PATCH(w http.ResponseWriter, r *http.Request) {
return
}
saveFolder := filepath.Join("uploads", userSession.UserID.String(), file.ID.String())
pattern := fmt.Sprintf("%s/chunk_*", saveFolder)
chunkFiles, err := filepath.Glob(pattern)
prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
missingChunk := err != nil || len(chunkFiles) != int(file.TotalChunk)
existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
if err != nil {
app.Server.Logger.Error(err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
missingChunk := len(existingChunks) != int(file.TotalChunk)
fileData := types.FileData{
ID: newFile.ID.String(),

51
handler/file/upload/upload.go
View File

@ -3,13 +3,9 @@ package uploadHandler
import (
"fmt"
"github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/types"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
)
func POST(w http.ResponseWriter, r *http.Request) {
@ -19,17 +15,6 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
userSession := r.Context().Value("user").(types.User)
uploadDir := "uploads"
if _, err := os.Stat(uploadDir); os.IsNotExist(err) {
if err := os.Mkdir(uploadDir, os.ModePerm); err != nil {
app.Server.Logger.Error("error getting upload info: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
}
file, err := app.Server.Service.GetFile(fileID)
if err != nil {
app.Server.Logger.Error("error getting upload info: " + err.Error())
@ -43,34 +28,6 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
currentDir, err := os.Getwd()
if err != nil {
app.Server.Logger.Error("unable to get current directory")
w.WriteHeader(http.StatusInternalServerError)
return
}
basePath := filepath.Join(currentDir, uploadDir)
cleanBasePath := filepath.Clean(basePath)
saveFolder := filepath.Join(cleanBasePath, userSession.UserID.String(), file.ID.String())
cleanSaveFolder := filepath.Clean(saveFolder)
if !strings.HasPrefix(cleanSaveFolder, cleanBasePath) {
app.Server.Logger.Error("invalid path")
w.WriteHeader(http.StatusInternalServerError)
return
}
if _, err := os.Stat(saveFolder); os.IsNotExist(err) {
if err := os.MkdirAll(saveFolder, os.ModePerm); err != nil {
app.Server.Logger.Error("error creating save folder: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
}
fileByte, _, err := r.FormFile("chunk")
if err != nil {
app.Server.Logger.Error("error getting upload info: " + err.Error())
@ -79,15 +36,15 @@ func POST(w http.ResponseWriter, r *http.Request) {
}
defer fileByte.Close()
dst, err := os.OpenFile(filepath.Join(saveFolder, fmt.Sprintf("chunk_%d", index)), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
buffer, err := io.ReadAll(fileByte)
if err != nil {
app.Server.Logger.Error("error making upload folder: " + err.Error())
app.Server.Logger.Error("error copying byte to file dst: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
defer dst.Close()
if _, err := io.Copy(dst, fileByte); err != nil {
err = app.Server.Storage.Add(r.Context(), fmt.Sprintf("%s/%s/chunk_%d", file.OwnerID.String(), file.ID.String(), index), buffer)
if err != nil {
app.Server.Logger.Error("error copying byte to file dst: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
return

15
handler/file/visibility/visibility.go
View File

@ -7,7 +7,6 @@ import (
"github.com/fossyy/filekeeper/utils"
fileView "github.com/fossyy/filekeeper/view/client/file"
"net/http"
"path/filepath"
"strconv"
)
@ -32,11 +31,17 @@ func PUT(w http.ResponseWriter, r *http.Request) {
app.Server.Logger.Error(err.Error())
return
}
saveFolder := filepath.Join("uploads", userSession.UserID.String(), file.ID.String())
pattern := fmt.Sprintf("%s/chunk_*", saveFolder)
chunkFiles, err := filepath.Glob(pattern)
missingChunk := err != nil || len(chunkFiles) != int(file.TotalChunk)
prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
if err != nil {
app.Server.Logger.Error(err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
missingChunk := len(existingChunks) != int(file.TotalChunk)
fileData := types.FileData{
ID: file.ID.String(),
Name: file.Name,

35
handler/user/user.go
View File

@ -1,6 +1,7 @@
package userHandler
import (
"context"
"encoding/json"
"errors"
"fmt"
@ -15,7 +16,6 @@ import (
"github.com/gorilla/websocket"
"gorm.io/gorm"
"net/http"
"path/filepath"
"strings"
)
@ -196,34 +196,27 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
Name: newFile.Name,
Size: newFile.Size,
Downloaded: newFile.Downloaded,
Done: false,
}
fileData.Chunk = make(map[string]bool)
saveFolder := filepath.Join("uploads", userSession.UserID.String(), newFile.ID.String())
prefix := fmt.Sprintf("%s/%s/chunk_", userSession.UserID.String(), newFile.ID.String())
pattern := fmt.Sprintf("%s/chunk_*", saveFolder)
chunkFiles, err := filepath.Glob(pattern)
existingChunks, err := app.Server.Storage.ListObjects(context.TODO(), prefix)
if err != nil {
app.Server.Logger.Error(err.Error())
fileData.Done = false
sendErrorResponse(conn, action.Action, "Unknown error")
continue
} else {
for i := 0; i <= int(newFile.TotalChunk); i++ {
for i := 0; i < int(newFile.TotalChunk); i++ {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = false
}
for _, chunkFile := range chunkFiles {
for _, chunkFile := range existingChunks {
var chunkIndex int
fmt.Sscanf(filepath.Base(chunkFile), "chunk_%d", &chunkIndex)
fmt.Sscanf(chunkFile, "chunk_%d", &chunkIndex)
fileData.Chunk[fmt.Sprintf("chunk_%d", chunkIndex)] = true
}
for i := 0; i <= int(newFile.TotalChunk); i++ {
if !fileData.Chunk[fmt.Sprintf("chunk_%d", i)] {
fileData.Done = false
break
}
}
}
sendSuccessResponseWithID(conn, action.Action, fileData, uploadNewFile.RequestID)
continue
@ -246,10 +239,8 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
Done: true,
}
saveFolder := filepath.Join("uploads", userSession.UserID.String(), fileData.ID.String())
pattern := fmt.Sprintf("%s/chunk_*", saveFolder)
chunkFiles, err := filepath.Glob(pattern)
prefix := fmt.Sprintf("%s/%s/chunk_", userSession.UserID.String(), file.ID.String())
existingChunks, err := app.Server.Storage.ListObjects(context.TODO(), prefix)
if err != nil {
app.Server.Logger.Error(err.Error())
fileData.Done = false
@ -257,9 +248,9 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
for i := 0; i < int(file.TotalChunk); i++ {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = false
}
for _, chunkFile := range chunkFiles {
for _, chunkFile := range existingChunks {
var chunkIndex int
fmt.Sscanf(filepath.Base(chunkFile), "chunk_%d", &chunkIndex)
fmt.Sscanf(chunkFile, "chunk_%d", &chunkIndex)
fileData.Chunk[fmt.Sprintf("chunk_%d", chunkIndex)] = true
}