bagas/filekeeper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement file caching for faster retrieval and reduced database load

Browse Source
This commit is contained in:
Bagas Aulia Rezki
2024-05-06 21:27:26 +07:00
parent df4f7cc9c8
commit 3e3e95bef4
10 changed files with 271 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
handler/upload/initialisation/initialisation.go
View File

@ -3,6 +3,8 @@ package initialisation
import (
"encoding/json"
"errors"
"fmt"
"github.com/fossyy/filekeeper/cache"
"io"
"net/http"
"os"
@ -37,7 +39,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
fileData, err := db.DB.GetUserFile(fileInfo.Name, userSession.UserID.String())
fileData, err := cache.GetUserFile(fileInfo.Name, userSession.UserID.String())
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
upload, err := handleNewUpload(userSession, fileInfo)
@ -52,27 +54,22 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
info, err := db.DB.GetUploadInfo(fileData.ID.String())
if err != nil {
log.Error(err.Error())
return
}
if info.Done {
if fileData.Done {
respondJSON(w, map[string]bool{"Done": true})
return
}
respondJSON(w, info)
fmt.Println("nih2 : ", fileData)
respondJSON(w, fileData)
}
func handleNewUpload(user types.User, file types.FileInfo) (models.FilesUploaded, error) {
func handleNewUpload(user types.User, file types.FileInfo) (models.File, error) {
uploadDir := "uploads"
if _, err := os.Stat(uploadDir); os.IsNotExist(err) {
log.Error(err.Error())
err := os.Mkdir(uploadDir, os.ModePerm)
if err != nil {
log.Error(err.Error())
return models.FilesUploaded{}, err
return models.File{}, err
}
}
@ -83,45 +80,33 @@ func handleNewUpload(user types.User, file types.FileInfo) (models.FilesUploaded
basePath := filepath.Join(currentDir, uploadDir)
saveFolder := filepath.Join(basePath, ownerID.String(), fileID.String())
if filepath.Dir(saveFolder) != filepath.Join(basePath, ownerID.String()) {
return models.FilesUploaded{}, errors.New("invalid path")
return models.File{}, errors.New("invalid path")
}
err := os.MkdirAll(saveFolder, os.ModePerm)
if err != nil {
log.Error(err.Error())
return models.FilesUploaded{}, err
return models.File{}, err
}
newFile := models.File{
ID: fileID,
OwnerID: ownerID,
Name: file.Name,
Size: file.Size,
Downloaded: 0,
ID: fileID,
OwnerID: ownerID,
Name: file.Name,
Size: file.Size,
Downloaded: 0,
UploadedByte: 0,
UploadedChunk: -1,
Done: false,
}
err = db.DB.CreateFile(&newFile)
if err != nil {
log.Error(err.Error())
return models.FilesUploaded{}, err
return models.File{}, err
}
filesUploaded := models.FilesUploaded{
UploadID: uuid.New(),
FileID: fileID,
OwnerID: ownerID,
Name: file.Name,
Size: file.Size,
Uploaded: -1,
Done: false,
}
err = db.DB.CreateUploadInfo(filesUploaded)
if err != nil {
log.Error(err.Error())
return models.FilesUploaded{}, err
}
return filesUploaded, nil
return newFile, nil
}
func respondJSON(w http.ResponseWriter, data interface{}) {

83
handler/upload/upload.go
View File

@ -1,22 +1,18 @@
package uploadHandler
import (
"errors"
"github.com/fossyy/filekeeper/db"
"github.com/fossyy/filekeeper/cache"
"github.com/fossyy/filekeeper/logger"
"github.com/fossyy/filekeeper/types"
filesView "github.com/fossyy/filekeeper/view/upload"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"sync"
"github.com/fossyy/filekeeper/logger"
filesView "github.com/fossyy/filekeeper/view/upload"
)
var log *logger.AggregatedLogger
var mu sync.Mutex
func init() {
log = logger.Logger()
@ -25,7 +21,7 @@ func init() {
func GET(w http.ResponseWriter, r *http.Request) {
component := filesView.Main("upload page")
if err := component.Render(r.Context(), w); err != nil {
handleError(w, err, http.StatusInternalServerError)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
@ -33,32 +29,31 @@ func GET(w http.ResponseWriter, r *http.Request) {
func POST(w http.ResponseWriter, r *http.Request) {
fileID := r.PathValue("id")
if err := r.ParseMultipartForm(32 << 20); err != nil {
handleError(w, err, http.StatusInternalServerError)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
userSession := r.Context().Value("user").(types.User)
if r.FormValue("done") == "true" {
db.DB.FinalizeFileUpload(fileID)
return
}
uploadDir := "uploads"
if err := createUploadDirectory(uploadDir); err != nil {
handleError(w, err, http.StatusInternalServerError)
return
if _, err := os.Stat(uploadDir); os.IsNotExist(err) {
if err := os.Mkdir(uploadDir, os.ModePerm); err != nil {
log.Error("error getting upload info: " + err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
file, err := db.DB.GetUploadInfo(fileID)
file, err := cache.GetFile(fileID)
if err != nil {
log.Error("error getting upload info: " + err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
currentDir, _ := os.Getwd()
basePath := filepath.Join(currentDir, uploadDir)
saveFolder := filepath.Join(basePath, userSession.UserID.String(), file.FileID.String())
saveFolder := filepath.Join(basePath, userSession.UserID.String(), file.ID.String())
if filepath.Dir(saveFolder) != filepath.Join(basePath, userSession.UserID.String()) {
log.Error("invalid path")
@ -66,49 +61,39 @@ func POST(w http.ResponseWriter, r *http.Request) {
return
}
fileByte, _, err := r.FormFile("chunk")
fileByte, fileHeader, err := r.FormFile("chunk")
if err != nil {
handleError(w, err, http.StatusInternalServerError)
log.Error("error getting upload info: " + err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer fileByte.Close()
dst, err := os.OpenFile(filepath.Join(saveFolder, file.Name), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
handleError(w, err, http.StatusInternalServerError)
return
}
defer dst.Close()
if _, err := io.Copy(dst, fileByte); err != nil {
handleError(w, err, http.StatusInternalServerError)
return
}
rawIndex := r.FormValue("index")
index, err := strconv.Atoi(rawIndex)
if err != nil {
return
}
db.DB.UpdateUpdateIndex(index, fileID)
}
func createUploadDirectory(uploadDir string) error {
if _, err := os.Stat(uploadDir); os.IsNotExist(err) {
if err := os.Mkdir(uploadDir, os.ModePerm); err != nil {
return err
}
}
return nil
}
file.UpdateProgress(int64(index), file.UploadedByte+int64(fileHeader.Size))
func handleCookieError(w http.ResponseWriter, r *http.Request, err error) {
if errors.Is(err, http.ErrNoCookie) {
http.Redirect(w, r, "/signin", http.StatusSeeOther)
dst, err := os.OpenFile(filepath.Join(saveFolder, file.Name), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
log.Error("error making upload folder: " + err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
handleError(w, err, http.StatusInternalServerError)
}
func handleError(w http.ResponseWriter, err error, status int) {
http.Error(w, err.Error(), status)
log.Error(err.Error())
defer dst.Close()
if _, err := io.Copy(dst, fileByte); err != nil {
log.Error("error copying byte to file dst: " + err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if file.UploadedByte >= file.Size {
file.FinalizeFileUpload()
return
}
return
}