bagas/filekeeper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement new file upload and download mechanism

Browse Source
This commit is contained in:
Bagas Aulia Rezki
2024-09-08 16:43:50 +07:00
parent 29ab28fd93
commit 72cb594128
8 changed files with 187 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
db/database.go
View File

@ -235,27 +235,6 @@ func (db *mySQLdb) GetFiles(ownerID string) ([]*models.File, error) {
return files, err return files, err
} }
func (db *mySQLdb) UpdateUploadedByte(byte int64, fileID string) {
var file models.File
db.DB.Table("files").Where("id = ?", fileID).First(&file)
file.UploadedByte = byte
db.Save(&file)
}
func (db *mySQLdb) UpdateUploadedChunk(index int64, fileID string) {
var file models.File
db.DB.Table("files").Where("id = ?", fileID).First(&file)
file.UploadedChunk = index
db.Save(&file)
}
func (db *mySQLdb) FinalizeFileUpload(fileID string) {
var file models.File
db.DB.Table("files").Where("id = ?", fileID).First(&file)
file.Done = true
db.Save(&file)
}
func (db *mySQLdb) InitializeTotp(email string, secret string) error { func (db *mySQLdb) InitializeTotp(email string, secret string) error {
var user models.User var user models.User
err := db.DB.Table("users").Where("email = ?", email).First(&user).Error err := db.DB.Table("users").Where("email = ?", email).First(&user).Error
@ -316,7 +295,6 @@ func (db *postgresDB) GetAllUsers() ([]models.User, error) {
var users []models.User var users []models.User
err := db.DB.Table("users").Select("user_id, username, email").Find(&users).Error err := db.DB.Table("users").Select("user_id, username, email").Find(&users).Error
if err != nil { if err != nil {
fmt.Println(err)
return nil, err return nil, err
} }
return users, nil return users, nil
@ -368,26 +346,6 @@ func (db *postgresDB) GetFiles(ownerID string) ([]*models.File, error) {
return files, err return files, err
} }
func (db *postgresDB) UpdateUploadedByte(byte int64, fileID string) {
var file models.File
db.DB.Table("files").Where("id = $1", fileID).First(&file)
file.UploadedByte = byte
db.Save(&file)
}
func (db *postgresDB) UpdateUploadedChunk(index int64, fileID string) {
var file models.File
db.DB.Table("files").Where("id = $1", fileID).First(&file)
file.UploadedChunk = index
db.Save(&file)
}
func (db *postgresDB) FinalizeFileUpload(fileID string) {
var file models.File
db.DB.Table("files").Where("id = $1", fileID).First(&file)
file.Done = true
db.Save(&file)
}
func (db *postgresDB) InitializeTotp(email string, secret string) error { func (db *postgresDB) InitializeTotp(email string, secret string) error {
var user models.User var user models.User
err := db.DB.Table("users").Where("email = $1", email).First(&user).Error err := db.DB.Table("users").Where("email = $1", email).First(&user).Error

25
handler/download/file/file.go
View File

@ -1,7 +1,9 @@
package downloadFileHandler package downloadFileHandler
import ( import (
"fmt"
"github.com/fossyy/filekeeper/app" "github.com/fossyy/filekeeper/app"
"io"
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
@ -28,22 +30,21 @@ func GET(w http.ResponseWriter, r *http.Request) {
return return
} }
openFile, err := os.OpenFile(filepath.Join(saveFolder, file.Name), os.O_RDONLY, 0) w.Header().Set("Content-Disposition", "attachment; filename="+file.Name)
w.Header().Set("Content-Type", "application/octet-stream")
for i := 0; i <= int(file.TotalChunk); i++ {
chunkPath := filepath.Join(saveFolder, file.Name, fmt.Sprintf("chunk_%d", i))
chunkFile, err := os.Open(chunkPath)
if err != nil { if err != nil {
w.WriteHeader(http.StatusInternalServerError) http.Error(w, fmt.Sprintf("Error opening chunk: %v", err), http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return return
} }
defer openFile.Close() _, err = io.Copy(w, chunkFile)
chunkFile.Close()
stat, err := openFile.Stat()
if err != nil { if err != nil {
w.WriteHeader(http.StatusInternalServerError) http.Error(w, fmt.Sprintf("Error writing chunk: %v", err), http.StatusInternalServerError)
app.Server.Logger.Error(err.Error())
return return
} }
}
w.Header().Set("Content-Disposition", "attachment; filename="+stat.Name())
http.ServeContent(w, r, stat.Name(), stat.ModTime(), openFile)
return
} }

42
handler/upload/initialisation/initialisation.go
View File

@ -3,6 +3,7 @@ package initialisation
import ( import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"github.com/fossyy/filekeeper/app" "github.com/fossyy/filekeeper/app"
"io" "io"
"net/http" "net/http"
@ -38,6 +39,26 @@ func POST(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
return return
} }
fileData = &types.FileWithDetail{
ID: fileData.ID,
OwnerID: fileData.OwnerID,
Name: fileData.Name,
Size: fileData.Size,
Downloaded: fileData.Downloaded,
}
fileData.Chunk = make(map[string]bool)
fileData.Done = true
saveFolder := filepath.Join("uploads", userSession.UserID.String(), fileData.ID.String(), fileData.Name)
for i := 0; i <= int(fileInfo.Chunk-1); i++ {
fileName := fmt.Sprintf("%s/chunk_%d", saveFolder, i)
if _, err := os.Stat(fileName); os.IsNotExist(err) {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = false
fileData.Done = false
} else {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = true
}
}
respondJSON(w, upload) respondJSON(w, upload)
return return
} }
@ -45,11 +66,19 @@ func POST(w http.ResponseWriter, r *http.Request) {
return return
} }
if fileData.Done { fileData.Chunk = make(map[string]bool)
respondJSON(w, map[string]bool{"Done": true}) fileData.Done = true
return saveFolder := filepath.Join("uploads", userSession.UserID.String(), fileData.ID.String(), fileData.Name)
} for i := 0; i <= int(fileInfo.Chunk-1); i++ {
fileName := fmt.Sprintf("%s/chunk_%d", saveFolder, i)
if _, err := os.Stat(fileName); os.IsNotExist(err) {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = false
fileData.Done = false
} else {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = true
}
}
respondJSON(w, fileData) respondJSON(w, fileData)
} }
@ -85,10 +114,8 @@ func handleNewUpload(user types.User, file types.FileInfo) (models.File, error)
OwnerID: ownerID, OwnerID: ownerID,
Name: file.Name, Name: file.Name,
Size: file.Size, Size: file.Size,
TotalChunk: file.Chunk - 1,
Downloaded: 0, Downloaded: 0,
UploadedByte: 0,
UploadedChunk: -1,
Done: false,
} }
err = app.Server.Database.CreateFile(&newFile) err = app.Server.Database.CreateFile(&newFile)
@ -96,7 +123,6 @@ func handleNewUpload(user types.User, file types.FileInfo) (models.File, error)
app.Server.Logger.Error(err.Error()) app.Server.Logger.Error(err.Error())
return models.File{}, err return models.File{}, err
} }
return newFile, nil return newFile, nil
} }

157
handler/upload/upload.go
View File

@ -1,8 +1,16 @@
package uploadHandler package uploadHandler
import ( import (
"fmt"
"github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/types"
filesView "github.com/fossyy/filekeeper/view/client/upload" filesView "github.com/fossyy/filekeeper/view/client/upload"
"io"
"net/http" "net/http"
"os"
"path/filepath"
"strconv"
"strings"
) )
func GET(w http.ResponseWriter, r *http.Request) { func GET(w http.ResponseWriter, r *http.Request) {
@ -14,74 +22,85 @@ func GET(w http.ResponseWriter, r *http.Request) {
} }
func POST(w http.ResponseWriter, r *http.Request) { func POST(w http.ResponseWriter, r *http.Request) {
fileID := r.PathValue("id")
if err := r.ParseMultipartForm(32 << 20); err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
userSession := r.Context().Value("user").(types.User)
uploadDir := "uploads"
if _, err := os.Stat(uploadDir); os.IsNotExist(err) {
if err := os.Mkdir(uploadDir, os.ModePerm); err != nil {
app.Server.Logger.Error("error getting upload info: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
}
file, err := app.Server.Service.GetFile(fileID)
if err != nil {
app.Server.Logger.Error("error getting upload info: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
rawIndex := r.FormValue("index")
index, err := strconv.Atoi(rawIndex)
if err != nil {
return
}
currentDir, err := os.Getwd()
if err != nil {
app.Server.Logger.Error("unable to get current directory")
w.WriteHeader(http.StatusInternalServerError)
return
}
basePath := filepath.Join(currentDir, uploadDir)
cleanBasePath := filepath.Clean(basePath)
saveFolder := filepath.Join(cleanBasePath, userSession.UserID.String(), file.ID.String(), file.Name)
cleanSaveFolder := filepath.Clean(saveFolder)
if !strings.HasPrefix(cleanSaveFolder, cleanBasePath) {
app.Server.Logger.Error("invalid path")
w.WriteHeader(http.StatusInternalServerError)
return
}
if _, err := os.Stat(saveFolder); os.IsNotExist(err) {
if err := os.MkdirAll(saveFolder, os.ModePerm); err != nil {
app.Server.Logger.Error("error creating save folder: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
}
fileByte, _, err := r.FormFile("chunk")
if err != nil {
app.Server.Logger.Error("error getting upload info: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
defer fileByte.Close()
dst, err := os.OpenFile(filepath.Join(saveFolder, fmt.Sprintf("chunk_%d", index)), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
app.Server.Logger.Error("error making upload folder: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
defer dst.Close()
if _, err := io.Copy(dst, fileByte); err != nil {
app.Server.Logger.Error("error copying byte to file dst: " + err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
return return
//fileID := r.PathValue("id")
//if err := r.ParseMultipartForm(32 << 20); err != nil {
// w.WriteHeader(http.StatusInternalServerError)
// return
//}
//
//userSession := r.Context().Value("user").(types.User)
//
//uploadDir := "uploads"
//if _, err := os.Stat(uploadDir); os.IsNotExist(err) {
// if err := os.Mkdir(uploadDir, os.ModePerm); err != nil {
// app.Server.Logger.Error("error getting upload info: " + err.Error())
// w.WriteHeader(http.StatusInternalServerError)
// return
// }
//}
//
//file, err := app.Server.Service.GetFile(fileID)
//if err != nil {
// app.Server.Logger.Error("error getting upload info: " + err.Error())
// w.WriteHeader(http.StatusInternalServerError)
// return
//}
//
//currentDir, _ := os.Getwd()
//basePath := filepath.Join(currentDir, uploadDir)
//saveFolder := filepath.Join(basePath, userSession.UserID.String(), file.ID.String())
//
//if filepath.Dir(saveFolder) != filepath.Join(basePath, userSession.UserID.String()) {
// app.Server.Logger.Error("invalid path")
// w.WriteHeader(http.StatusInternalServerError)
// return
//}
//
//fileByte, fileHeader, err := r.FormFile("chunk")
//if err != nil {
// app.Server.Logger.Error("error getting upload info: " + err.Error())
// w.WriteHeader(http.StatusInternalServerError)
// return
//}
//defer fileByte.Close()
//
//rawIndex := r.FormValue("index")
//index, err := strconv.Atoi(rawIndex)
//if err != nil {
// return
//}
//
//file.UpdateProgress(int64(index), file.UploadedByte+int64(fileHeader.Size))
//
//dst, err := os.OpenFile(filepath.Join(saveFolder, file.Name), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
//if err != nil {
// app.Server.Logger.Error("error making upload folder: " + err.Error())
// w.WriteHeader(http.StatusInternalServerError)
// return
//}
//
//defer dst.Close()
//if _, err := io.Copy(dst, fileByte); err != nil {
// app.Server.Logger.Error("error copying byte to file dst: " + err.Error())
// w.WriteHeader(http.StatusInternalServerError)
// return
//}
//
//if file.UploadedByte >= file.Size {
// file.FinalizeFileUpload()
// return
//}
//return
} }

14
public/upload.js
View File

@ -32,7 +32,7 @@ async function handleFile(file){
if (responseData.Done === false) { if (responseData.Done === false) {
addNewUploadElement(file) addNewUploadElement(file)
const fileChunks = await splitFile(file, chunkSize); const fileChunks = await splitFile(file, chunkSize);
await uploadChunks(file.name,file.size, fileChunks, responseData.UploadedChunk, responseData.ID); await uploadChunks(file.name,file.size, fileChunks, responseData.Chunk, responseData.ID);
} else { } else {
alert("file already uploaded") alert("file already uploaded")
} }
@ -123,7 +123,7 @@ async function splitFile(file, chunkSize) {
return fileChunks; return fileChunks;
} }
async function uploadChunks(name, size, chunks, uploadedChunk= -1, FileID) { async function uploadChunks(name, size, chunks, chunkArray, FileID) {
let byteUploaded = 0 let byteUploaded = 0
let progress1 = document.getElementById(`progress-${name}-1`); let progress1 = document.getElementById(`progress-${name}-1`);
let progress2 = document.getElementById(`progress-${name}-2`); let progress2 = document.getElementById(`progress-${name}-2`);
@ -132,7 +132,7 @@ async function uploadChunks(name, size, chunks, uploadedChunk= -1, FileID) {
for (let index = 0; index < chunks.length; index++) { for (let index = 0; index < chunks.length; index++) {
const percentComplete = Math.round((index + 1) / chunks.length * 100); const percentComplete = Math.round((index + 1) / chunks.length * 100);
const chunk = chunks[index]; const chunk = chunks[index];
if (!(index <= uploadedChunk)) { if (!(chunkArray["chunk_"+index])) {
const formData = new FormData(); const formData = new FormData();
formData.append('name', name); formData.append('name', name);
formData.append('chunk', chunk); formData.append('chunk', chunk);
@ -152,13 +152,19 @@ async function uploadChunks(name, size, chunks, uploadedChunk= -1, FileID) {
const totalTime = (endTime - startTime) / 1000; const totalTime = (endTime - startTime) / 1000;
const uploadSpeed = chunk.size / totalTime / 1024 / 1024; const uploadSpeed = chunk.size / totalTime / 1024 / 1024;
byteUploaded += chunk.size byteUploaded += chunk.size
console.log(byteUploaded)
progress3.innerText = `${uploadSpeed.toFixed(2)} MB/s`; progress3.innerText = `${uploadSpeed.toFixed(2)} MB/s`;
progress4.innerText = `Uploading ${percentComplete}% - ${convertFileSize(byteUploaded)} of ${ convertFileSize(size)}`; progress4.innerText = `Uploading ${percentComplete}% - ${convertFileSize(byteUploaded)} of ${ convertFileSize(size)}`;
} else { } else {
progress1.setAttribute("aria-valuenow", percentComplete); progress1.setAttribute("aria-valuenow", percentComplete);
progress2.style.width = `${percentComplete}%`; progress2.style.width = `${percentComplete}%`;
progress3.innerText = `Fixing Missing Byte`;
progress4.innerText = `Uploading Missing Byte ${percentComplete}% - ${convertFileSize(byteUploaded)} of ${ convertFileSize(size)}`;
byteUploaded += chunk.size byteUploaded += chunk.size
} }
} }
console.log(chunks)
console.log(chunkArray)
progress3.innerText = `Done`;
progress4.innerText = `File Uploaded 100% - ${convertFileSize(byteUploaded)} of ${ convertFileSize(size)}`;
} }

42
service/service.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"github.com/fossyy/filekeeper/app" "github.com/fossyy/filekeeper/app"
"github.com/fossyy/filekeeper/types" "github.com/fossyy/filekeeper/types"
"github.com/fossyy/filekeeper/types/models"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"time" "time"
) )
@ -21,7 +22,7 @@ func NewService(db types.Database, cache types.CachingServer) *Service {
} }
} }
func (r *Service) GetUser(ctx context.Context, email string) (*types.UserWithExpired, error) { func (r *Service) GetUser(ctx context.Context, email string) (*models.User, error) {
userJSON, err := app.Server.Cache.GetCache(ctx, "UserCache:"+email) userJSON, err := app.Server.Cache.GetCache(ctx, "UserCache:"+email)
if err == redis.Nil { if err == redis.Nil {
userData, err := r.db.GetUser(email) userData, err := r.db.GetUser(email)
@ -29,13 +30,12 @@ func (r *Service) GetUser(ctx context.Context, email string) (*types.UserWithExp
return nil, err return nil, err
} }
user := &types.UserWithExpired{ user := &models.User{
UserID: userData.UserID, UserID: userData.UserID,
Username: userData.Username, Username: userData.Username,
Email: userData.Email, Email: userData.Email,
Password: userData.Password, Password: userData.Password,
Totp: userData.Totp, Totp: userData.Totp,
AccessAt: time.Now(),
} }
newUserJSON, _ := json.Marshal(user) newUserJSON, _ := json.Marshal(user)
@ -50,7 +50,7 @@ func (r *Service) GetUser(ctx context.Context, email string) (*types.UserWithExp
return nil, err return nil, err
} }
var user types.UserWithExpired var user models.User
err = json.Unmarshal([]byte(userJSON), &user) err = json.Unmarshal([]byte(userJSON), &user)
if err != nil { if err != nil {
return nil, err return nil, err
@ -66,7 +66,7 @@ func (r *Service) DeleteUser(email string) {
} }
} }
func (r *Service) GetFile(id string) (*types.FileWithExpired, error) { func (r *Service) GetFile(id string) (*models.File, error) {
fileJSON, err := r.cache.GetCache(context.Background(), "FileCache:"+id) fileJSON, err := r.cache.GetCache(context.Background(), "FileCache:"+id)
if err == redis.Nil { if err == redis.Nil {
uploadData, err := r.db.GetFile(id) uploadData, err := r.db.GetFile(id)
@ -74,30 +74,18 @@ func (r *Service) GetFile(id string) (*types.FileWithExpired, error) {
return nil, err return nil, err
} }
fileCache := &types.FileWithExpired{ newFileJSON, _ := json.Marshal(uploadData)
ID: uploadData.ID,
OwnerID: uploadData.OwnerID,
Name: uploadData.Name,
Size: uploadData.Size,
Downloaded: uploadData.Downloaded,
UploadedByte: uploadData.UploadedByte,
UploadedChunk: uploadData.UploadedChunk,
Done: uploadData.Done,
AccessAt: time.Now(),
}
newFileJSON, _ := json.Marshal(fileCache)
err = r.cache.SetCache(context.Background(), "FileCache:"+id, newFileJSON, time.Hour*24) err = r.cache.SetCache(context.Background(), "FileCache:"+id, newFileJSON, time.Hour*24)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return fileCache, nil return uploadData, nil
} }
if err != nil { if err != nil {
return nil, err return nil, err
} }
var fileCache types.FileWithExpired var fileCache models.File
err = json.Unmarshal([]byte(fileJSON), &fileCache) err = json.Unmarshal([]byte(fileJSON), &fileCache)
if err != nil { if err != nil {
return nil, err return nil, err
@ -105,16 +93,18 @@ func (r *Service) GetFile(id string) (*types.FileWithExpired, error) {
return &fileCache, nil return &fileCache, nil
} }
func (r *Service) GetUserFile(name, ownerID string) (*types.FileWithExpired, error) { func (r *Service) GetUserFile(name, ownerID string) (*types.FileWithDetail, error) {
fileData, err := r.db.GetUserFile(name, ownerID) fileData, err := r.db.GetUserFile(name, ownerID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
file, err := r.GetFile(fileData.ID.String()) dada := &types.FileWithDetail{
if err != nil { ID: fileData.ID,
return nil, err OwnerID: fileData.OwnerID,
Name: fileData.Name,
Size: fileData.Size,
Downloaded: fileData.Downloaded,
} }
return dada, nil
return file, nil
} }

4
types/models/models.go
View File

@ -15,8 +15,6 @@ type File struct {
OwnerID uuid.UUID `gorm:"not null"` OwnerID uuid.UUID `gorm:"not null"`
Name string `gorm:"not null"` Name string `gorm:"not null"`
Size int64 `gorm:"not null"` Size int64 `gorm:"not null"`
TotalChunk int64 `gorm:"not null"`
Downloaded int64 `gorm:"not null;default=0"` Downloaded int64 `gorm:"not null;default=0"`
UploadedByte int64 `gorm:"not null;default=0"`
UploadedChunk int64 `gorm:"not null;default=0"`
Done bool `gorm:"not null;default=false"`
} }

25
types/types.go
View File

@ -33,25 +33,14 @@ type FileData struct {
Downloaded int64 Downloaded int64
} }
type UserWithExpired struct { type FileWithDetail struct {
UserID uuid.UUID
Username string
Email string
Password string
Totp string
AccessAt time.Time
}
type FileWithExpired struct {
ID uuid.UUID ID uuid.UUID
OwnerID uuid.UUID OwnerID uuid.UUID
Name string Name string
Size int64 Size int64
Downloaded int64 Downloaded int64
UploadedByte int64 Chunk map[string]bool
UploadedChunk int64
Done bool Done bool
AccessAt time.Time
} }
type Database interface { type Database interface {
@ -68,10 +57,6 @@ type Database interface {
GetUserFile(name string, ownerID string) (*models.File, error) GetUserFile(name string, ownerID string) (*models.File, error)
GetFiles(ownerID string) ([]*models.File, error) GetFiles(ownerID string) ([]*models.File, error)
UpdateUploadedByte(index int64, fileID string)
UpdateUploadedChunk(index int64, fileID string)
FinalizeFileUpload(fileID string)
InitializeTotp(email string, secret string) error InitializeTotp(email string, secret string) error
} }
@ -83,8 +68,8 @@ type CachingServer interface {
} }
type Services interface { type Services interface {
GetUser(ctx context.Context, email string) (*UserWithExpired, error) GetUser(ctx context.Context, email string) (*models.User, error)
DeleteUser(email string) DeleteUser(email string)
GetFile(id string) (*FileWithExpired, error) GetFile(id string) (*models.File, error)
GetUserFile(name, ownerID string) (*FileWithExpired, error) GetUserFile(name, ownerID string) (*FileWithDetail, error)
} }