bagas/filekeeper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #66 from fossyy/staging

Browse Source 
Optimize file chunk validation by reducing system calls
This commit is contained in:
Bagas Aulia Rezki
2024-09-16 22:04:07 +07:00
committed by GitHub
parent 1e001a53b5 e2c0f0b12c
commit ef553530e2
3 changed files with 58 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
handler/file/file.go
View File

@ -8,7 +8,6 @@ import (
"github.com/fossyy/filekeeper/utils"
fileView "github.com/fossyy/filekeeper/view/client/file"
"net/http"
"os"
"path/filepath"
"strconv"
)
@ -22,17 +21,15 @@ func GET(w http.ResponseWriter, r *http.Request) {
return
}
var filesData []types.FileData
for _, file := range files {
saveFolder := filepath.Join("uploads", userSession.UserID.String(), file.ID.String(), file.Name)
missingChunk := false
for j := 0; j < int(file.TotalChunk); j++ {
fileName := fmt.Sprintf("%s/chunk_%d", saveFolder, j)
if _, err := os.Stat(fileName); os.IsNotExist(err) {
missingChunk = true
break
}
}
pattern := fmt.Sprintf("%s/chunk_*", saveFolder)
chunkFiles, err := filepath.Glob(pattern)
missingChunk := err != nil || len(chunkFiles) != int(file.TotalChunk)
filesData = append(filesData, types.FileData{
ID: file.ID.String(),
Name: file.Name,

12
handler/file/visibility/visibility.go
View File

@ -7,7 +7,6 @@ import (
"github.com/fossyy/filekeeper/utils"
fileView "github.com/fossyy/filekeeper/view/client/file"
"net/http"
"os"
"path/filepath"
"strconv"
)
@ -34,15 +33,10 @@ func PUT(w http.ResponseWriter, r *http.Request) {
return
}
saveFolder := filepath.Join("uploads", userSession.UserID.String(), file.ID.String(), file.Name)
missingChunk := false
for j := 0; j < int(file.TotalChunk); j++ {
fileName := fmt.Sprintf("%s/chunk_%d", saveFolder, j)
pattern := fmt.Sprintf("%s/chunk_*", saveFolder)
chunkFiles, err := filepath.Glob(pattern)
if _, err := os.Stat(fileName); os.IsNotExist(err) {
missingChunk = true
break
}
}
missingChunk := err != nil || len(chunkFiles) != int(file.TotalChunk)
fileData := types.FileData{
ID: file.ID.String(),
Name: file.Name,

67
handler/user/user.go
View File

@ -15,7 +15,6 @@ import (
"github.com/gorilla/websocket"
"gorm.io/gorm"
"net/http"
"os"
"path/filepath"
"strings"
)
@ -199,15 +198,31 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
Downloaded: newFile.Downloaded,
}
fileData.Chunk = make(map[string]bool)
fileData.Done = true
saveFolder := filepath.Join("uploads", userSession.UserID.String(), newFile.ID.String(), newFile.Name)
for i := 0; i <= int(newFile.TotalChunk); i++ {
fileName := fmt.Sprintf("%s/chunk_%d", saveFolder, i)
if _, err := os.Stat(fileName); os.IsNotExist(err) {
pattern := fmt.Sprintf("%s/chunk_*", saveFolder)
chunkFiles, err := filepath.Glob(pattern)
if err != nil {
app.Server.Logger.Error(err.Error())
fileData.Done = false
} else {
for i := 0; i <= int(newFile.TotalChunk); i++ {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = false
fileData.Done = false
} else {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = true
}
for _, chunkFile := range chunkFiles {
var chunkIndex int
fmt.Sscanf(filepath.Base(chunkFile), "chunk_%d", &chunkIndex)
fileData.Chunk[fmt.Sprintf("chunk_%d", chunkIndex)] = true
}
for i := 0; i <= int(newFile.TotalChunk); i++ {
if !fileData.Chunk[fmt.Sprintf("chunk_%d", i)] {
fileData.Done = false
break
}
}
}
sendSuccessResponseWithID(conn, action.Action, fileData, uploadNewFile.RequestID)
@ -227,21 +242,37 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
Name: file.Name,
Size: file.Size,
Downloaded: file.Downloaded,
Done: false,
Chunk: make(map[string]bool),
Done: true,
}
fileData.Chunk = make(map[string]bool)
fileData.Done = true
saveFolder := filepath.Join("uploads", userSession.UserID.String(), fileData.ID.String(), fileData.Name)
for i := 0; i <= int(file.TotalChunk-1); i++ {
fileName := fmt.Sprintf("%s/chunk_%d", saveFolder, i)
if _, err := os.Stat(fileName); os.IsNotExist(err) {
saveFolder := filepath.Join("uploads", userSession.UserID.String(), fileData.ID.String(), fileData.Name)
pattern := fmt.Sprintf("%s/chunk_*", saveFolder)
chunkFiles, err := filepath.Glob(pattern)
if err != nil {
app.Server.Logger.Error(err.Error())
fileData.Done = false
} else {
for i := 0; i <= int(file.TotalChunk); i++ {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = false
fileData.Done = false
} else {
fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = true
}
for _, chunkFile := range chunkFiles {
var chunkIndex int
fmt.Sscanf(filepath.Base(chunkFile), "chunk_%d", &chunkIndex)
fileData.Chunk[fmt.Sprintf("chunk_%d", chunkIndex)] = true
}
for i := 0; i <= int(file.TotalChunk); i++ {
if !fileData.Chunk[fmt.Sprintf("chunk_%d", i)] {
fileData.Done = false
break
}
}
}
sendSuccessResponseWithID(conn, action.Action, fileData, uploadNewFile.RequestID)
continue
case Ping: