diff --git a/middleware/gin.go b/middleware/gin.go deleted file mode 100644 index a9b07b6..0000000 --- a/middleware/gin.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - * @Author: LinkLeong link@icewhale.com - * @Date: 2021-10-08 10:29:08 - * @LastEditors: LinkLeong - * @LastEditTime: 2022-07-22 11:06:07 - * @FilePath: /CasaOS/middleware/gin.go - * @Description: - * @Website: https://www.casaos.io - * Copyright (c) 2022 by icewhale, All Rights Reserved. - */ -package middleware - -import ( - "fmt" - "net/http" - "runtime/debug" - "strings" - - "github.com/IceWhaleTech/CasaOS/pkg/utils/loger" - "github.com/gin-gonic/gin" - "go.uber.org/zap" -) - -func Cors() gin.HandlerFunc { - return func(c *gin.Context) { - method := c.Request.Method - - c.Header("Access-Control-Allow-Origin", "*") - c.Header("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE,UPDATE") - // 允许跨域设置可以返回其他子段，可以自定义字段 - c.Header("Access-Control-Allow-Headers", "Authorization, Content-Length, X-CSRF-Token, Token,session,Language,Content-Type,Access-Control-Allow-Origin,Access-Control-Allow-Headers,Access-Control-Allow-Methods,Connection,Host,Origin,Referer,User-Agent,X-Requested-With") - // 允许浏览器（客户端）可以解析的头部 （重要） - c.Header("Access-Control-Expose-Headers", "Content-Length, Access-Control-Allow-Origin, Access-Control-Allow-Headers") - // c.Writer.Header().Set("Access-Control-Allow-Headers", "Accept, Authorization, Content-Type, Content-Length, X-CSRF-Token, Token, session, Origin, Host, Connection, Accept-Encoding, Accept-Language, X-Requested-With") - // 设置缓存时间 - c.Header("Access-Control-Max-Age", "172800") - c.Header("Access-Control-Allow-Credentials", "true") - c.Set("Content-Type", "application/json") - //} - - // 允许类型校验 - if method == "OPTIONS" { - c.JSON(http.StatusOK, "ok!") - } - - defer func() { - if err := recover(); err != nil { - fmt.Println(err) - debug.PrintStack() - } - }() - - c.Next() - } -} - -func WriteLog() gin.HandlerFunc { - return func(c *gin.Context) { - if !strings.Contains(c.Request.URL.String(), "password") { - loger.Info("request:", zap.Any("path", c.Request.URL.String()), zap.Any("param", c.Params), zap.Any("query", c.Request.URL.Query()), zap.Any("method", c.Request.Method)) - c.Next() - } - } -} diff --git a/pkg/utils/file/file.go b/pkg/utils/file/file.go index e7fed65..1fd9de2 100644 --- a/pkg/utils/file/file.go +++ b/pkg/utils/file/file.go @@ -362,21 +362,27 @@ func WriteToFullPath(data []byte, fullPath string, perm fs.FileMode) error { func SpliceFiles(dir, path string, length int, startPoint int) error { fullPath := path - IsNotExistCreateFile(fullPath) + if err := IsNotExistCreateFile(fullPath); err != nil { + return err + } file, _ := os.OpenFile(fullPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0o666, ) + defer file.Close() + bufferedWriter := bufio.NewWriter(file) + + // todo: here should have a goroutine to remove each partial file after it is read, to save disk space + for i := 0; i < length+startPoint; i++ { data, err := ioutil.ReadFile(dir + "/" + strconv.Itoa(i+startPoint)) if err != nil { return err } - _, err = bufferedWriter.Write(data) - if err != nil { + if _, err := bufferedWriter.Write(data); err != nil { // recommend to use https://github.com/iceber/iouring-go for faster write return err } } diff --git a/route/route.go b/route/route.go index 8b254f8..a876849 100644 --- a/route/route.go +++ b/route/route.go @@ -1,8 +1,10 @@ package route import ( + "os" + + "github.com/IceWhaleTech/CasaOS-Common/middleware" "github.com/IceWhaleTech/CasaOS-Common/utils/jwt" - "github.com/IceWhaleTech/CasaOS/middleware" "github.com/IceWhaleTech/CasaOS/pkg/config" v1 "github.com/IceWhaleTech/CasaOS/route/v1" @@ -11,12 +13,22 @@ import ( ) func InitRouter() *gin.Engine { - r := gin.Default() + ginMode := gin.ReleaseMode + if config.ServerInfo.RunMode != "" { + ginMode = config.ServerInfo.RunMode + } + if os.Getenv(gin.EnvGinMode) != "" { + ginMode = os.Getenv(gin.EnvGinMode) + } + gin.SetMode(ginMode) + r := gin.New() + r.Use(gin.Recovery()) r.Use(middleware.Cors()) - r.Use(middleware.WriteLog()) r.Use(gzip.Gzip(gzip.DefaultCompression)) - gin.SetMode(config.ServerInfo.RunMode) + if ginMode != gin.ReleaseMode { + r.Use(middleware.WriteLog()) + } // r.StaticFS("/ui", http.FS(web.Static)) // r.GET("/", WebUIHome) diff --git a/route/v1/file.go b/route/v1/file.go index 396074d..64a5f69 100644 --- a/route/v1/file.go +++ b/route/v1/file.go @@ -18,9 +18,11 @@ import ( "github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/pkg/utils/common_err" "github.com/IceWhaleTech/CasaOS/pkg/utils/file" + "github.com/IceWhaleTech/CasaOS/pkg/utils/loger" "github.com/IceWhaleTech/CasaOS/service" "github.com/gin-gonic/gin" uuid "github.com/satori/go.uuid" + "go.uber.org/zap" ) // @Summary 读取文件 @@ -47,7 +49,7 @@ func GetFilerContent(c *gin.Context) { }) return } - //文件读取任务是将文件内容读取到内存中。 + // 文件读取任务是将文件内容读取到内存中。 info, err := ioutil.ReadFile(filePath) if err != nil { c.JSON(common_err.SERVICE_ERROR, model.Result{ @@ -83,7 +85,6 @@ func GetLocalFile(c *gin.Context) { return } c.File(path) - return } // @Summary download @@ -96,7 +97,6 @@ func GetLocalFile(c *gin.Context) { // @Success 200 {string} string "ok" // @Router /file/download [get] func GetDownloadFile(c *gin.Context) { - t := c.Query("format") files := c.Query("files") @@ -135,11 +135,11 @@ func GetDownloadFile(c *gin.Context) { } if !info.IsDir() { - //打开文件 + // 打开文件 fileTmp, _ := os.Open(filePath) defer fileTmp.Close() - //获取文件的名称 + // 获取文件的名称 fileName := path.Base(filePath) c.Header("Content-Disposition", "attachment; filename*=utf-8''"+url2.PathEscape(fileName)) c.File(filePath) @@ -179,7 +179,6 @@ func GetDownloadFile(c *gin.Context) { log.Printf("Failed to archive %s: %v", fname, err) } } - } func GetDownloadSingleFile(c *gin.Context) { @@ -202,7 +201,7 @@ func GetDownloadSingleFile(c *gin.Context) { defer fileTmp.Close() fileName := path.Base(filePath) - //c.Header("Content-Disposition", "inline") + // c.Header("Content-Disposition", "inline") c.Header("Content-Disposition", "attachment; filename*=utf-8''"+url2.PathEscape(fileName)) c.File(filePath) } @@ -248,7 +247,7 @@ func DirPath(c *gin.Context) { info[i].Extensions = ex } } - //Hide the files or folders in operation + // Hide the files or folders in operation fileQueue := make(map[string]string) if len(service.OpStrArr) > 0 { for _, v := range service.OpStrArr { @@ -361,7 +360,6 @@ func PostCreateFile(c *gin.Context) { // @Success 200 {string} string "ok" // @Router /file/upload [get] func GetFileUpload(c *gin.Context) { - relative := c.Query("relativePath") fileName := c.Query("filename") chunkNumber := c.Query("chunkNumber") @@ -405,55 +403,92 @@ func PostFileUpload(c *gin.Context) { hash := file.GetHashByContent([]byte(fileName)) if len(path) == 0 { - c.JSON(common_err.INVALID_PARAMS, model.Result{Success: common_err.INVALID_PARAMS, Message: common_err.GetMsg(common_err.INVALID_PARAMS)}) + loger.Error("path should not be empty") + c.JSON(http.StatusBadRequest, model.Result{Success: common_err.INVALID_PARAMS, Message: common_err.GetMsg(common_err.INVALID_PARAMS)}) return } + tempDir := filepath.Join(path, ".temp", hash+strconv.Itoa(totalChunks)) + "/" if fileName != relative { dirPath = strings.TrimSuffix(relative, fileName) tempDir += dirPath - file.MkDir(path + "/" + dirPath) + if err := file.MkDir(path + "/" + dirPath); err != nil { + loger.Error("error when trying to create `"+path+"/"+dirPath+"`", zap.Error(err)) + c.JSON(http.StatusInternalServerError, model.Result{Success: common_err.SERVICE_ERROR, Message: err.Error()}) + return + } } path += "/" + relative if !file.CheckNotExist(tempDir + chunkNumber) { - file.RMDir(tempDir + chunkNumber) + if err := file.RMDir(tempDir + chunkNumber); err != nil { + loger.Error("error when trying to remove existing `"+tempDir+chunkNumber+"`", zap.Error(err)) + c.JSON(http.StatusInternalServerError, model.Result{Success: common_err.SERVICE_ERROR, Message: err.Error()}) + return + } } if totalChunks > 1 { - file.IsNotExistMkDir(tempDir) - - out, _ := os.OpenFile(tempDir+chunkNumber, os.O_WRONLY|os.O_CREATE, 0644) - defer out.Close() - _, err := io.Copy(out, f) - if err != nil { - c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + if err := file.IsNotExistMkDir(tempDir); err != nil { + loger.Error("error when trying to create `"+tempDir+"`", zap.Error(err)) + c.JSON(http.StatusInternalServerError, model.Result{Success: common_err.SERVICE_ERROR, Message: err.Error()}) return } + + out, err := os.OpenFile(tempDir+chunkNumber, os.O_WRONLY|os.O_CREATE, 0o644) + if err != nil { + loger.Error("error when trying to open `"+tempDir+chunkNumber+"` for creation", zap.Error(err)) + c.JSON(http.StatusInternalServerError, model.Result{Success: common_err.SERVICE_ERROR, Message: err.Error()}) + return + } + + defer out.Close() + + if _, err := io.Copy(out, f); err != nil { // recommend to use https://github.com/iceber/iouring-go for faster copy + loger.Error("error when trying to write to `"+tempDir+chunkNumber+"`", zap.Error(err)) + c.JSON(http.StatusInternalServerError, model.Result{Success: common_err.SERVICE_ERROR, Message: err.Error()}) + return + } + + fileNum, err := ioutil.ReadDir(tempDir) + if err != nil { + loger.Error("error when trying to read number of files under `"+tempDir+"`", zap.Error(err)) + c.JSON(http.StatusInternalServerError, model.Result{Success: common_err.SERVICE_ERROR, Message: err.Error()}) + return + } + + if totalChunks == len(fileNum) { + if err := file.SpliceFiles(tempDir, path, totalChunks, 1); err != nil { + loger.Error("error when trying to splice files under `"+tempDir+"`", zap.Error(err)) + c.JSON(http.StatusInternalServerError, model.Result{Success: common_err.SERVICE_ERROR, Message: err.Error()}) + return + } + + if err := file.RMDir(tempDir); err != nil { + loger.Error("error when trying to remove `"+tempDir+"`", zap.Error(err)) + c.JSON(http.StatusInternalServerError, model.Result{Success: common_err.SERVICE_ERROR, Message: err.Error()}) + return + } + } } else { - out, _ := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644) - defer out.Close() - _, err := io.Copy(out, f) + out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0o644) if err != nil { - c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + loger.Error("error when trying to open `"+path+"` for creation", zap.Error(err)) + c.JSON(http.StatusInternalServerError, model.Result{Success: common_err.SERVICE_ERROR, Message: err.Error()}) return } - c.JSON(http.StatusOK, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS)}) - return - } - fileNum, err := ioutil.ReadDir(tempDir) - if err != nil { - c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) - return - } - if totalChunks == len(fileNum) { - file.SpliceFiles(tempDir, path, totalChunks, 1) - file.RMDir(tempDir) - } - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS)}) + defer out.Close() + + if _, err := io.Copy(out, f); err != nil { // recommend to use https://github.com/iceber/iouring-go for faster copy + loger.Error("error when trying to write to `"+path+"`", zap.Error(err)) + c.JSON(http.StatusInternalServerError, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + } + c.JSON(http.StatusOK, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS)}) } // @Summary copy or move file @@ -465,7 +500,6 @@ func PostFileUpload(c *gin.Context) { // @Success 200 {string} string "ok" // @Router /file/operate [post] func PostOperateFileOrDir(c *gin.Context) { - list := model.FileOperate{} c.ShouldBind(&list) @@ -515,7 +549,6 @@ func PostOperateFileOrDir(c *gin.Context) { // @Success 200 {string} string "ok" // @Router /file/delete [delete] func DeleteFile(c *gin.Context) { - paths := []string{} c.ShouldBind(&paths) if len(paths) == 0 { @@ -547,7 +580,6 @@ func DeleteFile(c *gin.Context) { // @Success 200 {string} string "ok" // @Router /file/update [put] func PutFileContent(c *gin.Context) { - fi := model.FileUpdate{} c.ShouldBind(&fi) @@ -557,7 +589,7 @@ func PutFileContent(c *gin.Context) { c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.FILE_ALREADY_EXISTS, Message: common_err.GetMsg(common_err.FILE_ALREADY_EXISTS)}) return } - //err := os.Remove(path) + // err := os.Remove(path) err := os.RemoveAll(fi.FilePath) if err != nil { c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.FILE_DELETE_ERROR, Message: common_err.GetMsg(common_err.FILE_DELETE_ERROR), Data: err})