CasaOS/pkg/utils/file/file.go
2023-03-20 19:47:13 +08:00

768 lines
17 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package file
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"io/ioutil"
"log"
"mime/multipart"
"os"
"path"
path2 "path"
"path/filepath"
"strconv"
"strings"
"github.com/mholt/archiver/v3"
)
// GetSize get the file size
func GetSize(f multipart.File) (int, error) {
content, err := ioutil.ReadAll(f)
return len(content), err
}
// GetExt get the file ext
func GetExt(fileName string) string {
return path.Ext(fileName)
}
// CheckNotExist check if the file exists
func CheckNotExist(src string) bool {
_, err := os.Stat(src)
return os.IsNotExist(err)
}
// CheckPermission check if the file has permission
func CheckPermission(src string) bool {
_, err := os.Stat(src)
return os.IsPermission(err)
}
// IsNotExistMkDir create a directory if it does not exist
func IsNotExistMkDir(src string) error {
if notExist := CheckNotExist(src); notExist {
if err := MkDir(src); err != nil {
return err
}
}
return nil
}
// MkDir create a directory
func MkDir(src string) error {
err := os.MkdirAll(src, os.ModePerm)
if err != nil {
return err
}
os.Chmod(src, 0o777)
return nil
}
// RMDir remove a directory
func RMDir(src string) error {
err := os.RemoveAll(src)
if err != nil {
return err
}
os.Remove(src)
return nil
}
// Open a file according to a specific mode
func Open(name string, flag int, perm os.FileMode) (*os.File, error) {
f, err := os.OpenFile(name, flag, perm)
if err != nil {
return nil, err
}
return f, nil
}
// MustOpen maximize trying to open the file
func MustOpen(fileName, filePath string) (*os.File, error) {
//dir, err := os.Getwd()
//if err != nil {
// return nil, fmt.Errorf("os.Getwd err: %v", err)
//}
src := filePath
perm := CheckPermission(src)
if perm == true {
return nil, fmt.Errorf("file.CheckPermission Permission denied src: %s", src)
}
err := IsNotExistMkDir(src)
if err != nil {
return nil, fmt.Errorf("file.IsNotExistMkDir src: %s, err: %v", src, err)
}
f, err := Open(src+fileName, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0o644)
if err != nil {
return nil, fmt.Errorf("Fail to OpenFile :%v", err)
}
return f, nil
}
// 判断所给路径文件/文件夹是否存在
func Exists(path string) bool {
_, err := os.Stat(path) // os.Stat获取文件信息
if err != nil {
if os.IsExist(err) {
return true
}
return false
}
return true
}
// 判断所给路径是否为文件夹
func IsDir(path string) bool {
s, err := os.Stat(path)
if err != nil {
return false
}
return s.IsDir()
}
// 判断所给路径是否为文件
func IsFile(path string) bool {
return !IsDir(path)
}
func CreateFile(path string) error {
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
return nil
}
func CreateFileAndWriteContent(path string, content string) error {
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0o666)
if err != nil {
return err
}
defer file.Close()
write := bufio.NewWriter(file)
write.WriteString(content)
write.Flush()
return nil
}
// IsNotExistMkDir create a directory if it does not exist
func IsNotExistCreateFile(src string) error {
if notExist := CheckNotExist(src); notExist {
if err := CreateFile(src); err != nil {
return err
}
}
return nil
}
func ReadFullFile(path string) []byte {
file, err := os.Open(path)
if err != nil {
return []byte("")
}
defer file.Close()
content, err := ioutil.ReadAll(file)
if err != nil {
return []byte("")
}
return content
}
// File copies a single file from src to dst
func CopyFile(src, dst, style string) error {
var err error
var srcfd *os.File
var dstfd *os.File
var srcinfo os.FileInfo
lastPath := src[strings.LastIndex(src, "/")+1:]
if !strings.HasSuffix(dst, "/") {
dst += "/"
}
dst += lastPath
if Exists(dst) {
if style == "skip" {
return nil
} else {
os.Remove(dst)
}
}
if srcfd, err = os.Open(src); err != nil {
return err
}
defer srcfd.Close()
if dstfd, err = os.Create(dst); err != nil {
return err
}
defer dstfd.Close()
if _, err = io.Copy(dstfd, srcfd); err != nil {
return err
}
if srcinfo, err = os.Stat(src); err != nil {
return err
}
return os.Chmod(dst, srcinfo.Mode())
}
/**
* @description:
* @param {*} src
* @param {*} dst
* @param {string} style
* @return {*}
* @method:
* @router:
*/
func CopySingleFile(src, dst, style string) error {
var err error
var srcfd *os.File
var dstfd *os.File
var srcinfo os.FileInfo
if Exists(dst) {
if style == "skip" {
return nil
} else {
os.Remove(dst)
}
}
if srcfd, err = os.Open(src); err != nil {
return err
}
defer srcfd.Close()
if dstfd, err = os.Create(dst); err != nil {
return err
}
defer dstfd.Close()
if _, err = io.Copy(dstfd, srcfd); err != nil {
return err
}
if srcinfo, err = os.Stat(src); err != nil {
return err
}
return os.Chmod(dst, srcinfo.Mode())
}
// Check for duplicate file names
func GetNoDuplicateFileName(fullPath string) string {
path, fileName := filepath.Split(fullPath)
fileSuffix := path2.Ext(fileName)
filenameOnly := strings.TrimSuffix(fileName, fileSuffix)
for i := 0; Exists(fullPath); i++ {
fullPath = path2.Join(path, filenameOnly+"("+strconv.Itoa(i+1)+")"+fileSuffix)
}
return fullPath
}
// Dir copies a whole directory recursively
func CopyDir(src string, dst string, style string) error {
var err error
var fds []os.FileInfo
var srcinfo os.FileInfo
if srcinfo, err = os.Stat(src); err != nil {
return err
}
if !srcinfo.IsDir() {
if err = CopyFile(src, dst, style); err != nil {
fmt.Println(err)
}
return nil
}
// dstPath := dst
lastPath := src[strings.LastIndex(src, "/")+1:]
dst += "/" + lastPath
// for i := 0; Exists(dst); i++ {
// dst = dstPath + "/" + lastPath + strconv.Itoa(i+1)
// }
if Exists(dst) {
if style == "skip" {
return nil
} else {
os.Remove(dst)
}
}
if err = os.MkdirAll(dst, srcinfo.Mode()); err != nil {
return err
}
if fds, err = ioutil.ReadDir(src); err != nil {
return err
}
for _, fd := range fds {
srcfp := path.Join(src, fd.Name())
dstfp := dst // path.Join(dst, fd.Name())
if fd.IsDir() {
if err = CopyDir(srcfp, dstfp, style); err != nil {
fmt.Println(err)
}
} else {
if err = CopyFile(srcfp, dstfp, style); err != nil {
fmt.Println(err)
}
}
}
return nil
}
func WriteToPath(data []byte, path, name string) error {
fullPath := path
if strings.HasSuffix(path, "/") {
fullPath += name
} else {
fullPath += "/" + name
}
return WriteToFullPath(data, fullPath, 0o666)
}
func WriteToFullPath(data []byte, fullPath string, perm fs.FileMode) error {
if err := IsNotExistCreateFile(fullPath); err != nil {
return err
}
file, err := os.OpenFile(fullPath,
os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
perm,
)
if err != nil {
return err
}
defer file.Close()
_, err = file.Write(data)
return err
}
// 最终拼接
func SpliceFiles(dir, path string, length int, startPoint int) error {
fullPath := path
if err := IsNotExistCreateFile(fullPath); err != nil {
return err
}
file, _ := os.OpenFile(fullPath,
os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
0o666,
)
defer file.Close()
bufferedWriter := bufio.NewWriter(file)
// todo: here should have a goroutine to remove each partial file after it is read, to save disk space
for i := 0; i < length+startPoint-1; i++ {
data, err := ioutil.ReadFile(dir + "/" + strconv.Itoa(i+startPoint))
if err != nil {
return err
}
if _, err := bufferedWriter.Write(data); err != nil { // recommend to use https://github.com/iceber/iouring-go for faster write
return err
}
}
bufferedWriter.Flush()
return nil
}
func GetCompressionAlgorithm(t string) (string, archiver.Writer, error) {
switch t {
case "zip", "":
return ".zip", archiver.NewZip(), nil
case "tar":
return ".tar", archiver.NewTar(), nil
case "targz":
return ".tar.gz", archiver.NewTarGz(), nil
case "tarbz2":
return ".tar.bz2", archiver.NewTarBz2(), nil
case "tarxz":
return ".tar.xz", archiver.NewTarXz(), nil
case "tarlz4":
return ".tar.lz4", archiver.NewTarLz4(), nil
case "tarsz":
return ".tar.sz", archiver.NewTarSz(), nil
default:
return "", nil, errors.New("format not implemented")
}
}
func AddFile(ar archiver.Writer, path, commonPath string) error {
info, err := os.Stat(path)
if err != nil {
return err
}
if !info.IsDir() && !info.Mode().IsRegular() {
return nil
}
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
if path != commonPath {
filename := info.Name()
err = ar.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: info,
CustomName: filename,
},
ReadCloser: file,
})
if err != nil {
return err
}
}
if info.IsDir() {
names, err := file.Readdirnames(0)
if err != nil {
return err
}
for _, name := range names {
err = AddFile(ar, filepath.Join(path, name), commonPath)
if err != nil {
log.Printf("Failed to archive %v", err)
}
}
}
return nil
}
func CommonPrefix(sep byte, paths ...string) string {
// Handle special cases.
switch len(paths) {
case 0:
return ""
case 1:
return path.Clean(paths[0])
}
// Note, we treat string as []byte, not []rune as is often
// done in Go. (And sep as byte, not rune). This is because
// most/all supported OS' treat paths as string of non-zero
// bytes. A filename may be displayed as a sequence of Unicode
// runes (typically encoded as UTF-8) but paths are
// not required to be valid UTF-8 or in any normalized form
// (e.g. "é" (U+00C9) and "é" (U+0065,U+0301) are different
// file names.
c := []byte(path.Clean(paths[0]))
// We add a trailing sep to handle the case where the
// common prefix directory is included in the path list
// (e.g. /home/user1, /home/user1/foo, /home/user1/bar).
// path.Clean will have cleaned off trailing / separators with
// the exception of the root directory, "/" (in which case we
// make it "//", but this will get fixed up to "/" bellow).
c = append(c, sep)
// Ignore the first path since it's already in c
for _, v := range paths[1:] {
// Clean up each path before testing it
v = path.Clean(v) + string(sep)
// Find the first non-common byte and truncate c
if len(v) < len(c) {
c = c[:len(v)]
}
for i := 0; i < len(c); i++ {
if v[i] != c[i] {
c = c[:i]
break
}
}
}
// Remove trailing non-separator characters and the final separator
for i := len(c) - 1; i >= 0; i-- {
if c[i] == sep {
c = c[:i]
break
}
}
return string(c)
}
func GetFileOrDirSize(path string) (int64, error) {
fileInfo, err := os.Stat(path)
if err != nil {
return 0, err
}
if fileInfo.IsDir() {
return DirSizeB(path + "/")
}
return fileInfo.Size(), nil
}
// getFileSize get file size by path(B)
func DirSizeB(path string) (int64, error) {
var size int64
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
if !info.IsDir() {
size += info.Size()
}
return err
})
return size, err
}
func MoveFile(sourcePath, destPath string) error {
inputFile, err := os.Open(sourcePath)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
outputFile, err := os.Create(destPath)
if err != nil {
inputFile.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer outputFile.Close()
_, err = io.Copy(outputFile, inputFile)
inputFile.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
err = os.Remove(sourcePath)
if err != nil {
return fmt.Errorf("Failed removing original file: %s", err)
}
return nil
}
func ReadLine(lineNumber int, path string) string {
file, err := os.Open(path)
if err != nil {
return ""
}
fileScanner := bufio.NewScanner(file)
lineCount := 1
for fileScanner.Scan() {
if lineCount == lineNumber {
return fileScanner.Text()
}
lineCount++
}
defer file.Close()
return ""
}
func NameAccumulation(name string, dir string) string {
path := filepath.Join(dir, name)
if _, err := os.Stat(path); os.IsNotExist(err) {
return name
}
base := name
strings.Split(base, "_")
index := strings.LastIndex(base, "_")
if index < 0 {
index = len(base)
}
for i := 1; ; i++ {
newPath := filepath.Join(dir, fmt.Sprintf("%s_%d", base[:index], i))
if _, err := os.Stat(newPath); os.IsNotExist(err) {
return fmt.Sprintf("%s_%d", base[:index], i)
}
}
}
// / 解析多个文件上传中,每个具体的文件的信息
// type FileHeader struct {
// ContentDisposition string
// Name string
// FileName string
// ///< 文件名
// ContentType string
// ContentLength int64
// }
// / 解析描述文件信息的头部
// / @return FileHeader 文件名等信息的结构体
// / @return bool 解析成功还是失败
func ParseFileHeader(h []byte, boundary []byte) (map[string]string, bool) {
arr := bytes.Split(h, boundary)
//var out_header FileHeader
//out_header.ContentLength = -1
const (
CONTENT_DISPOSITION = "Content-Disposition: "
NAME = "name=\""
FILENAME = "filename=\""
CONTENT_TYPE = "Content-Type: "
CONTENT_LENGTH = "Content-Length: "
)
result := make(map[string]string)
for _, item := range arr {
tarr := bytes.Split(item, []byte(";"))
if len(tarr) != 2 {
continue
}
tbyte := tarr[1]
fmt.Println(string(tbyte))
tbyte = bytes.ReplaceAll(tbyte, []byte("\r\n--"), []byte(""))
tbyte = bytes.ReplaceAll(tbyte, []byte("name=\""), []byte(""))
tempArr := bytes.Split(tbyte, []byte("\"\r\n\r\n"))
if len(tempArr) != 2 {
continue
}
bytes.HasPrefix(item, []byte("name="))
result[strings.TrimSpace(string(tempArr[0]))] = strings.TrimSpace(string(tempArr[1]))
}
// for _, item := range arr {
// if bytes.HasPrefix(item, []byte(CONTENT_DISPOSITION)) {
// l := len(CONTENT_DISPOSITION)
// arr1 := bytes.Split(item[l:], []byte("; "))
// out_header.ContentDisposition = string(arr1[0])
// if bytes.HasPrefix(arr1[1], []byte(NAME)) {
// out_header.Name = string(arr1[1][len(NAME) : len(arr1[1])-1])
// }
// l = len(arr1[2])
// if bytes.HasPrefix(arr1[2], []byte(FILENAME)) && arr1[2][l-1] == 0x22 {
// out_header.FileName = string(arr1[2][len(FILENAME) : l-1])
// }
// } else if bytes.HasPrefix(item, []byte(CONTENT_TYPE)) {
// l := len(CONTENT_TYPE)
// out_header.ContentType = string(item[l:])
// } else if bytes.HasPrefix(item, []byte(CONTENT_LENGTH)) {
// l := len(CONTENT_LENGTH)
// s := string(item[l:])
// content_length, err := strconv.ParseInt(s, 10, 64)
// if err != nil {
// log.Printf("content length error:%s", string(item))
// return out_header, false
// } else {
// out_header.ContentLength = content_length
// }
// } else {
// log.Printf("unknown:%s\n", string(item))
// }
// }
//fmt.Println(result)
// if len(out_header.FileName) == 0 {
// return out_header, false
// }
return result, true
}
// / 从流中一直读到文件的末位
// / @return []byte 没有写到文件且又属于下一个文件的数据
// / @return bool 是否已经读到流的末位了
// / @return error 是否发生错误
func ReadToBoundary(boundary []byte, stream io.ReadCloser, target io.WriteCloser) ([]byte, bool, error) {
read_data := make([]byte, 1024*8)
read_data_len := 0
buf := make([]byte, 1024*4)
b_len := len(boundary)
reach_end := false
for !reach_end {
read_len, err := stream.Read(buf)
if err != nil {
if err != io.EOF && read_len <= 0 {
return nil, true, err
}
reach_end = true
}
//todo: 下面这一句很蠢,值得优化
copy(read_data[read_data_len:], buf[:read_len]) //追加到另一块buffer仅仅只是为了搜索方便
read_data_len += read_len
if read_data_len < b_len+4 {
continue
}
loc := bytes.Index(read_data[:read_data_len], boundary)
if loc >= 0 {
//找到了结束位置
target.Write(read_data[:loc-4])
return read_data[loc:read_data_len], reach_end, nil
}
target.Write(read_data[:read_data_len-b_len-4])
copy(read_data[0:], read_data[read_data_len-b_len-4:])
read_data_len = b_len + 4
}
target.Write(read_data[:read_data_len])
return nil, reach_end, nil
}
// / 解析表单的头部
// / @param read_data 已经从流中读到的数据
// / @param read_total 已经从流中读到的数据长度
// / @param boundary 表单的分割字符串
// / @param stream 输入流
// / @return FileHeader 文件名等信息头
// /[]byte 已经从流中读到的部分
// /error 是否发生错误
func ParseFromHead(read_data []byte, read_total int, boundary []byte, stream io.ReadCloser) (map[string]string, []byte, error) {
buf := make([]byte, 1024*8)
found_boundary := false
boundary_loc := -1
for {
read_len, err := stream.Read(buf)
if err != nil {
if err != io.EOF {
return nil, nil, err
}
break
}
if read_total+read_len > cap(read_data) {
return nil, nil, fmt.Errorf("not found boundary")
}
copy(read_data[read_total:], buf[:read_len])
read_total += read_len
if !found_boundary {
boundary_loc = bytes.LastIndex(read_data[:read_total], boundary)
if boundary_loc == -1 {
continue
}
found_boundary = true
}
start_loc := boundary_loc + len(boundary)
fmt.Println(string(read_data))
file_head_loc := bytes.Index(read_data[start_loc:read_total], []byte("\r\n\r\n"))
if file_head_loc == -1 {
continue
}
file_head_loc += start_loc
ret := false
headMap, ret := ParseFileHeader(read_data, boundary)
if !ret {
return headMap, nil, fmt.Errorf("ParseFileHeader fail:%s", string(read_data[start_loc:file_head_loc]))
}
return headMap, read_data[file_head_loc+4 : read_total], nil
}
return nil, nil, fmt.Errorf("reach to sream EOF")
}