diff --git a/internal/op/fs.go b/internal/op/fs.go deleted file mode 100644 index 576a4b664..000000000 --- a/internal/op/fs.go +++ /dev/null @@ -1,545 +0,0 @@ -package op - -import ( - "context" - "os" - stdpath "path" - "time" - - "github.com/IceWhaleTech/CasaOS/internal/driver" - "github.com/IceWhaleTech/CasaOS/model" - "github.com/IceWhaleTech/CasaOS/pkg/generic_sync" - "github.com/IceWhaleTech/CasaOS/pkg/singleflight" - "github.com/IceWhaleTech/CasaOS/pkg/utils" - "github.com/Xhofe/go-cache" - "github.com/pkg/errors" - pkgerr "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -// In order to facilitate adding some other things before and after file op - -var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) -var listG singleflight.Group[[]model.Obj] - -func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) { - key := Key(storage, path) - objs, ok := listCache.Get(key) - if ok { - for i, obj := range objs { - if obj.GetName() == oldObj.GetName() { - objs[i] = newObj - break - } - } - listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) - } -} - -func delCacheObj(storage driver.Driver, path string, obj model.Obj) { - key := Key(storage, path) - objs, ok := listCache.Get(key) - if ok { - for i, oldObj := range objs { - if oldObj.GetName() == obj.GetName() { - objs = append(objs[:i], objs[i+1:]...) - break - } - } - listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) - } -} - -var addSortDebounceMap generic_sync.MapOf[string, func(func())] - -func addCacheObj(storage driver.Driver, path string, newObj model.Obj) { - key := Key(storage, path) - objs, ok := listCache.Get(key) - if ok { - for i, obj := range objs { - if obj.GetName() == newObj.GetName() { - objs[i] = newObj - return - } - } - - // Simple separation of files and folders - if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() { - objs = append(objs, newObj) - } else { - objs = append([]model.Obj{newObj}, objs...) - } - - if storage.Config().LocalSort { - debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute)) - log.Debug("addCacheObj: wait start sort") - debounce(func() { - log.Debug("addCacheObj: start sort") - model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) - addSortDebounceMap.Delete(key) - }) - } - - listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) - } -} - -func ClearCache(storage driver.Driver, path string) { - listCache.Del(Key(storage, path)) -} - -func Key(storage driver.Driver, path string) string { - return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path)) -} - -// List files in storage, not contains virtual file -func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) { - if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) - } - path = utils.FixAndCleanPath(path) - log.Debugf("op.List %s", path) - key := Key(storage, path) - if !utils.IsBool(refresh...) { - if files, ok := listCache.Get(key); ok { - log.Debugf("use cache when list %s", path) - return files, nil - } - } - dir, err := GetUnwrap(ctx, storage, path) - if err != nil { - return nil, errors.WithMessage(err, "failed get dir") - } - log.Debugf("list dir: %+v", dir) - if !dir.IsDir() { - return nil, errors.WithStack(errors.New("not a folder")) - } - objs, err, _ := listG.Do(key, func() ([]model.Obj, error) { - files, err := storage.List(ctx, dir, args) - if err != nil { - return nil, errors.Wrapf(err, "failed to list objs") - } - // set path - for _, f := range files { - if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" { - s.SetPath(stdpath.Join(dir.GetPath(), f.GetName())) - } - } - // warp obj name - model.WrapObjsName(files) - // call hooks - go func(reqPath string, files []model.Obj) { - for _, hook := range ObjsUpdateHooks { - hook(args.ReqPath, files) - } - }(args.ReqPath, files) - - // sort objs - if storage.Config().LocalSort { - model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) - } - model.ExtractFolder(files, storage.GetStorage().ExtractFolder) - - if !storage.Config().NoCache { - if len(files) > 0 { - log.Debugf("set cache: %s => %+v", key, files) - listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) - } else { - log.Debugf("del cache: %s", key) - listCache.Del(key) - } - } - return files, nil - }) - return objs, err -} - -// Get object from list of files -func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { - path = utils.FixAndCleanPath(path) - log.Debugf("op.Get %s", path) - - // is root folder - if utils.PathEqual(path, "/") { - var rootObj model.Obj - switch r := storage.GetAddition().(type) { - case driver.IRootId: - rootObj = &model.Object{ - ID: r.GetRootId(), - Name: RootName, - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, - Path: path, - } - case driver.IRootPath: - rootObj = &model.Object{ - Path: r.GetRootPath(), - Name: RootName, - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, - } - default: - if storage, ok := storage.(driver.Getter); ok { - obj, err := storage.GetRoot(ctx) - if err != nil { - return nil, errors.WithMessage(err, "failed get root obj") - } - rootObj = obj - } - } - if rootObj == nil { - return nil, errors.Errorf("please implement IRootPath or IRootId or Getter method") - } - return &model.ObjWrapName{ - Name: RootName, - Obj: rootObj, - }, nil - } - - // not root folder - dir, name := stdpath.Split(path) - files, err := List(ctx, storage, dir, model.ListArgs{}) - if err != nil { - return nil, errors.WithMessage(err, "failed get parent list") - } - for _, f := range files { - // TODO maybe copy obj here - if f.GetName() == name { - return f, nil - } - } - log.Debugf("cant find obj with name: %s", name) - return nil, errors.WithStack(errors.New("object not found")) -} - -func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { - obj, err := Get(ctx, storage, path) - if err != nil { - return nil, err - } - return model.UnwrapObjs(obj), err -} - -var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16)) -var linkG singleflight.Group[*model.Link] - -// Link get link, if is an url. should have an expiry time -func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { - if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) - } - file, err := GetUnwrap(ctx, storage, path) - if err != nil { - return nil, nil, errors.WithMessage(err, "failed to get file") - } - if file.IsDir() { - return nil, nil, errors.WithStack(errors.New("not a file")) - } - key := Key(storage, path) + ":" + args.IP - if link, ok := linkCache.Get(key); ok { - return link, file, nil - } - fn := func() (*model.Link, error) { - link, err := storage.Link(ctx, file, args) - if err != nil { - return nil, errors.Wrapf(err, "failed get link") - } - if link.Expiration != nil { - linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration)) - } - return link, nil - } - link, err, _ := linkG.Do(key, fn) - return link, file, err -} - -// Other api -func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) { - obj, err := GetUnwrap(ctx, storage, args.Path) - if err != nil { - return nil, errors.WithMessagef(err, "failed to get obj") - } - if o, ok := storage.(driver.Other); ok { - return o.Other(ctx, model.OtherArgs{ - Obj: obj, - Method: args.Method, - Data: args.Data, - }) - } else { - return nil, errors.New("not implement") - } -} - -var mkdirG singleflight.Group[interface{}] - -func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) - } - path = utils.FixAndCleanPath(path) - key := Key(storage, path) - _, err, _ := mkdirG.Do(key, func() (interface{}, error) { - // check if dir exists - f, err := GetUnwrap(ctx, storage, path) - if err != nil { - if errors.Is(pkgerr.Cause(err), errors.New("object not found")) { - parentPath, dirName := stdpath.Split(path) - err = MakeDir(ctx, storage, parentPath) - if err != nil { - return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) - } - parentDir, err := GetUnwrap(ctx, storage, parentPath) - // this should not happen - if err != nil { - return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) - } - - switch s := storage.(type) { - case driver.MkdirResult: - var newObj model.Obj - newObj, err = s.MakeDir(ctx, parentDir, dirName) - if err == nil { - if newObj != nil { - addCacheObj(storage, parentPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - ClearCache(storage, parentPath) - } - } - case driver.Mkdir: - err = s.MakeDir(ctx, parentDir, dirName) - if err == nil && !utils.IsBool(lazyCache...) { - ClearCache(storage, parentPath) - } - default: - return nil, errors.New("not implement") - } - return nil, errors.WithStack(err) - } - return nil, errors.WithMessage(err, "failed to check if dir exists") - } - // dir exists - if f.IsDir() { - return nil, nil - } - // dir to make is a file - return nil, errors.New("file exists") - }) - return err -} - -func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) - } - srcPath = utils.FixAndCleanPath(srcPath) - dstDirPath = utils.FixAndCleanPath(dstDirPath) - srcRawObj, err := Get(ctx, storage, srcPath) - if err != nil { - return errors.WithMessage(err, "failed to get src object") - } - srcObj := model.UnwrapObjs(srcRawObj) - dstDir, err := GetUnwrap(ctx, storage, dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed to get dst dir") - } - srcDirPath := stdpath.Dir(srcPath) - - switch s := storage.(type) { - case driver.MoveResult: - var newObj model.Obj - newObj, err = s.Move(ctx, srcObj, dstDir) - if err == nil { - delCacheObj(storage, srcDirPath, srcRawObj) - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - ClearCache(storage, dstDirPath) - } - } - case driver.Move: - err = s.Move(ctx, srcObj, dstDir) - if err == nil { - delCacheObj(storage, srcDirPath, srcRawObj) - if !utils.IsBool(lazyCache...) { - ClearCache(storage, dstDirPath) - } - } - default: - return errors.New("not implement") - } - return errors.WithStack(err) -} - -func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string, lazyCache ...bool) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) - } - srcPath = utils.FixAndCleanPath(srcPath) - srcRawObj, err := Get(ctx, storage, srcPath) - if err != nil { - return errors.WithMessage(err, "failed to get src object") - } - srcObj := model.UnwrapObjs(srcRawObj) - srcDirPath := stdpath.Dir(srcPath) - - switch s := storage.(type) { - case driver.RenameResult: - var newObj model.Obj - newObj, err = s.Rename(ctx, srcObj, dstName) - if err == nil { - if newObj != nil { - updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - ClearCache(storage, srcDirPath) - } - } - case driver.Rename: - err = s.Rename(ctx, srcObj, dstName) - if err == nil && !utils.IsBool(lazyCache...) { - ClearCache(storage, srcDirPath) - } - default: - return errors.New("not implement") - } - return errors.WithStack(err) -} - -// Copy Just copy file[s] in a storage -func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) - } - srcPath = utils.FixAndCleanPath(srcPath) - dstDirPath = utils.FixAndCleanPath(dstDirPath) - srcObj, err := GetUnwrap(ctx, storage, srcPath) - if err != nil { - return errors.WithMessage(err, "failed to get src object") - } - dstDir, err := GetUnwrap(ctx, storage, dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed to get dst dir") - } - - switch s := storage.(type) { - case driver.CopyResult: - var newObj model.Obj - newObj, err = s.Copy(ctx, srcObj, dstDir) - if err == nil { - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - ClearCache(storage, dstDirPath) - } - } - case driver.Copy: - err = s.Copy(ctx, srcObj, dstDir) - if err == nil && !utils.IsBool(lazyCache...) { - ClearCache(storage, dstDirPath) - } - default: - return errors.New("not implement") - } - return errors.WithStack(err) -} - -func Remove(ctx context.Context, storage driver.Driver, path string) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) - } - path = utils.FixAndCleanPath(path) - rawObj, err := Get(ctx, storage, path) - if err != nil { - // if object not found, it's ok - if errors.Is(pkgerr.Cause(err), errors.New("object not found")) { - return nil - } - return errors.WithMessage(err, "failed to get object") - } - dirPath := stdpath.Dir(path) - - switch s := storage.(type) { - case driver.Remove: - err = s.Remove(ctx, model.UnwrapObjs(rawObj)) - if err == nil { - delCacheObj(storage, dirPath, rawObj) - } - default: - return errors.New("not implement") - } - return errors.WithStack(err) -} - -func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file *model.FileStream, up driver.UpdateProgress, lazyCache ...bool) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) - } - defer func() { - if f, ok := file.GetReadCloser().(*os.File); ok { - err := os.RemoveAll(f.Name()) - if err != nil { - log.Errorf("failed to remove file [%s]", f.Name()) - } - } - }() - defer func() { - if err := file.Close(); err != nil { - log.Errorf("failed to close file streamer, %v", err) - } - }() - // if file exist and size = 0, delete it - dstDirPath = utils.FixAndCleanPath(dstDirPath) - dstPath := stdpath.Join(dstDirPath, file.GetName()) - fi, err := GetUnwrap(ctx, storage, dstPath) - if err == nil { - if fi.GetSize() == 0 { - err = Remove(ctx, storage, dstPath) - if err != nil { - return errors.WithMessagef(err, "failed remove file that exist and have size 0") - } - } else { - file.Old = fi - } - } - err = MakeDir(ctx, storage, dstDirPath) - if err != nil { - return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) - } - parentDir, err := GetUnwrap(ctx, storage, dstDirPath) - // this should not happen - if err != nil { - return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) - } - // if up is nil, set a default to prevent panic - if up == nil { - up = func(p int) {} - } - - switch s := storage.(type) { - case driver.PutResult: - var newObj model.Obj - newObj, err = s.Put(ctx, parentDir, file, up) - if err == nil { - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - ClearCache(storage, dstDirPath) - } - } - case driver.Put: - err = s.Put(ctx, parentDir, file, up) - if err == nil && !utils.IsBool(lazyCache...) { - ClearCache(storage, dstDirPath) - } - default: - return errors.New("not implement") - } - log.Debugf("put file [%s] done", file.GetName()) - //if err == nil { - // //clear cache - // key := stdpath.Join(storage.GetStorage().MountPath, dstDirPath) - // listCache.Del(key) - //} - return errors.WithStack(err) -}