From 7db3975b1837ce25ba504655900a0bc0930214ce Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 4 Oct 2023 16:27:08 +0800 Subject: [PATCH 01/10] wip: refactor offline download (#5331) * base tool * working: aria2 --- internal/bootstrap/data/setting.go | 4 - internal/model/setting.go | 2 +- internal/offline_download/add.go | 76 +++++++++++ internal/offline_download/all_test.go | 13 ++ internal/offline_download/aria2/aria2.go | 73 ++++++++++ internal/offline_download/aria2/notify.go | 70 ++++++++++ internal/offline_download/base.go | 56 ++++++++ internal/offline_download/monitor.go | 157 ++++++++++++++++++++++ internal/offline_download/tools.go | 33 +++++ internal/offline_download/util.go | 27 ++++ server/handles/aria2.go | 4 +- 11 files changed, 508 insertions(+), 7 deletions(-) create mode 100644 internal/offline_download/add.go create mode 100644 internal/offline_download/all_test.go create mode 100644 internal/offline_download/aria2/aria2.go create mode 100644 internal/offline_download/aria2/notify.go create mode 100644 internal/offline_download/base.go create mode 100644 internal/offline_download/monitor.go create mode 100644 internal/offline_download/tools.go create mode 100644 internal/offline_download/util.go diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index ca17b6b148f..70e237c0666 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -141,10 +141,6 @@ func InitialSettings() []model.SettingItem { {Key: conf.ForwardDirectLinkParams, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL}, {Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, - // aria2 settings - {Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, - {Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, - // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,none", Group: model.INDEX}, diff --git a/internal/model/setting.go b/internal/model/setting.go index f4202ee022c..3b2c30f1361 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -6,7 +6,7 @@ const ( STYLE PREVIEW GLOBAL - ARIA2 + OFFLINE_DOWNLOAD INDEX SSO ) diff --git a/internal/offline_download/add.go b/internal/offline_download/add.go new file mode 100644 index 00000000000..67147f35c79 --- /dev/null +++ b/internal/offline_download/add.go @@ -0,0 +1,76 @@ +package offline_download + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/task" + "github.com/google/uuid" + "github.com/pkg/errors" +) + +type AddURIArgs struct { + URI string + DstDirPath string + Tool string +} + +func AddURI(ctx context.Context, args *AddURIArgs) error { + // get tool + tool, err := Tools.Get(args.Tool) + if err != nil { + return errors.Wrapf(err, "failed get tool") + } + // check storage + storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + // check is it could upload + if storage.Config().NoUpload { + return errors.WithStack(errs.UploadNotSupported) + } + // check path is valid + obj, err := op.Get(ctx, storage, dstDirActualPath) + if err != nil { + if !errs.IsObjectNotFound(err) { + return errors.WithMessage(err, "failed get object") + } + } else { + if !obj.IsDir() { + // can't add to a file + return errors.WithStack(errs.NotFolder) + } + } + + uid := uuid.NewString() + tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) + signal := make(chan int) + gid, err := tool.AddURI(&AddUriArgs{ + Uri: args.URI, + UID: uid, + TempDir: tempDir, + Signal: signal, + }) + if err != nil { + return errors.Wrapf(err, "failed to add uri %s", args.URI) + } + DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{ + ID: gid, + Name: fmt.Sprintf("download %s to [%s](%s)", args.URI, storage.GetStorage().MountPath, dstDirActualPath), + Func: func(tsk *task.Task[string]) error { + m := &Monitor{ + tsk: tsk, + tempDir: tempDir, + dstDirPath: args.DstDirPath, + signal: signal, + } + return m.Loop() + }, + })) + return nil +} diff --git a/internal/offline_download/all_test.go b/internal/offline_download/all_test.go new file mode 100644 index 00000000000..e8b32e09b3d --- /dev/null +++ b/internal/offline_download/all_test.go @@ -0,0 +1,13 @@ +package offline_download + +import "testing" + +func TestGetFiles(t *testing.T) { + files, err := GetFiles("..") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + t.Log(file.Name, file.Size, file.Path) + } +} diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go new file mode 100644 index 00000000000..947b890c9a9 --- /dev/null +++ b/internal/offline_download/aria2/aria2.go @@ -0,0 +1,73 @@ +package aria2 + +import ( + "context" + "fmt" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/aria2/rpc" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +var notify = NewNotify() + +type Aria2 struct { + client rpc.Client +} + +func (a *Aria2) Items() []model.SettingItem { + // aria2 settings + return []model.SettingItem{ + {Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } +} + +func (a *Aria2) Init() (string, error) { + a.client = nil + uri := setting.GetStr(conf.Aria2Uri) + secret := setting.GetStr(conf.Aria2Secret) + c, err := rpc.New(context.Background(), uri, secret, 4*time.Second, notify) + if err != nil { + return "", errors.Wrap(err, "failed to init aria2 client") + } + version, err := c.GetVersion() + if err != nil { + return "", errors.Wrapf(err, "failed get aria2 version") + } + a.client = c + log.Infof("using aria2 version: %s", version.Version) + return fmt.Sprintf("aria2 version: %s", version.Version), nil +} + +func (a *Aria2) IsReady() bool { + //TODO implement me + panic("implement me") +} + +func (a *Aria2) AddURI(args *offline_download.AddUriArgs) (string, error) { + //TODO implement me + panic("implement me") +} + +func (a *Aria2) Remove(tid string) error { + //TODO implement me + panic("implement me") +} + +func (a *Aria2) Status(tid string) (*offline_download.Status, error) { + //TODO implement me + panic("implement me") +} + +func (a *Aria2) GetFile(tid string) *offline_download.File { + //TODO implement me + panic("implement me") +} + +var _ offline_download.Tool = (*Aria2)(nil) diff --git a/internal/offline_download/aria2/notify.go b/internal/offline_download/aria2/notify.go new file mode 100644 index 00000000000..056fe5147b4 --- /dev/null +++ b/internal/offline_download/aria2/notify.go @@ -0,0 +1,70 @@ +package aria2 + +import ( + "github.com/alist-org/alist/v3/pkg/aria2/rpc" + "github.com/alist-org/alist/v3/pkg/generic_sync" +) + +const ( + Downloading = iota + Paused + Stopped + Completed + Errored +) + +type Notify struct { + Signals generic_sync.MapOf[string, chan int] +} + +func NewNotify() *Notify { + return &Notify{Signals: generic_sync.MapOf[string, chan int]{}} +} + +func (n *Notify) OnDownloadStart(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Downloading + } + } +} + +func (n *Notify) OnDownloadPause(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Paused + } + } +} + +func (n *Notify) OnDownloadStop(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Stopped + } + } +} + +func (n *Notify) OnDownloadComplete(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Completed + } + } +} + +func (n *Notify) OnDownloadError(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Errored + } + } +} + +func (n *Notify) OnBtDownloadComplete(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Completed + } + } +} diff --git a/internal/offline_download/base.go b/internal/offline_download/base.go new file mode 100644 index 00000000000..2276a6a586f --- /dev/null +++ b/internal/offline_download/base.go @@ -0,0 +1,56 @@ +package offline_download + +import ( + "io" + "os" + + "github.com/alist-org/alist/v3/internal/model" +) + +type AddUriArgs struct { + Uri string + UID string + TempDir string + Signal chan int +} + +type Status struct { + Progress int + NewTID string + Completed bool + Status string + Err error +} + +type Tool interface { + // Items return the setting items the tool need + Items() []model.SettingItem + Init() (string, error) + IsReady() bool + // AddURI add an uri to download, return the task id + AddURI(args *AddUriArgs) (string, error) + // Remove the task if an error occurred + Remove(tid string) error + // Status return the status of the download task, if an error occurred, return the error in Status.Err + Status(tid string) (*Status, error) + // GetFile return an io.ReadCloser as the download file, if nil, means walk the temp dir to get the files + GetFile(tid string) *File +} + +type File struct { + io.ReadCloser + Name string + Size int64 + Path string +} + +func (f *File) GetReadCloser() (io.ReadCloser, error) { + if f.ReadCloser != nil { + return f.ReadCloser, nil + } + file, err := os.Open(f.Path) + if err != nil { + return nil, err + } + return file, nil +} diff --git a/internal/offline_download/monitor.go b/internal/offline_download/monitor.go new file mode 100644 index 00000000000..f446f268ece --- /dev/null +++ b/internal/offline_download/monitor.go @@ -0,0 +1,157 @@ +package offline_download + +import ( + "fmt" + "os" + "path" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/task" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type Monitor struct { + tool Tool + tsk *task.Task[string] + tempDir string + retried int + dstDirPath string + finish chan struct{} + signal chan int +} + +func (m *Monitor) Loop() error { + m.finish = make(chan struct{}) + var ( + err error + ok bool + ) +outer: + for { + select { + case <-m.tsk.Ctx.Done(): + err := m.tool.Remove(m.tsk.ID) + return err + case <-m.signal: + ok, err = m.Update() + if ok { + break outer + } + case <-time.After(time.Second * 2): + ok, err = m.Update() + if ok { + break outer + } + } + } + if err != nil { + return err + } + m.tsk.SetStatus("aria2 download completed, transferring") + <-m.finish + m.tsk.SetStatus("completed") + return nil +} + +// Update download status, return true if download completed +func (m *Monitor) Update() (bool, error) { + info, err := m.tool.Status(m.tsk.ID) + if err != nil { + m.retried++ + log.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried) + return false, nil + } + if m.retried > 5 { + return true, errors.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried) + } + m.retried = 0 + m.tsk.SetProgress(info.Progress) + m.tsk.SetStatus("tool: " + info.Status) + if info.NewTID != "" { + log.Debugf("followen by: %+v", info.NewTID) + DownTaskManager.RawTasks().Delete(m.tsk.ID) + m.tsk.ID = info.NewTID + DownTaskManager.RawTasks().Store(m.tsk.ID, m.tsk) + return false, nil + } + // if download completed + if info.Completed { + err := m.Complete() + return true, errors.WithMessage(err, "failed to transfer file") + } + // if download failed + if info.Err != nil { + return true, errors.Errorf("failed to download %s, error: %s", m.tsk.ID, info.Err.Error()) + } + return false, nil +} + +var TransferTaskManager = task.NewTaskManager(3, func(k *uint64) { + atomic.AddUint64(k, 1) +}) + +func (m *Monitor) Complete() error { + // check dstDir again + storage, dstDirActualPath, err := op.GetStorageAndActualPath(m.dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + var files []*File + if f := m.tool.GetFile(m.tsk.ID); f != nil { + files = append(files, f) + } else { + files, err = GetFiles(m.tempDir) + if err != nil { + return errors.Wrapf(err, "failed to get files") + } + } + // upload files + var wg sync.WaitGroup + wg.Add(len(files)) + go func() { + wg.Wait() + err := os.RemoveAll(m.tempDir) + m.finish <- struct{}{} + if err != nil { + log.Errorf("failed to remove aria2 temp dir: %+v", err.Error()) + } + }() + for i, _ := range files { + file := files[i] + TransferTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ + Name: fmt.Sprintf("transfer %s to [%s](%s)", file.Path, storage.GetStorage().MountPath, dstDirActualPath), + Func: func(tsk *task.Task[uint64]) error { + defer wg.Done() + mimetype := utils.GetMimeType(file.Path) + rc, err := file.GetReadCloser() + if err != nil { + return errors.Wrapf(err, "failed to open file %s", file.Path) + } + stream := &model.FileStream{ + Obj: &model.Object{ + Name: path.Base(file.Path), + Size: file.Size, + Modified: time.Now(), + IsFolder: false, + }, + ReadCloser: rc, + Mimetype: mimetype, + } + relDir, err := filepath.Rel(m.tempDir, filepath.Dir(file.Path)) + if err != nil { + log.Errorf("find relation directory error: %v", err) + } + newDistDir := filepath.Join(dstDirActualPath, relDir) + return op.Put(tsk.Ctx, storage, newDistDir, stream, tsk.SetProgress) + }, + })) + } + return nil +} diff --git a/internal/offline_download/tools.go b/internal/offline_download/tools.go new file mode 100644 index 00000000000..ca328f24e14 --- /dev/null +++ b/internal/offline_download/tools.go @@ -0,0 +1,33 @@ +package offline_download + +import ( + "fmt" + + "github.com/alist-org/alist/v3/pkg/task" +) + +var ( + Tools = make(ToolsManager) + DownTaskManager = task.NewTaskManager[string](3) +) + +type ToolsManager map[string]Tool + +func (t ToolsManager) Get(name string) (Tool, error) { + if tool, ok := t[name]; ok { + return tool, nil + } + return nil, fmt.Errorf("tool %s not found", name) +} + +func (t ToolsManager) Add(name string, tool Tool) { + t[name] = tool +} + +func (t ToolsManager) Names() []string { + names := make([]string, 0, len(t)) + for name := range t { + names = append(names, name) + } + return names +} diff --git a/internal/offline_download/util.go b/internal/offline_download/util.go new file mode 100644 index 00000000000..8f1c14843f6 --- /dev/null +++ b/internal/offline_download/util.go @@ -0,0 +1,27 @@ +package offline_download + +import ( + "os" + "path/filepath" +) + +func GetFiles(dir string) ([]*File, error) { + var files []*File + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + files = append(files, &File{ + Name: info.Name(), + Size: info.Size(), + Path: path, + }) + } + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} diff --git a/server/handles/aria2.go b/server/handles/aria2.go index 325367a796a..a78eb65941c 100644 --- a/server/handles/aria2.go +++ b/server/handles/aria2.go @@ -21,8 +21,8 @@ func SetAria2(c *gin.Context) { return } items := []model.SettingItem{ - {Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, - {Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, + {Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) From ea9a3432ab9b5f128f5a3ca0a24139e479a94516 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 4 Oct 2023 20:59:11 +0800 Subject: [PATCH 02/10] refactor: change type of percentage to float64 --- drivers/123/upload.go | 2 +- drivers/189/util.go | 2 +- drivers/189pc/utils.go | 6 +++--- drivers/aliyundrive/driver.go | 5 +++-- drivers/baidu_netdisk/driver.go | 2 +- drivers/baidu_photo/driver.go | 2 +- drivers/dropbox/driver.go | 2 +- drivers/mega/driver.go | 7 ++++--- drivers/mopan/driver.go | 2 +- drivers/onedrive/util.go | 2 +- drivers/onedrive_app/util.go | 2 +- drivers/quark_uc/driver.go | 2 +- drivers/s3/driver.go | 5 +++-- drivers/teambition/util.go | 2 +- drivers/terabox/driver.go | 2 +- drivers/trainbit/driver.go | 3 +-- drivers/wopan/driver.go | 2 +- internal/aria2/monitor.go | 5 +++-- internal/driver/driver.go | 4 ++-- internal/offline_download/base.go | 10 ++++++---- internal/offline_download/monitor.go | 13 ++++++++----- internal/offline_download/util.go | 7 ++++--- internal/op/fs.go | 2 +- internal/qbittorrent/monitor.go | 5 +++-- pkg/task/task.go | 6 +++--- pkg/utils/io.go | 7 ++++--- server/handles/task.go | 12 ++++++------ 27 files changed, 66 insertions(+), 55 deletions(-) diff --git a/drivers/123/upload.go b/drivers/123/upload.go index ae28d6aa519..6f6221f1148 100644 --- a/drivers/123/upload.go +++ b/drivers/123/upload.go @@ -107,7 +107,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi if err != nil { return err } - up(j * 100 / chunkCount) + up(float64(j) * 100 / float64(chunkCount)) } } // complete s3 upload diff --git a/drivers/189/util.go b/drivers/189/util.go index 680ce252133..0b4c0633d7b 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -380,7 +380,7 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F if err != nil { return err } - up(int(i * 100 / count)) + up(float64(i) * 100 / float64(count)) } fileMd5 := hex.EncodeToString(md5Sum.Sum(nil)) sliceMd5 := fileMd5 diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 1868aeb25ff..5e403a830e4 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -513,7 +513,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo if err != nil { return err } - up(int(threadG.Success()) * 100 / count) + up(float64(threadG.Success()) * 100 / float64(count)) return nil }) } @@ -676,7 +676,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode return err } - up(int(threadG.Success()) * 100 / len(uploadUrls)) + up(float64(threadG.Success()) * 100 / float64(len(uploadUrls))) uploadProgress.UploadParts[i] = "" return nil }) @@ -812,7 +812,7 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model if _, err := tempFile.Seek(status.GetSize(), io.SeekStart); err != nil { return nil, err } - up(int(status.GetSize()/file.GetSize()) * 100) + up(float64(status.GetSize()) / float64(file.GetSize()) * 100) } return y.OldUploadCommit(ctx, status.FileCommitUrl, status.UploadFileId) diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index f11452629a4..83c3f522452 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -7,7 +7,6 @@ import ( "encoding/base64" "encoding/hex" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "math" "math/big" @@ -15,6 +14,8 @@ import ( "os" "time" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" @@ -304,7 +305,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil } res.Body.Close() if count > 0 { - up(i * 100 / count) + up(float64(i) * 100 / float64(count)) } } var resp2 base.Json diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 141e2fc0fd9..5a4716d00f6 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -274,7 +274,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F if err != nil { return err } - up(int(threadG.Success()) * 100 / len(precreateResp.BlockList)) + up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) precreateResp.BlockList[i] = -1 return nil }) diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index 9105260d94d..c29bc110095 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -329,7 +329,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if err != nil { return err } - up(int(threadG.Success()) * 100 / len(precreateResp.BlockList)) + up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) precreateResp.BlockList[i] = -1 return nil }) diff --git a/drivers/dropbox/driver.go b/drivers/dropbox/driver.go index 7559d645858..95148b94e96 100644 --- a/drivers/dropbox/driver.go +++ b/drivers/dropbox/driver.go @@ -203,7 +203,7 @@ func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt _ = res.Body.Close() if count > 0 { - up((i + 1) * 100 / count) + up(float64(i+1) * 100 / float64(count)) } offset += byteSize diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go index c1ae9f7f6c9..9fa1d0eeafa 100644 --- a/drivers/mega/driver.go +++ b/drivers/mega/driver.go @@ -4,11 +4,12 @@ import ( "context" "errors" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/rclone/rclone/lib/readers" "io" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/rclone/rclone/lib/readers" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -169,7 +170,7 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return err } - up(id * 100 / u.Chunks()) + up(float64(id) * 100 / float64(u.Chunks())) } _, err = u.Finish() diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index bd2de2b30af..78ec0423cc3 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -308,7 +308,7 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre if resp.StatusCode != http.StatusOK { return fmt.Errorf("upload err,code=%d", resp.StatusCode) } - up(100 * int(threadG.Success()) / len(parts)) + up(100 * float64(threadG.Success()) / float64(len(parts))) initUpdload.PartInfos[i] = "" return nil }) diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 0539e098682..a0c6fa8fcbf 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -203,7 +203,7 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil return errors.New(string(data)) } res.Body.Close() - up(int(finish * 100 / stream.GetSize())) + up(float64(finish) * 100 / float64(stream.GetSize())) } return nil } diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index a40424bd5da..6a061f1f3bc 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -194,7 +194,7 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model. return errors.New(string(data)) } res.Body.Close() - up(int(finish * 100 / stream.GetSize())) + up(float64(finish) * 100 / float64(stream.GetSize())) } return nil } diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index 7c254022a92..291189ce088 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -209,7 +209,7 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File } md5s = append(md5s, m) partNumber++ - up(int(100 * (total - left) / total)) + up(100 * float64(total-left) / float64(total)) } err = d.upCommit(pre, md5s) if err != nil { diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index dd643f5d76e..c8099ee43dd 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -4,13 +4,14 @@ import ( "bytes" "context" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "net/url" stdpath "path" "strings" "time" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/aws/aws-sdk-go/aws/session" @@ -104,7 +105,7 @@ func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) e }, Reader: io.NopCloser(bytes.NewReader([]byte{})), Mimetype: "application/octet-stream", - }, func(int) {}) + }, func(float64) {}) } func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { diff --git a/drivers/teambition/util.go b/drivers/teambition/util.go index 04f222de95f..c39ffb18286 100644 --- a/drivers/teambition/util.go +++ b/drivers/teambition/util.go @@ -189,7 +189,7 @@ func (d *Teambition) chunkUpload(ctx context.Context, file model.FileStreamer, t if err != nil { return nil, err } - up(i * 100 / newChunk.Chunks) + up(float64(i) * 100 / float64(newChunk.Chunks)) } _, err = base.RestyClient.R().SetHeader("Authorization", token).Post( fmt.Sprintf("https://%s.teambition.net/upload/chunk/%s", diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go index b5287f5a7f8..c9662fce03a 100644 --- a/drivers/terabox/driver.go +++ b/drivers/terabox/driver.go @@ -213,7 +213,7 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt } log.Debugln(res.String()) if len(precreateResp.BlockList) > 0 { - up(i * 100 / len(precreateResp.BlockList)) + up(float64(i) * 100 / float64(len(precreateResp.BlockList))) } } _, err = d.create(rawPath, stream.GetSize(), 0, precreateResp.Uploadid, block_list_str) diff --git a/drivers/trainbit/driver.go b/drivers/trainbit/driver.go index 63bd0627f63..795b2fb8a2e 100644 --- a/drivers/trainbit/driver.go +++ b/drivers/trainbit/driver.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "math" "net/http" "net/url" "strings" @@ -128,7 +127,7 @@ func (d *Trainbit) Put(ctx context.Context, dstDir model.Obj, stream model.FileS stream, func(byteNum int) { total += int64(byteNum) - up(int(math.Round(float64(total) / float64(stream.GetSize()) * 100))) + up(float64(total) / float64(stream.GetSize()) * 100) }, } req, err := http.NewRequest(http.MethodPost, endpoint.String(), progressReader) diff --git a/drivers/wopan/driver.go b/drivers/wopan/driver.go index a3f222e8ef3..e5e26c94a08 100644 --- a/drivers/wopan/driver.go +++ b/drivers/wopan/driver.go @@ -159,7 +159,7 @@ func (d *Wopan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre ContentType: stream.GetMimetype(), }, dstDir.GetID(), d.FamilyID, wopan.Upload2COption{ OnProgress: func(current, total int64) { - up(int(100 * current / total)) + up(100 * float64(current) / float64(total)) }, }) return err diff --git a/internal/aria2/monitor.go b/internal/aria2/monitor.go index 77265b372b1..aaef3fd7c70 100644 --- a/internal/aria2/monitor.go +++ b/internal/aria2/monitor.go @@ -2,7 +2,6 @@ package aria2 import ( "fmt" - "github.com/alist-org/alist/v3/internal/stream" "os" "path" "path/filepath" @@ -11,6 +10,8 @@ import ( "sync/atomic" "time" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/task" @@ -100,7 +101,7 @@ func (m *Monitor) Update() (bool, error) { downloaded = 0 } progress := float64(downloaded) / float64(total) * 100 - m.tsk.SetProgress(int(progress)) + m.tsk.SetProgress(progress) switch info.Status { case "complete": err := m.Complete() diff --git a/internal/driver/driver.go b/internal/driver/driver.go index e0a7c93d908..781e85325ee 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -109,7 +109,7 @@ type PutResult interface { Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error) } -type UpdateProgress func(percentage int) +type UpdateProgress func(percentage float64) type Progress struct { Total int64 @@ -120,7 +120,7 @@ type Progress struct { func (p *Progress) Write(b []byte) (n int, err error) { n = len(b) p.Done += int64(n) - p.up(int(float64(p.Done) / float64(p.Total) * 100)) + p.up(float64(p.Done) / float64(p.Total) * 100) return } diff --git a/internal/offline_download/base.go b/internal/offline_download/base.go index 2276a6a586f..10476bf777f 100644 --- a/internal/offline_download/base.go +++ b/internal/offline_download/base.go @@ -3,6 +3,7 @@ package offline_download import ( "io" "os" + "time" "github.com/alist-org/alist/v3/internal/model" ) @@ -15,7 +16,7 @@ type AddUriArgs struct { } type Status struct { - Progress int + Progress float64 NewTID string Completed bool Status string @@ -39,9 +40,10 @@ type Tool interface { type File struct { io.ReadCloser - Name string - Size int64 - Path string + Name string + Size int64 + Path string + Modified time.Time } func (f *File) GetReadCloser() (io.ReadCloser, error) { diff --git a/internal/offline_download/monitor.go b/internal/offline_download/monitor.go index f446f268ece..2c0e7ff86bd 100644 --- a/internal/offline_download/monitor.go +++ b/internal/offline_download/monitor.go @@ -11,6 +11,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/task" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" @@ -134,22 +135,24 @@ func (m *Monitor) Complete() error { if err != nil { return errors.Wrapf(err, "failed to open file %s", file.Path) } - stream := &model.FileStream{ + s := &stream.FileStream{ + Ctx: nil, Obj: &model.Object{ Name: path.Base(file.Path), Size: file.Size, - Modified: time.Now(), + Modified: file.Modified, IsFolder: false, }, - ReadCloser: rc, - Mimetype: mimetype, + Reader: rc, + Mimetype: mimetype, + Closers: utils.NewClosers(rc), } relDir, err := filepath.Rel(m.tempDir, filepath.Dir(file.Path)) if err != nil { log.Errorf("find relation directory error: %v", err) } newDistDir := filepath.Join(dstDirActualPath, relDir) - return op.Put(tsk.Ctx, storage, newDistDir, stream, tsk.SetProgress) + return op.Put(tsk.Ctx, storage, newDistDir, s, tsk.SetProgress) }, })) } diff --git a/internal/offline_download/util.go b/internal/offline_download/util.go index 8f1c14843f6..a1950ef30fd 100644 --- a/internal/offline_download/util.go +++ b/internal/offline_download/util.go @@ -13,9 +13,10 @@ func GetFiles(dir string) ([]*File, error) { } if !info.IsDir() { files = append(files, &File{ - Name: info.Name(), - Size: info.Size(), - Path: path, + Name: info.Name(), + Size: info.Size(), + Path: path, + Modified: info.ModTime(), }) } return nil diff --git a/internal/op/fs.go b/internal/op/fs.go index 8ee6993e091..9fe7d5e6a3f 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -534,7 +534,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod } // if up is nil, set a default to prevent panic if up == nil { - up = func(p int) {} + up = func(p float64) {} } switch s := storage.(type) { diff --git a/internal/qbittorrent/monitor.go b/internal/qbittorrent/monitor.go index 12bb4ad21c5..bfb1bcf42e1 100644 --- a/internal/qbittorrent/monitor.go +++ b/internal/qbittorrent/monitor.go @@ -2,13 +2,14 @@ package qbittorrent import ( "fmt" - "github.com/alist-org/alist/v3/internal/stream" "os" "path/filepath" "sync" "sync/atomic" "time" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/task" @@ -85,7 +86,7 @@ func (m *Monitor) update() (bool, error) { } progress := float64(info.Completed) / float64(info.Size) * 100 - m.tsk.SetProgress(int(progress)) + m.tsk.SetProgress(progress) switch info.State { case UPLOADING, PAUSEDUP, QUEUEDUP, STALLEDUP, FORCEDUP, CHECKINGUP: err = m.complete() diff --git a/pkg/task/task.go b/pkg/task/task.go index f47eb7472ce..5b634f10cdb 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -26,7 +26,7 @@ type Task[K comparable] struct { Name string state string // pending, running, finished, canceling, canceled, errored status string - progress int + progress float64 Error error @@ -41,11 +41,11 @@ func (t *Task[K]) SetStatus(status string) { t.status = status } -func (t *Task[K]) SetProgress(percentage int) { +func (t *Task[K]) SetProgress(percentage float64) { t.progress = percentage } -func (t Task[K]) GetProgress() int { +func (t Task[K]) GetProgress() float64 { return t.progress } diff --git a/pkg/utils/io.go b/pkg/utils/io.go index d106531bd3d..6852e28a83d 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -5,10 +5,11 @@ import ( "context" "errors" "fmt" - "golang.org/x/exp/constraints" "io" "time" + "golang.org/x/exp/constraints" + log "github.com/sirupsen/logrus" ) @@ -21,7 +22,7 @@ func (rf readerFunc) Read(p []byte) (n int, err error) { return rf(p) } // CopyWithCtx slightly modified function signature: // - context has been added in order to propagate cancellation // - I do not return the number of bytes written, has it is not useful in my use case -func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, progress func(percentage int)) error { +func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, progress func(percentage float64)) error { // Copy will call the Reader and Writer interface multiple time, in order // to copy by chunk (avoiding loading the whole file in memory). // I insert the ability to cancel before read time as it is the earliest @@ -40,7 +41,7 @@ func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, p n, err := in.Read(p) if s > 0 && (err == nil || err == io.EOF) { finish += int64(n) - progress(int(finish / s)) + progress(float64(finish) / float64(s)) } return n, err } diff --git a/server/handles/task.go b/server/handles/task.go index d76bb586e27..8d311b46cce 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -12,12 +12,12 @@ import ( ) type TaskInfo struct { - ID string `json:"id"` - Name string `json:"name"` - State string `json:"state"` - Status string `json:"status"` - Progress int `json:"progress"` - Error string `json:"error"` + ID string `json:"id"` + Name string `json:"name"` + State string `json:"state"` + Status string `json:"status"` + Progress float64 `json:"progress"` + Error string `json:"error"` } type K2Str[K comparable] func(k K) string From 0acb2d60730e0e55018ebad7dd6ba65412d66ff3 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 4 Oct 2023 22:23:45 +0800 Subject: [PATCH 03/10] wip: adapt aria2 --- internal/offline_download/add.go | 6 ++- internal/offline_download/all_test.go | 12 +++-- internal/offline_download/aria2/aria2.go | 63 ++++++++++++++++++++---- internal/offline_download/base.go | 2 +- 4 files changed, 67 insertions(+), 16 deletions(-) diff --git a/internal/offline_download/add.go b/internal/offline_download/add.go index 67147f35c79..253d8a705b3 100644 --- a/internal/offline_download/add.go +++ b/internal/offline_download/add.go @@ -25,6 +25,10 @@ func AddURI(ctx context.Context, args *AddURIArgs) error { if err != nil { return errors.Wrapf(err, "failed get tool") } + // check tool is ready + if !tool.IsReady() { + return errors.Wrapf(err, "tool %s is not ready", args.Tool) + } // check storage storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath) if err != nil { @@ -57,7 +61,7 @@ func AddURI(ctx context.Context, args *AddURIArgs) error { Signal: signal, }) if err != nil { - return errors.Wrapf(err, "failed to add uri %s", args.URI) + return errors.Wrapf(err, "[%s] failed to add uri %s", args.Tool, args.URI) } DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{ ID: gid, diff --git a/internal/offline_download/all_test.go b/internal/offline_download/all_test.go index e8b32e09b3d..2ba6cf27162 100644 --- a/internal/offline_download/all_test.go +++ b/internal/offline_download/all_test.go @@ -1,13 +1,17 @@ -package offline_download +package offline_download_test -import "testing" +import ( + "testing" + + "github.com/alist-org/alist/v3/internal/offline_download" +) func TestGetFiles(t *testing.T) { - files, err := GetFiles("..") + files, err := offline_download.GetFiles("..") if err != nil { t.Fatal(err) } for _, file := range files { - t.Log(file.Name, file.Size, file.Path) + t.Log(file.Name, file.Size, file.Path, file.Modified) } } diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go index 947b890c9a9..67d8b9c88e5 100644 --- a/internal/offline_download/aria2/aria2.go +++ b/internal/offline_download/aria2/aria2.go @@ -3,6 +3,7 @@ package aria2 import ( "context" "fmt" + "strconv" "time" "github.com/alist-org/alist/v3/internal/conf" @@ -46,28 +47,70 @@ func (a *Aria2) Init() (string, error) { } func (a *Aria2) IsReady() bool { - //TODO implement me - panic("implement me") + return a.client != nil } func (a *Aria2) AddURI(args *offline_download.AddUriArgs) (string, error) { - //TODO implement me - panic("implement me") + options := map[string]interface{}{ + "dir": args.TempDir, + } + gid, err := a.client.AddURI([]string{args.Uri}, options) + if err != nil { + return "", err + } + return gid, nil } func (a *Aria2) Remove(tid string) error { - //TODO implement me - panic("implement me") + _, err := a.client.Remove(tid) + return err } func (a *Aria2) Status(tid string) (*offline_download.Status, error) { - //TODO implement me - panic("implement me") + info, err := a.client.TellStatus(tid) + if err != nil { + return nil, err + } + total, err := strconv.ParseUint(info.TotalLength, 10, 64) + if err != nil { + total = 0 + } + downloaded, err := strconv.ParseUint(info.CompletedLength, 10, 64) + if err != nil { + downloaded = 0 + } + s := &offline_download.Status{ + Completed: info.Status == "complete", + Err: err, + } + s.Progress = float64(downloaded) / float64(total) * 100 + if len(info.FollowedBy) != 0 { + s.NewTID = info.FollowedBy[0] + notify.Signals.Delete(tid) + //notify.Signals.Store(gid, m.c) + } + switch info.Status { + case "complete": + s.Completed = true + case "error": + s.Err = errors.Errorf("failed to download %s, error: %s", tid, info.ErrorMessage) + case "active": + s.Status = "aria2: " + info.Status + if info.Seeder == "true" { + s.Completed = true + } + case "waiting", "paused": + s.Status = "aria2: " + info.Status + case "removed": + s.Err = errors.Errorf("failed to download %s, removed", tid) + default: + return nil, errors.Errorf("[aria2] unknown status %s", info.Status) + } + return s, nil } func (a *Aria2) GetFile(tid string) *offline_download.File { - //TODO implement me - panic("implement me") + return nil } var _ offline_download.Tool = (*Aria2)(nil) diff --git a/internal/offline_download/base.go b/internal/offline_download/base.go index 10476bf777f..a0768e5ad9c 100644 --- a/internal/offline_download/base.go +++ b/internal/offline_download/base.go @@ -30,7 +30,7 @@ type Tool interface { IsReady() bool // AddURI add an uri to download, return the task id AddURI(args *AddUriArgs) (string, error) - // Remove the task if an error occurred + // Remove the download if task been canceled Remove(tid string) error // Status return the status of the download task, if an error occurred, return the error in Status.Err Status(tid string) (*Status, error) From 0380d7fff98365583e6f6cac888e1ddba120eaf8 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 5 Oct 2023 13:38:35 +0800 Subject: [PATCH 04/10] wip: use items in offline_download --- cmd/root.go | 2 ++ cmd/server.go | 1 - internal/bootstrap/data/setting.go | 2 ++ internal/offline_download/all.go | 5 +++++ internal/offline_download/aria2/aria2.go | 16 ++++++++++------ internal/offline_download/{ => tool}/add.go | 2 +- internal/offline_download/{ => tool}/all_test.go | 6 +++--- internal/offline_download/{ => tool}/base.go | 2 +- internal/offline_download/{ => tool}/monitor.go | 2 +- internal/offline_download/{ => tool}/tools.go | 11 ++++++++++- internal/offline_download/{ => tool}/util.go | 2 +- 11 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 internal/offline_download/all.go rename internal/offline_download/{ => tool}/add.go (98%) rename internal/offline_download/{ => tool}/all_test.go (57%) rename internal/offline_download/{ => tool}/base.go (97%) rename internal/offline_download/{ => tool}/monitor.go (99%) rename internal/offline_download/{ => tool}/tools.go (71%) rename internal/offline_download/{ => tool}/util.go (94%) diff --git a/cmd/root.go b/cmd/root.go index 297eb7f8940..6bd82b7a4a3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,6 +5,8 @@ import ( "os" "github.com/alist-org/alist/v3/cmd/flags" + _ "github.com/alist-org/alist/v3/drivers" + _ "github.com/alist-org/alist/v3/internal/offline_download" "github.com/spf13/cobra" ) diff --git a/cmd/server.go b/cmd/server.go index 94a60c7208f..f78d82b829c 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -13,7 +13,6 @@ import ( "time" "github.com/alist-org/alist/v3/cmd/flags" - _ "github.com/alist-org/alist/v3/drivers" "github.com/alist-org/alist/v3/internal/bootstrap" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/pkg/utils" diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 70e237c0666..5740b510199 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -4,6 +4,7 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" @@ -168,6 +169,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, {Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE}, } + initialSettingItems = append(initialSettingItems, tool.Tools.Items()...) if flags.Dev { initialSettingItems = append(initialSettingItems, []model.SettingItem{ {Key: "test_deprecated", Value: "test_value", Type: conf.TypeString, Flag: model.DEPRECATED}, diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go new file mode 100644 index 00000000000..a91cb7e109d --- /dev/null +++ b/internal/offline_download/all.go @@ -0,0 +1,5 @@ +package offline_download + +import ( + _ "github.com/alist-org/alist/v3/internal/offline_download/aria2" +) diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go index 67d8b9c88e5..a2a6135ddfa 100644 --- a/internal/offline_download/aria2/aria2.go +++ b/internal/offline_download/aria2/aria2.go @@ -8,7 +8,7 @@ import ( "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/offline_download" + "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/aria2/rpc" "github.com/pkg/errors" @@ -50,7 +50,7 @@ func (a *Aria2) IsReady() bool { return a.client != nil } -func (a *Aria2) AddURI(args *offline_download.AddUriArgs) (string, error) { +func (a *Aria2) AddURI(args *tool.AddUriArgs) (string, error) { options := map[string]interface{}{ "dir": args.TempDir, } @@ -66,7 +66,7 @@ func (a *Aria2) Remove(tid string) error { return err } -func (a *Aria2) Status(tid string) (*offline_download.Status, error) { +func (a *Aria2) Status(tid string) (*tool.Status, error) { info, err := a.client.TellStatus(tid) if err != nil { return nil, err @@ -79,7 +79,7 @@ func (a *Aria2) Status(tid string) (*offline_download.Status, error) { if err != nil { downloaded = 0 } - s := &offline_download.Status{ + s := &tool.Status{ Completed: info.Status == "complete", Err: err, } @@ -109,8 +109,12 @@ func (a *Aria2) Status(tid string) (*offline_download.Status, error) { return s, nil } -func (a *Aria2) GetFile(tid string) *offline_download.File { +func (a *Aria2) GetFile(tid string) *tool.File { return nil } -var _ offline_download.Tool = (*Aria2)(nil) +var _ tool.Tool = (*Aria2)(nil) + +func init() { + tool.Tools.Add("aria2", &Aria2{}) +} diff --git a/internal/offline_download/add.go b/internal/offline_download/tool/add.go similarity index 98% rename from internal/offline_download/add.go rename to internal/offline_download/tool/add.go index 253d8a705b3..ce613838a23 100644 --- a/internal/offline_download/add.go +++ b/internal/offline_download/tool/add.go @@ -1,4 +1,4 @@ -package offline_download +package tool import ( "context" diff --git a/internal/offline_download/all_test.go b/internal/offline_download/tool/all_test.go similarity index 57% rename from internal/offline_download/all_test.go rename to internal/offline_download/tool/all_test.go index 2ba6cf27162..27da5e32a89 100644 --- a/internal/offline_download/all_test.go +++ b/internal/offline_download/tool/all_test.go @@ -1,13 +1,13 @@ -package offline_download_test +package tool_test import ( "testing" - "github.com/alist-org/alist/v3/internal/offline_download" + "github.com/alist-org/alist/v3/internal/offline_download/tool" ) func TestGetFiles(t *testing.T) { - files, err := offline_download.GetFiles("..") + files, err := tool.GetFiles("..") if err != nil { t.Fatal(err) } diff --git a/internal/offline_download/base.go b/internal/offline_download/tool/base.go similarity index 97% rename from internal/offline_download/base.go rename to internal/offline_download/tool/base.go index a0768e5ad9c..8d5a5282e30 100644 --- a/internal/offline_download/base.go +++ b/internal/offline_download/tool/base.go @@ -1,4 +1,4 @@ -package offline_download +package tool import ( "io" diff --git a/internal/offline_download/monitor.go b/internal/offline_download/tool/monitor.go similarity index 99% rename from internal/offline_download/monitor.go rename to internal/offline_download/tool/monitor.go index 2c0e7ff86bd..79d82117761 100644 --- a/internal/offline_download/monitor.go +++ b/internal/offline_download/tool/monitor.go @@ -1,4 +1,4 @@ -package offline_download +package tool import ( "fmt" diff --git a/internal/offline_download/tools.go b/internal/offline_download/tool/tools.go similarity index 71% rename from internal/offline_download/tools.go rename to internal/offline_download/tool/tools.go index ca328f24e14..b7eacbd2b9b 100644 --- a/internal/offline_download/tools.go +++ b/internal/offline_download/tool/tools.go @@ -1,8 +1,9 @@ -package offline_download +package tool import ( "fmt" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/task" ) @@ -31,3 +32,11 @@ func (t ToolsManager) Names() []string { } return names } + +func (t ToolsManager) Items() []model.SettingItem { + var items []model.SettingItem + for _, tool := range t { + items = append(items, tool.Items()...) + } + return items +} diff --git a/internal/offline_download/util.go b/internal/offline_download/tool/util.go similarity index 94% rename from internal/offline_download/util.go rename to internal/offline_download/tool/util.go index a1950ef30fd..4344b89f5a9 100644 --- a/internal/offline_download/util.go +++ b/internal/offline_download/tool/util.go @@ -1,4 +1,4 @@ -package offline_download +package tool import ( "os" From 12dfb60a666f56feb4b31377721007d740c128e3 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 5 Oct 2023 22:13:02 +0800 Subject: [PATCH 05/10] wip: use tool manager --- cmd/server.go | 3 +- internal/bootstrap/aria2.go | 16 ---- internal/bootstrap/offline_download.go | 17 ++++ internal/bootstrap/qbittorrent.go | 15 ---- internal/model/user.go | 9 +-- internal/offline_download/aria2/aria2.go | 19 ++++- internal/offline_download/tool/add.go | 20 +++-- internal/offline_download/tool/base.go | 13 +-- internal/offline_download/tool/monitor.go | 9 +-- internal/offline_download/tool/util.go | 6 +- .../handles/{aria2.go => offline_download.go} | 61 ++++++++++---- server/handles/qbittorrent.go | 79 ------------------- server/handles/task.go | 8 +- server/router.go | 6 +- 14 files changed, 115 insertions(+), 166 deletions(-) delete mode 100644 internal/bootstrap/aria2.go create mode 100644 internal/bootstrap/offline_download.go delete mode 100644 internal/bootstrap/qbittorrent.go rename server/handles/{aria2.go => offline_download.go} (53%) delete mode 100644 server/handles/qbittorrent.go diff --git a/cmd/server.go b/cmd/server.go index f78d82b829c..0678e3e1188 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -34,8 +34,7 @@ the address is defined in config file`, utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart) time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second) } - bootstrap.InitAria2() - bootstrap.InitQbittorrent() + bootstrap.InitOfflineDownloadTools() bootstrap.LoadStorages() if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) diff --git a/internal/bootstrap/aria2.go b/internal/bootstrap/aria2.go deleted file mode 100644 index 60017ddaa6f..00000000000 --- a/internal/bootstrap/aria2.go +++ /dev/null @@ -1,16 +0,0 @@ -package bootstrap - -import ( - "github.com/alist-org/alist/v3/internal/aria2" - "github.com/alist-org/alist/v3/pkg/utils" -) - -func InitAria2() { - go func() { - _, err := aria2.InitClient(2) - if err != nil { - //utils.Log.Errorf("failed to init aria2 client: %+v", err) - utils.Log.Infof("Aria2 not ready.") - } - }() -} diff --git a/internal/bootstrap/offline_download.go b/internal/bootstrap/offline_download.go new file mode 100644 index 00000000000..26e04071b10 --- /dev/null +++ b/internal/bootstrap/offline_download.go @@ -0,0 +1,17 @@ +package bootstrap + +import ( + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func InitOfflineDownloadTools() { + for k, v := range tool.Tools { + res, err := v.Init() + if err != nil { + utils.Log.Warnf("init tool %s failed: %s", k, err) + } else { + utils.Log.Infof("init tool %s success: %s", k, res) + } + } +} diff --git a/internal/bootstrap/qbittorrent.go b/internal/bootstrap/qbittorrent.go deleted file mode 100644 index 315977ebe3f..00000000000 --- a/internal/bootstrap/qbittorrent.go +++ /dev/null @@ -1,15 +0,0 @@ -package bootstrap - -import ( - "github.com/alist-org/alist/v3/internal/qbittorrent" - "github.com/alist-org/alist/v3/pkg/utils" -) - -func InitQbittorrent() { - go func() { - err := qbittorrent.InitClient() - if err != nil { - utils.Log.Infof("qbittorrent not ready.") - } - }() -} diff --git a/internal/model/user.go b/internal/model/user.go index d7b2863cebe..46fe9bd301d 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -32,7 +32,7 @@ type User struct { // Determine permissions by bit // 0: can see hidden files // 1: can access without password - // 2: can add aria2 tasks + // 2: can add offline download tasks // 3: can mkdir and upload // 4: can rename // 5: can move @@ -40,7 +40,6 @@ type User struct { // 7: can remove // 8: webdav read // 9: webdav write - // 10: can add qbittorrent tasks Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -83,7 +82,7 @@ func (u *User) CanAccessWithoutPassword() bool { return u.IsAdmin() || (u.Permission>>1)&1 == 1 } -func (u *User) CanAddAria2Tasks() bool { +func (u *User) CanAddOfflineDownloadTasks() bool { return u.IsAdmin() || (u.Permission>>2)&1 == 1 } @@ -115,10 +114,6 @@ func (u *User) CanWebdavManage() bool { return u.IsAdmin() || (u.Permission>>9)&1 == 1 } -func (u *User) CanAddQbittorrentTasks() bool { - return u.IsAdmin() || (u.Permission>>10)&1 == 1 -} - func (u *User) JoinPath(reqPath string) (string, error) { return utils.JoinBasePath(u.BasePath, reqPath) } diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go index a2a6135ddfa..f2b9628c240 100644 --- a/internal/offline_download/aria2/aria2.go +++ b/internal/offline_download/aria2/aria2.go @@ -50,11 +50,11 @@ func (a *Aria2) IsReady() bool { return a.client != nil } -func (a *Aria2) AddURI(args *tool.AddUriArgs) (string, error) { +func (a *Aria2) AddURL(args *tool.AddUrlArgs) (string, error) { options := map[string]interface{}{ "dir": args.TempDir, } - gid, err := a.client.AddURI([]string{args.Uri}, options) + gid, err := a.client.AddURI([]string{args.Url}, options) if err != nil { return "", err } @@ -109,7 +109,20 @@ func (a *Aria2) Status(tid string) (*tool.Status, error) { return s, nil } -func (a *Aria2) GetFile(tid string) *tool.File { +func (a *Aria2) GetFiles(tid string) []tool.File { + //files, err := a.client.GetFiles(tid) + //if err != nil { + // return nil + //} + //return utils.MustSliceConvert(files, func(f rpc.FileInfo) tool.File { + // return tool.File{ + // //ReadCloser: nil, + // Name: path.Base(f.Path), + // Size: f.Length, + // Path: "", + // Modified: time.Time{}, + // } + //}) return nil } diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index ce613838a23..ceaf92d3bc3 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -13,13 +13,13 @@ import ( "github.com/pkg/errors" ) -type AddURIArgs struct { - URI string +type AddURLArgs struct { + URL string DstDirPath string Tool string } -func AddURI(ctx context.Context, args *AddURIArgs) error { +func AddURL(ctx context.Context, args *AddURLArgs) error { // get tool tool, err := Tools.Get(args.Tool) if err != nil { @@ -27,7 +27,10 @@ func AddURI(ctx context.Context, args *AddURIArgs) error { } // check tool is ready if !tool.IsReady() { - return errors.Wrapf(err, "tool %s is not ready", args.Tool) + // try to init tool + if _, err := tool.Init(); err != nil { + return errors.Wrapf(err, "failed init tool %s", args.Tool) + } } // check storage storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath) @@ -54,20 +57,21 @@ func AddURI(ctx context.Context, args *AddURIArgs) error { uid := uuid.NewString() tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) signal := make(chan int) - gid, err := tool.AddURI(&AddUriArgs{ - Uri: args.URI, + gid, err := tool.AddURL(&AddUrlArgs{ + Url: args.URL, UID: uid, TempDir: tempDir, Signal: signal, }) if err != nil { - return errors.Wrapf(err, "[%s] failed to add uri %s", args.Tool, args.URI) + return errors.Wrapf(err, "[%s] failed to add uri %s", args.Tool, args.URL) } DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{ ID: gid, - Name: fmt.Sprintf("download %s to [%s](%s)", args.URI, storage.GetStorage().MountPath, dstDirActualPath), + Name: fmt.Sprintf("download %s to [%s](%s)", args.URL, storage.GetStorage().MountPath, dstDirActualPath), Func: func(tsk *task.Task[string]) error { m := &Monitor{ + tool: tool, tsk: tsk, tempDir: tempDir, dstDirPath: args.DstDirPath, diff --git a/internal/offline_download/tool/base.go b/internal/offline_download/tool/base.go index 8d5a5282e30..4689635b54e 100644 --- a/internal/offline_download/tool/base.go +++ b/internal/offline_download/tool/base.go @@ -8,8 +8,8 @@ import ( "github.com/alist-org/alist/v3/internal/model" ) -type AddUriArgs struct { - Uri string +type AddUrlArgs struct { + Url string UID string TempDir string Signal chan int @@ -28,17 +28,18 @@ type Tool interface { Items() []model.SettingItem Init() (string, error) IsReady() bool - // AddURI add an uri to download, return the task id - AddURI(args *AddUriArgs) (string, error) + // AddURL add an uri to download, return the task id + AddURL(args *AddUrlArgs) (string, error) // Remove the download if task been canceled Remove(tid string) error // Status return the status of the download task, if an error occurred, return the error in Status.Err Status(tid string) (*Status, error) - // GetFile return an io.ReadCloser as the download file, if nil, means walk the temp dir to get the files - GetFile(tid string) *File + // GetFiles return the files of the download task, if nil, means walk the temp dir to get the files + GetFiles(tid string) []File } type File struct { + // ReadCloser for http client io.ReadCloser Name string Size int64 diff --git a/internal/offline_download/tool/monitor.go b/internal/offline_download/tool/monitor.go index 79d82117761..984bda17cb7 100644 --- a/internal/offline_download/tool/monitor.go +++ b/internal/offline_download/tool/monitor.go @@ -3,7 +3,6 @@ package tool import ( "fmt" "os" - "path" "path/filepath" "sync" "sync/atomic" @@ -104,9 +103,9 @@ func (m *Monitor) Complete() error { if err != nil { return errors.WithMessage(err, "failed get storage") } - var files []*File - if f := m.tool.GetFile(m.tsk.ID); f != nil { - files = append(files, f) + var files []File + if f := m.tool.GetFiles(m.tsk.ID); f != nil { + files = f } else { files, err = GetFiles(m.tempDir) if err != nil { @@ -138,7 +137,7 @@ func (m *Monitor) Complete() error { s := &stream.FileStream{ Ctx: nil, Obj: &model.Object{ - Name: path.Base(file.Path), + Name: filepath.Base(file.Path), Size: file.Size, Modified: file.Modified, IsFolder: false, diff --git a/internal/offline_download/tool/util.go b/internal/offline_download/tool/util.go index 4344b89f5a9..4258eff61e0 100644 --- a/internal/offline_download/tool/util.go +++ b/internal/offline_download/tool/util.go @@ -5,14 +5,14 @@ import ( "path/filepath" ) -func GetFiles(dir string) ([]*File, error) { - var files []*File +func GetFiles(dir string) ([]File, error) { + var files []File err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { - files = append(files, &File{ + files = append(files, File{ Name: info.Name(), Size: info.Size(), Path: path, diff --git a/server/handles/aria2.go b/server/handles/offline_download.go similarity index 53% rename from server/handles/aria2.go rename to server/handles/offline_download.go index a78eb65941c..72948ebaf9e 100644 --- a/server/handles/aria2.go +++ b/server/handles/offline_download.go @@ -4,7 +4,9 @@ import ( "github.com/alist-org/alist/v3/internal/aria2" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/qbittorrent" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" ) @@ -36,30 +38,51 @@ func SetAria2(c *gin.Context) { common.SuccessResp(c, version) } -type AddAria2Req struct { +type SetQbittorrentReq struct { + Url string `json:"url" form:"url"` + Seedtime string `json:"seedtime" form:"seedtime"` +} + +func SetQbittorrent(c *gin.Context) { + var req SetQbittorrentReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + items := []model.SettingItem{ + {Key: conf.QbittorrentUrl, Value: req.Url, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.QbittorrentSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + if err := qbittorrent.InitClient(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +func OfflineDownloadTools(c *gin.Context) { + tools := tool.Tools.Names() + common.SuccessResp(c, tools) +} + +type AddOfflineDownloadReq struct { Urls []string `json:"urls"` Path string `json:"path"` + Tool string `json:"tool"` } -func AddAria2(c *gin.Context) { +func AddOfflineDownload(c *gin.Context) { user := c.MustGet("user").(*model.User) - if !user.CanAddAria2Tasks() { + if !user.CanAddOfflineDownloadTasks() { common.ErrorStrResp(c, "permission denied", 403) return } - if !aria2.IsAria2Ready() { - // try to init client - _, err := aria2.InitClient(2) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if !aria2.IsAria2Ready() { - common.ErrorStrResp(c, "aria2 still not ready after init", 500) - return - } - } - var req AddAria2Req + + var req AddOfflineDownloadReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return @@ -70,7 +93,11 @@ func AddAria2(c *gin.Context) { return } for _, url := range req.Urls { - err := aria2.AddURI(c, url, reqPath) + err := tool.AddURL(c, &tool.AddURLArgs{ + URL: url, + DstDirPath: reqPath, + Tool: req.Tool, + }) if err != nil { common.ErrorResp(c, err, 500) return diff --git a/server/handles/qbittorrent.go b/server/handles/qbittorrent.go deleted file mode 100644 index b22804546ef..00000000000 --- a/server/handles/qbittorrent.go +++ /dev/null @@ -1,79 +0,0 @@ -package handles - -import ( - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/qbittorrent" - "github.com/alist-org/alist/v3/server/common" - "github.com/gin-gonic/gin" -) - -type SetQbittorrentReq struct { - Url string `json:"url" form:"url"` - Seedtime string `json:"seedtime" form:"seedtime"` -} - -func SetQbittorrent(c *gin.Context) { - var req SetQbittorrentReq - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - items := []model.SettingItem{ - {Key: conf.QbittorrentUrl, Value: req.Url, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, - {Key: conf.QbittorrentSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE}, - } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - if err := qbittorrent.InitClient(); err != nil { - common.ErrorResp(c, err, 500) - return - } - common.SuccessResp(c, "ok") -} - -type AddQbittorrentReq struct { - Urls []string `json:"urls"` - Path string `json:"path"` -} - -func AddQbittorrent(c *gin.Context) { - user := c.MustGet("user").(*model.User) - if !user.CanAddQbittorrentTasks() { - common.ErrorStrResp(c, "permission denied", 403) - return - } - if !qbittorrent.IsQbittorrentReady() { - // try to init client - err := qbittorrent.InitClient() - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if !qbittorrent.IsQbittorrentReady() { - common.ErrorStrResp(c, "qbittorrent still not ready after init", 500) - return - } - } - var req AddQbittorrentReq - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - reqPath, err := user.JoinPath(req.Path) - if err != nil { - common.ErrorResp(c, err, 403) - return - } - for _, url := range req.Urls { - err := qbittorrent.AddURL(c, url, reqPath) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - } - common.SuccessResp(c) -} diff --git a/server/handles/task.go b/server/handles/task.go index 8d311b46cce..15e8067248a 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -3,8 +3,8 @@ package handles import ( "strconv" - "github.com/alist-org/alist/v3/internal/aria2" "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/qbittorrent" "github.com/alist-org/alist/v3/pkg/task" "github.com/alist-org/alist/v3/server/common" @@ -116,10 +116,12 @@ func taskRoute[K comparable](g *gin.RouterGroup, manager *task.Manager[K], k2Str } func SetupTaskRoute(g *gin.RouterGroup) { - taskRoute(g.Group("/aria2_down"), aria2.DownTaskManager, strK2Str, str2StrK) - taskRoute(g.Group("/aria2_transfer"), aria2.TransferTaskManager, uint64K2Str, str2Uint64K) taskRoute(g.Group("/upload"), fs.UploadTaskManager, uint64K2Str, str2Uint64K) taskRoute(g.Group("/copy"), fs.CopyTaskManager, uint64K2Str, str2Uint64K) taskRoute(g.Group("/qbit_down"), qbittorrent.DownTaskManager, strK2Str, str2StrK) taskRoute(g.Group("/qbit_transfer"), qbittorrent.TransferTaskManager, uint64K2Str, str2Uint64K) + //taskRoute(g.Group("/aria2_down"), aria2.DownTaskManager, strK2Str, str2StrK) + //taskRoute(g.Group("/aria2_transfer"), aria2.TransferTaskManager, uint64K2Str, str2Uint64K) + taskRoute(g.Group("/offline_download"), tool.DownTaskManager, strK2Str, str2StrK) + taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager, uint64K2Str, str2Uint64K) } diff --git a/server/router.go b/server/router.go index 92ede88bfde..7f179231ecd 100644 --- a/server/router.go +++ b/server/router.go @@ -70,6 +70,7 @@ func Init(e *gin.Engine) { // no need auth public := api.Group("/public") public.Any("/settings", handles.PublicSettings) + public.Any("/offline_download_tools", handles.OfflineDownloadTools) _fs(auth.Group("/fs")) admin(auth.Group("/admin", middlewares.AuthAdmin)) @@ -155,8 +156,9 @@ func _fs(g *gin.RouterGroup) { g.PUT("/put", middlewares.FsUp, handles.FsStream) g.PUT("/form", middlewares.FsUp, handles.FsForm) g.POST("/link", middlewares.AuthAdmin, handles.Link) - g.POST("/add_aria2", handles.AddAria2) - g.POST("/add_qbit", handles.AddQbittorrent) + //g.POST("/add_aria2", handles.AddOfflineDownload) + //g.POST("/add_qbit", handles.AddQbittorrent) + g.POST("/add_offline_download", handles.AddOfflineDownload) } func Cors(r *gin.Engine) { From 1490da8b53a18dec6cff8cdd0f4fa444f6c2c6a2 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 6 Oct 2023 16:02:29 +0800 Subject: [PATCH 06/10] wip: adapt qBittorrent --- internal/bootstrap/data/setting.go | 4 - internal/offline_download/all.go | 1 + internal/offline_download/qbit/qbit.go | 80 ++++++ pkg/qbittorrent/client.go | 366 +++++++++++++++++++++++++ server/handles/offline_download.go | 12 +- 5 files changed, 455 insertions(+), 8 deletions(-) create mode 100644 internal/offline_download/qbit/qbit.go create mode 100644 pkg/qbittorrent/client.go diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 5740b510199..f5ca599ccd4 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -164,10 +164,6 @@ func InitialSettings() []model.SettingItem { {Key: conf.SSODefaultDir, Value: "/", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOCompatibilityMode, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PUBLIC}, - - // qbittorrent settings - {Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, - {Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE}, } initialSettingItems = append(initialSettingItems, tool.Tools.Items()...) if flags.Dev { diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index a91cb7e109d..0c7853cb13f 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -2,4 +2,5 @@ package offline_download import ( _ "github.com/alist-org/alist/v3/internal/offline_download/aria2" + _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" ) diff --git a/internal/offline_download/qbit/qbit.go b/internal/offline_download/qbit/qbit.go new file mode 100644 index 00000000000..a9006f13416 --- /dev/null +++ b/internal/offline_download/qbit/qbit.go @@ -0,0 +1,80 @@ +package qbit + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/qbittorrent" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/pkg/errors" +) + +type QBittorrent struct { + client qbittorrent.Client +} + +func (a *QBittorrent) Items() []model.SettingItem { + // aria2 settings + return []model.SettingItem{ + {Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } +} + +func (a *QBittorrent) Init() (string, error) { + a.client = nil + url := setting.GetStr(conf.QbittorrentUrl) + qbClient, err := qbittorrent.New(url) + if err != nil { + return "", err + } + a.client = qbClient + return "ok", nil +} + +func (a *QBittorrent) IsReady() bool { + return a.client != nil +} + +func (a *QBittorrent) AddURL(args *tool.AddUrlArgs) (string, error) { + err := a.client.AddFromLink(args.Url, args.TempDir, args.UID) + if err != nil { + return "", err + } + return args.UID, nil +} + +func (a *QBittorrent) Remove(tid string) error { + err := a.client.Delete(tid, true) + return err +} + +func (a *QBittorrent) Status(tid string) (*tool.Status, error) { + info, err := a.client.GetInfo(tid) + if err != nil { + return nil, err + } + s := &tool.Status{} + s.Progress = float64(info.Completed) / float64(info.Size) * 100 + switch info.State { + case qbittorrent.UPLOADING, qbittorrent.PAUSEDUP, qbittorrent.QUEUEDUP, qbittorrent.STALLEDUP, qbittorrent.FORCEDUP, qbittorrent.CHECKINGUP: + s.Completed = true + case qbittorrent.ALLOCATING, qbittorrent.DOWNLOADING, qbittorrent.METADL, qbittorrent.PAUSEDDL, qbittorrent.QUEUEDDL, qbittorrent.STALLEDDL, qbittorrent.CHECKINGDL, qbittorrent.FORCEDDL, qbittorrent.CHECKINGRESUMEDATA, qbittorrent.MOVING: + s.Status = "[qBittorrent] downloading" + case qbittorrent.ERROR, qbittorrent.MISSINGFILES, qbittorrent.UNKNOWN: + s.Err = errors.Errorf("[qBittorrent] failed to download %s, error: %s", tid, info.State) + default: + s.Err = errors.Errorf("[qBittorrent] unknown error occurred downloading %s", tid) + } + return s, nil +} + +func (a *QBittorrent) GetFiles(tid string) []tool.File { + return nil +} + +var _ tool.Tool = (*QBittorrent)(nil) + +func init() { + tool.Tools.Add("qBittorrent", &QBittorrent{}) +} diff --git a/pkg/qbittorrent/client.go b/pkg/qbittorrent/client.go new file mode 100644 index 00000000000..ec3f7e7b00c --- /dev/null +++ b/pkg/qbittorrent/client.go @@ -0,0 +1,366 @@ +package qbittorrent + +import ( + "bytes" + "errors" + "io" + "mime/multipart" + "net/http" + "net/http/cookiejar" + "net/url" + + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Client interface { + AddFromLink(link string, savePath string, id string) error + GetInfo(id string) (TorrentInfo, error) + GetFiles(id string) ([]FileInfo, error) + Delete(id string, deleteFiles bool) error +} + +type client struct { + url *url.URL + client http.Client + Client +} + +func New(webuiUrl string) (Client, error) { + u, err := url.Parse(webuiUrl) + if err != nil { + return nil, err + } + + jar, err := cookiejar.New(nil) + if err != nil { + return nil, err + } + var c = &client{ + url: u, + client: http.Client{Jar: jar}, + } + + err = c.checkAuthorization() + if err != nil { + return nil, err + } + return c, nil +} + +func (c *client) checkAuthorization() error { + // check authorization + if c.authorized() { + return nil + } + + // check authorization after logging in + err := c.login() + if err != nil { + return err + } + if c.authorized() { + return nil + } + return errors.New("unauthorized qbittorrent url") +} + +func (c *client) authorized() bool { + resp, err := c.post("/api/v2/app/version", nil) + if err != nil { + return false + } + return resp.StatusCode == 200 // the status code will be 403 if not authorized +} + +func (c *client) login() error { + // prepare HTTP request + v := url.Values{} + v.Set("username", c.url.User.Username()) + passwd, _ := c.url.User.Password() + v.Set("password", passwd) + resp, err := c.post("/api/v2/auth/login", v) + if err != nil { + return err + } + + // check result + body := make([]byte, 2) + _, err = resp.Body.Read(body) + if err != nil { + return err + } + if string(body) != "Ok" { + return errors.New("failed to login into qBittorrent webui with url: " + c.url.String()) + } + return nil +} + +func (c *client) post(path string, data url.Values) (*http.Response, error) { + u := c.url.JoinPath(path) + u.User = nil // remove userinfo for requests + + req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(data.Encode()))) + if err != nil { + return nil, err + } + if data != nil { + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + if resp.Cookies() != nil { + c.client.Jar.SetCookies(u, resp.Cookies()) + } + return resp, nil +} + +func (c *client) AddFromLink(link string, savePath string, id string) error { + err := c.checkAuthorization() + if err != nil { + return err + } + + buf := new(bytes.Buffer) + writer := multipart.NewWriter(buf) + + addField := func(name string, value string) { + if err != nil { + return + } + err = writer.WriteField(name, value) + } + addField("urls", link) + addField("savepath", savePath) + addField("tags", "alist-"+id) + addField("autoTMM", "false") + if err != nil { + return err + } + + err = writer.Close() + if err != nil { + return err + } + + u := c.url.JoinPath("/api/v2/torrents/add") + u.User = nil // remove userinfo for requests + req, err := http.NewRequest("POST", u.String(), buf) + if err != nil { + return err + } + req.Header.Add("Content-Type", writer.FormDataContentType()) + + resp, err := c.client.Do(req) + if err != nil { + return err + } + + // check result + body := make([]byte, 2) + _, err = resp.Body.Read(body) + if err != nil { + return err + } + if resp.StatusCode != 200 || string(body) != "Ok" { + return errors.New("failed to add qBittorrent task: " + link) + } + return nil +} + +type TorrentStatus string + +const ( + ERROR TorrentStatus = "error" + MISSINGFILES TorrentStatus = "missingFiles" + UPLOADING TorrentStatus = "uploading" + PAUSEDUP TorrentStatus = "pausedUP" + QUEUEDUP TorrentStatus = "queuedUP" + STALLEDUP TorrentStatus = "stalledUP" + CHECKINGUP TorrentStatus = "checkingUP" + FORCEDUP TorrentStatus = "forcedUP" + ALLOCATING TorrentStatus = "allocating" + DOWNLOADING TorrentStatus = "downloading" + METADL TorrentStatus = "metaDL" + PAUSEDDL TorrentStatus = "pausedDL" + QUEUEDDL TorrentStatus = "queuedDL" + STALLEDDL TorrentStatus = "stalledDL" + CHECKINGDL TorrentStatus = "checkingDL" + FORCEDDL TorrentStatus = "forcedDL" + CHECKINGRESUMEDATA TorrentStatus = "checkingResumeData" + MOVING TorrentStatus = "moving" + UNKNOWN TorrentStatus = "unknown" +) + +// https://github.com/DGuang21/PTGo/blob/main/app/client/client_distributer.go +type TorrentInfo struct { + AddedOn int `json:"added_on"` // 将 torrent 添加到客户端的时间（Unix Epoch） + AmountLeft int64 `json:"amount_left"` // 剩余大小（字节） + AutoTmm bool `json:"auto_tmm"` // 此 torrent 是否由 Automatic Torrent Management 管理 + Availability float64 `json:"availability"` // 当前百分比 + Category string `json:"category"` // + Completed int64 `json:"completed"` // 完成的传输数据量（字节） + CompletionOn int `json:"completion_on"` // Torrent 完成的时间（Unix Epoch） + ContentPath string `json:"content_path"` // torrent 内容的绝对路径（多文件 torrent 的根路径，单文件 torrent 的绝对文件路径） + DlLimit int `json:"dl_limit"` // Torrent 下载速度限制（字节/秒） + Dlspeed int `json:"dlspeed"` // Torrent 下载速度（字节/秒） + Downloaded int64 `json:"downloaded"` // 已经下载大小 + DownloadedSession int64 `json:"downloaded_session"` // 此会话下载的数据量 + Eta int `json:"eta"` // + FLPiecePrio bool `json:"f_l_piece_prio"` // 如果第一个最后一块被优先考虑，则为true + ForceStart bool `json:"force_start"` // 如果为此 torrent 启用了强制启动，则为true + Hash string `json:"hash"` // + LastActivity int `json:"last_activity"` // 上次活跃的时间（Unix Epoch） + MagnetURI string `json:"magnet_uri"` // 与此 torrent 对应的 Magnet URI + MaxRatio float64 `json:"max_ratio"` // 种子/上传停止种子前的最大共享比率 + MaxSeedingTime int `json:"max_seeding_time"` // 停止种子种子前的最长种子时间（秒） + Name string `json:"name"` // + NumComplete int `json:"num_complete"` // + NumIncomplete int `json:"num_incomplete"` // + NumLeechs int `json:"num_leechs"` // 连接到的 leechers 的数量 + NumSeeds int `json:"num_seeds"` // 连接到的种子数 + Priority int `json:"priority"` // 速度优先。如果队列被禁用或 torrent 处于种子模式，则返回 -1 + Progress float64 `json:"progress"` // 进度 + Ratio float64 `json:"ratio"` // Torrent 共享比率 + RatioLimit int `json:"ratio_limit"` // + SavePath string `json:"save_path"` + SeedingTime int `json:"seeding_time"` // Torrent 完成用时（秒） + SeedingTimeLimit int `json:"seeding_time_limit"` // max_seeding_time + SeenComplete int `json:"seen_complete"` // 上次 torrent 完成的时间 + SeqDl bool `json:"seq_dl"` // 如果启用顺序下载，则为true + Size int64 `json:"size"` // + State TorrentStatus `json:"state"` // 参见https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-(qBittorrent-4.1)#get-torrent-list + SuperSeeding bool `json:"super_seeding"` // 如果启用超级播种，则为true + Tags string `json:"tags"` // Torrent 的逗号连接标签列表 + TimeActive int `json:"time_active"` // 总活动时间（秒） + TotalSize int64 `json:"total_size"` // 此 torrent 中所有文件的总大小（字节）（包括未选择的文件） + Tracker string `json:"tracker"` // 第一个具有工作状态的tracker。如果没有tracker在工作，则返回空字符串。 + TrackersCount int `json:"trackers_count"` // + UpLimit int `json:"up_limit"` // 上传限制 + Uploaded int64 `json:"uploaded"` // 累计上传 + UploadedSession int64 `json:"uploaded_session"` // 当前session累计上传 + Upspeed int `json:"upspeed"` // 上传速度（字节/秒） +} + +type InfoNotFoundError struct { + Id string + Err error +} + +func (i InfoNotFoundError) Error() string { + return "there should be exactly one task with tag \"alist-" + i.Id + "\"" +} + +func NewInfoNotFoundError(id string) InfoNotFoundError { + return InfoNotFoundError{Id: id} +} + +func (c *client) GetInfo(id string) (TorrentInfo, error) { + var infos []TorrentInfo + + err := c.checkAuthorization() + if err != nil { + return TorrentInfo{}, err + } + + v := url.Values{} + v.Set("tag", "alist-"+id) + response, err := c.post("/api/v2/torrents/info", v) + if err != nil { + return TorrentInfo{}, err + } + + body, err := io.ReadAll(response.Body) + if err != nil { + return TorrentInfo{}, err + } + err = utils.Json.Unmarshal(body, &infos) + if err != nil { + return TorrentInfo{}, err + } + if len(infos) != 1 { + return TorrentInfo{}, NewInfoNotFoundError(id) + } + return infos[0], nil +} + +type FileInfo struct { + Index int `json:"index"` + Name string `json:"name"` + Size int64 `json:"size"` + Progress float32 `json:"progress"` + Priority int `json:"priority"` + IsSeed bool `json:"is_seed"` + PieceRange []int `json:"piece_range"` + Availability float32 `json:"availability"` +} + +func (c *client) GetFiles(id string) ([]FileInfo, error) { + var infos []FileInfo + + err := c.checkAuthorization() + if err != nil { + return []FileInfo{}, err + } + + tInfo, err := c.GetInfo(id) + if err != nil { + return []FileInfo{}, err + } + + v := url.Values{} + v.Set("hash", tInfo.Hash) + response, err := c.post("/api/v2/torrents/files", v) + if err != nil { + return []FileInfo{}, err + } + + body, err := io.ReadAll(response.Body) + if err != nil { + return []FileInfo{}, err + } + err = utils.Json.Unmarshal(body, &infos) + if err != nil { + return []FileInfo{}, err + } + return infos, nil +} + +func (c *client) Delete(id string, deleteFiles bool) error { + err := c.checkAuthorization() + if err != nil { + return err + } + + info, err := c.GetInfo(id) + if err != nil { + return err + } + v := url.Values{} + v.Set("hashes", info.Hash) + if deleteFiles { + v.Set("deleteFiles", "true") + } else { + v.Set("deleteFiles", "false") + } + response, err := c.post("/api/v2/torrents/delete", v) + if err != nil { + return err + } + if response.StatusCode != 200 { + return errors.New("failed to delete qbittorrent task") + } + + v = url.Values{} + v.Set("tags", "alist-"+id) + response, err = c.post("/api/v2/torrents/deleteTags", v) + if err != nil { + return err + } + if response.StatusCode != 200 { + return errors.New("failed to delete qbittorrent tag") + } + return nil +} diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 72948ebaf9e..cf9c1775bae 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -1,12 +1,10 @@ package handles import ( - "github.com/alist-org/alist/v3/internal/aria2" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/qbittorrent" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" ) @@ -30,7 +28,8 @@ func SetAria2(c *gin.Context) { common.ErrorResp(c, err, 500) return } - version, err := aria2.InitClient(2) + _tool, err := tool.Tools.Get("aria2") + version, err := _tool.Init() if err != nil { common.ErrorResp(c, err, 500) return @@ -57,7 +56,12 @@ func SetQbittorrent(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if err := qbittorrent.InitClient(); err != nil { + _tool, err := tool.Tools.Get("qBittorrent") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } From 9fb9efb7047159c1f0fcb294a849aa118917d64b Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 6 Oct 2023 22:32:05 +0800 Subject: [PATCH 07/10] chore: fix typo --- internal/offline_download/qbit/qbit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/offline_download/qbit/qbit.go b/internal/offline_download/qbit/qbit.go index a9006f13416..594088f0eb2 100644 --- a/internal/offline_download/qbit/qbit.go +++ b/internal/offline_download/qbit/qbit.go @@ -14,7 +14,7 @@ type QBittorrent struct { } func (a *QBittorrent) Items() []model.SettingItem { - // aria2 settings + // qBittorrent settings return []model.SettingItem{ {Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, From ce6e486666767879b6a90a49959c4fa3a0f8d50a Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 20 Oct 2023 21:14:15 +0800 Subject: [PATCH 08/10] Squashed commit of the following: commit 4fc0a77565702f9bf498485d42336502f2ee9776 Author: Andy Hsu Date: Fri Oct 20 21:06:25 2023 +0800 fix(baidu_netdisk): upload file > 4GB (close #5392) commit aaffaee2b54fc067d240ea0c20ea3c2f39615d6e Author: gmugu <94156510@qq.com> Date: Thu Oct 19 19:17:53 2023 +0800 perf(webdav): support request with cookies (#5391) commit 8ef8023c20bfeee97ec82155b52eae0d80b1410e Author: NewbieOrange Date: Thu Oct 19 19:17:09 2023 +0800 fix(aliyundrive_open): upload progress for normal upload (#5398) commit cdfbe6dcf2b361e4c93c2703c2f8c9bddeac0ee6 Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Wed Oct 18 16:27:07 2023 +0800 fix: hash gcid empty file (#5394) commit 94d028743abf8e0d736f80c0ec4fb294a1cc064c Author: Andy Hsu Date: Sat Oct 14 13:17:51 2023 +0800 ci: remove `pr-welcome` label when close issue [skip ci] commit 7f7335435c2f32a3eef76fac4c4f783d9d8624fd Author: itsHenry <2671230065@qq.com> Date: Sat Oct 14 13:12:46 2023 +0800 feat(cloudreve): support thumbnail (#5373 close #5348) * feat(cloudreve): support thumbnail * chore: remove unnecessary code commit b9e192b29cffddf14a0dfb2d3885def57a56ce16 Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Thu Oct 12 20:57:12 2023 +0800 fix(115): limit request rate (#5367 close #5275) * fix(115):limit request rate * chore(115): fix unit of `limit_rate` --------- Co-authored-by: Andy Hsu commit 69a98eaef612b58596e5c26c341b6d7cedecdf19 Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Wed Oct 11 22:01:55 2023 +0800 fix(deps): update module github.com/aliyun/aliyun-oss-go-sdk to v2.2.9+incompatible (#5141) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit 1ebc96a4e5220c979fd581bb3b5640e9436f6665 Author: Andy Hsu Date: Tue Oct 10 18:32:00 2023 +0800 fix(wopan): fatal error concurrent map writes (close #5352) commit 66e2324cac75cb3ef05af45dbdd10b124d534aff Author: Andy Hsu Date: Tue Oct 10 18:23:11 2023 +0800 chore(deps): upgrade dependencies commit 7600dc28df137c439e538b4257731c33a63db9b5 Author: Andy Hsu Date: Tue Oct 10 18:13:58 2023 +0800 fix(aliyundrive_open): change default api to raw server (close #5358) commit 8ef89ad0a496d5acc398794c0afa4f77c67ad371 Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Tue Oct 10 18:08:27 2023 +0800 fix(baidu_netdisk): hash and `error 2` (#5356) * fix(baidu):hash and error:2 * fix:invalid memory address commit 35d672217dde69e65b41b1fcd9786c1cfebcdc45 Author: jeffmingup <1960588251@qq.com> Date: Sun Oct 8 19:29:45 2023 +0800 fix(onedrive_app): incorrect api on `_accessToken` (#5346) commit 1a283bb2720eff6d1b0c1dd6f1667a6449905a9b Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Fri Oct 6 16:04:39 2023 +0800 feat(google_drive): add `hash_info`, `ctime`, `thumbnail` (#5334) commit a008f54f4d5eda5738abfd54bf1abf1e18c08430 Author: nkh0472 <67589323+nkh0472@users.noreply.github.com> Date: Thu Oct 5 13:10:51 2023 +0800 docs: minor language improvements (#5329) [skip ci] --- ...ssue_rm_working.yml => issue_on_close.yml} | 2 +- README.md | 8 +- drivers/115/driver.go | 42 +++++++++- drivers/115/meta.go | 7 +- drivers/aliyundrive_open/meta.go | 2 +- drivers/aliyundrive_open/upload.go | 3 +- drivers/baidu_netdisk/driver.go | 56 +++++++------ drivers/baidu_netdisk/types.go | 5 +- drivers/baidu_netdisk/util.go | 82 +++++++++++++++---- drivers/cloudreve/driver.go | 6 +- drivers/cloudreve/types.go | 17 ++-- drivers/cloudreve/util.go | 20 +++++ drivers/google_drive/types.go | 16 +++- drivers/google_drive/util.go | 5 +- drivers/onedrive_app/util.go | 4 +- drivers/webdav/util.go | 8 ++ go.mod | 14 ++-- go.sum | 15 ++++ internal/model/obj.go | 19 ++++- internal/stream/util.go | 8 +- pkg/gowebdav/client.go | 10 +++ pkg/utils/hash.go | 4 + pkg/utils/hash/gcid.go | 12 +-- 23 files changed, 279 insertions(+), 86 deletions(-) rename .github/workflows/{issue_rm_working.yml => issue_on_close.yml} (90%) diff --git a/.github/workflows/issue_rm_working.yml b/.github/workflows/issue_on_close.yml similarity index 90% rename from .github/workflows/issue_rm_working.yml rename to .github/workflows/issue_on_close.yml index 5618136615d..c90c2c0cb6d 100644 --- a/.github/workflows/issue_rm_working.yml +++ b/.github/workflows/issue_on_close.yml @@ -14,4 +14,4 @@ jobs: actions: 'remove-labels' token: ${{ secrets.GITHUB_TOKEN }} issue-number: ${{ github.event.issue.number }} - labels: 'working' \ No newline at end of file + labels: 'working,pr-welcome' diff --git a/README.md b/README.md index d5e88e4322e..3f8fc4eebbd 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] ## Features -- [x] Multiple storage +- [x] Multiple storages - [x] Local storage - [x] [Aliyundrive](https://www.aliyundrive.com/) - [x] OneDrive / Sharepoint ([global](https://www.office.com/), [cn](https://portal.partner.microsoftonline.cn),de,us) @@ -86,7 +86,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] Protected routes (password protection and authentication) - [x] WebDav (see https://alist.nn.ci/guide/webdav.html for details) - [x] [Docker Deploy](https://hub.docker.com/r/xhofe/alist) -- [x] Cloudflare workers proxy +- [x] Cloudflare Workers proxy - [x] File/Folder package download - [x] Web upload(Can allow visitors to upload), delete, mkdir, rename, move and copy - [x] Offline download @@ -103,7 +103,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] ## Discussion -Please go to our [discussion forum](https://github.com/Xhofe/alist/discussions) for general questions, **issues are for bug reports and feature request only.** +Please go to our [discussion forum](https://github.com/Xhofe/alist/discussions) for general questions, **issues are for bug reports and feature requests only.** ## Sponsor @@ -127,7 +127,7 @@ Thanks goes to these wonderful people: The `AList` is open-source software licensed under the AGPL-3.0 license. ## Disclaimer -- This program is a free and open source project. It is designed to share files on the network disk, which is convenient for downloading and learning golang. Please abide by relevant laws and regulations when using it, and do not abuse it; +- This program is a free and open source project. It is designed to share files on the network disk, which is convenient for downloading and learning Golang. Please abide by relevant laws and regulations when using it, and do not abuse it; - This program is implemented by calling the official sdk/interface, without destroying the official interface behavior; - This program only does 302 redirect/traffic forwarding, and does not intercept, store, or tamper with any user data; - Before using this program, you should understand and bear the corresponding risks, including but not limited to account ban, download speed limit, etc., which is none of this program's business; diff --git a/drivers/115/driver.go b/drivers/115/driver.go index d4c8f553e9c..15f6b4087b4 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -2,19 +2,22 @@ package _115 import ( "context" + "strings" + driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" - "strings" + "golang.org/x/time/rate" ) type Pan115 struct { model.Storage Addition - client *driver115.Pan115Client + client *driver115.Pan115Client + limiter *rate.Limiter } func (d *Pan115) Config() driver.Config { @@ -26,14 +29,27 @@ func (d *Pan115) GetAddition() driver.Additional { } func (d *Pan115) Init(ctx context.Context) error { + if d.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) + } return d.login() } +func (d *Pan115) WaitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + func (d *Pan115) Drop(ctx context.Context) error { return nil } func (d *Pan115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } files, err := d.getFiles(dir.GetID()) if err != nil && !errors.Is(err, driver115.ErrNotExist) { return nil, err @@ -44,6 +60,9 @@ func (d *Pan115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } downloadInfo, err := d.client. DownloadWithUA(file.(*FileObj).PickCode, driver115.UA115Browser) if err != nil { @@ -57,6 +76,9 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } if _, err := d.client.Mkdir(parentDir.GetID(), dirName); err != nil { return err } @@ -64,22 +86,38 @@ func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin } func (d *Pan115) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Move(dstDir.GetID(), srcObj.GetID()) } func (d *Pan115) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Rename(srcObj.GetID(), newName) } func (d *Pan115) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Copy(dstDir.GetID(), srcObj.GetID()) } func (d *Pan115) Remove(ctx context.Context, obj model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Delete(obj.GetID()) } func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var ( fastInfo *driver115.UploadInitResp dirID = dstDir.GetID() diff --git a/drivers/115/meta.go b/drivers/115/meta.go index cd39c2be5f0..16ec22cdab8 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` - QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` - PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` + Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` + QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` + PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` + LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` driver.RootID } diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index af4d1257583..bd69211c77e 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -11,7 +11,7 @@ type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` - OauthTokenURL string `json:"oauth_token_url" default:"https://api.xhofe.top/alist/ali_open/token"` + OauthTokenURL string `json:"oauth_token_url" default:"https://api.nn.ci/alist/ali_open/token"` ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index e4a0cf7ec0d..73c37dcfbde 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -5,7 +5,6 @@ import ( "context" "encoding/base64" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" "io" "math" "net/http" @@ -16,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" @@ -258,6 +258,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m return nil, err } offset += partSize + up(i * 100 / count) } } else { log.Debugf("[aliyundrive_open] rapid upload success, file id: %s", createResp.FileId) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 5a4716d00f6..20810a768de 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -5,7 +5,6 @@ import ( "crypto/md5" "encoding/hex" "errors" - "fmt" "io" "math" "net/url" @@ -28,10 +27,9 @@ type BaiduNetdisk struct { Addition uploadThread int + vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) } -const DefaultSliceSize int64 = 4 * utils.MB - func (d *BaiduNetdisk) Config() driver.Config { return config } @@ -54,7 +52,11 @@ func (d *BaiduNetdisk) Init(ctx context.Context) error { "method": "uinfo", }, nil) log.Debugf("[baidu] get uinfo: %s", string(res)) - return err + if err != nil { + return err + } + d.vipType = utils.Json.Get(res, "vip_type").ToInt() + return nil } func (d *BaiduNetdisk) Drop(ctx context.Context) error { @@ -153,20 +155,13 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo } streamSize := stream.GetSize() - rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) - path := encodeURIComponent(rawPath) + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() blockList, _ := utils.Json.MarshalToString([]string{contentMd5}) - data := fmt.Sprintf("path=%s&size=%d&isdir=0&rtype=3&block_list=%s&local_mtime=%d&local_ctime=%d", - path, streamSize, blockList, mtime, ctime) - params := map[string]string{ - "method": "create", - } - log.Debugf("[baidu_netdisk] precreate data: %s", data) var newFile File - _, err := d.post("/xpan/file", params, data, &newFile) + _, err := d.create(path, streamSize, 0, "", blockList, &newFile, mtime, ctime) if err != nil { return nil, err } @@ -185,17 +180,18 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } streamSize := stream.GetSize() - count := int(math.Max(math.Ceil(float64(streamSize)/float64(DefaultSliceSize)), 1)) - lastBlockSize := streamSize % DefaultSliceSize + sliceSize := d.getSliceSize() + count := int(math.Max(math.Ceil(float64(streamSize)/float64(sliceSize)), 1)) + lastBlockSize := streamSize % sliceSize if streamSize > 0 && lastBlockSize == 0 { - lastBlockSize = DefaultSliceSize + lastBlockSize = sliceSize } //cal md5 for first 256k data const SliceSize int64 = 256 * 1024 // cal md5 blockList := make([]string, 0, count) - byteSize := DefaultSliceSize + byteSize := sliceSize fileMd5H := md5.New() sliceMd5H := md5.New() sliceMd5H2 := md5.New() @@ -218,9 +214,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) blockListStr, _ := utils.Json.MarshalToString(blockList) - - rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) - path := encodeURIComponent(rawPath) + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() @@ -228,13 +222,23 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // 尝试获取之前的进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { - data := fmt.Sprintf("path=%s&size=%d&isdir=0&autoinit=1&rtype=3&block_list=%s&content-md5=%s&slice-md5=%s&local_mtime=%d&local_ctime=%d", - path, streamSize, blockListStr, contentMd5, sliceMd5, mtime, ctime) params := map[string]string{ "method": "precreate", } - log.Debugf("[baidu_netdisk] precreate data: %s", data) - _, err = d.post("/xpan/file", params, data, &precreateResp) + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + "content-md5": contentMd5, + "slice-md5": sliceMd5, + } + joinTime(form, ctime, mtime) + + log.Debugf("[baidu_netdisk] precreate data: %s", form) + _, err = d.postForm("/xpan/file", params, form, &precreateResp) if err != nil { return nil, err } @@ -257,7 +261,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F break } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*DefaultSliceSize, DefaultSliceSize + i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize if partseq+1 == count { byteSize = lastBlockSize } @@ -290,7 +294,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // step.3 创建文件 var newFile File - _, err = d.create(rawPath, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) + _, err = d.create(path, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) if err != nil { return nil, err } diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index 81a491129f0..cbec0bcfcd6 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -1,7 +1,6 @@ package baidu_netdisk import ( - "github.com/alist-org/alist/v3/pkg/utils" "path" "strconv" "time" @@ -71,7 +70,9 @@ func fileToObj(f File) *model.ObjThumb { Modified: time.Unix(f.LocalMtime, 0), Ctime: time.Unix(f.LocalCtime, 0), IsFolder: f.Isdir == 1, - HashInfo: utils.NewHashInfo(utils.MD5, f.Md5), + + // 直接获取的MD5是错误的 + // HashInfo: utils.NewHashInfo(utils.MD5, f.Md5), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index b867fbaa9c3..6c51156c22f 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -4,9 +4,7 @@ import ( "errors" "fmt" "net/http" - "net/url" "strconv" - "strings" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -96,10 +94,10 @@ func (d *BaiduNetdisk) get(pathname string, params map[string]string, resp inter }, resp) } -func (d *BaiduNetdisk) post(pathname string, params map[string]string, data interface{}, resp interface{}) ([]byte, error) { +func (d *BaiduNetdisk) postForm(pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { return d.request("https://pan.baidu.com/rest/2.0"+pathname, http.MethodPost, func(req *resty.Request) { req.SetQueryParams(params) - req.SetBody(data) + req.SetFormData(form) }, resp) } @@ -154,6 +152,9 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model //if res.StatusCode() == 302 { u = res.Header().Get("location") //} + + updateObjMd5(file, "pan.baidu.com", u) + return &model.Link{ URL: u, Header: http.Header{ @@ -176,6 +177,9 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Li if err != nil { return nil, err } + + updateObjMd5(file, d.CustomCrackUA, resp.Info[0].Dlink) + return &model.Link{ URL: resp.Info[0].Dlink, Header: http.Header{ @@ -190,29 +194,73 @@ func (d *BaiduNetdisk) manage(opera string, filelist any) ([]byte, error) { "opera": opera, } marshal, _ := utils.Json.MarshalToString(filelist) - data := fmt.Sprintf("async=0&filelist=%s&ondup=fail", marshal) - return d.post("/xpan/file", params, data, nil) + return d.postForm("/xpan/file", params, map[string]string{ + "async": "0", + "filelist": marshal, + "ondup": "fail", + }, nil) } func (d *BaiduNetdisk) create(path string, size int64, isdir int, uploadid, block_list string, resp any, mtime, ctime int64) ([]byte, error) { params := map[string]string{ "method": "create", } - data := "" - if mtime == 0 || ctime == 0 { - data = fmt.Sprintf("path=%s&size=%d&isdir=%d&rtype=3", encodeURIComponent(path), size, isdir) - } else { - data = fmt.Sprintf("path=%s&size=%d&isdir=%d&rtype=3&local_mtime=%d&local_ctime=%d", encodeURIComponent(path), size, isdir, mtime, ctime) + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(size, 10), + "isdir": strconv.Itoa(isdir), + "rtype": "3", + } + if mtime != 0 && ctime != 0 { + joinTime(form, ctime, mtime) } if uploadid != "" { - data += fmt.Sprintf("&uploadid=%s&block_list=%s", uploadid, block_list) + form["uploadid"] = uploadid } - return d.post("/xpan/file", params, data, resp) + if block_list != "" { + form["block_list"] = block_list + } + return d.postForm("/xpan/file", params, form, resp) } -func encodeURIComponent(str string) string { - r := url.QueryEscape(str) - r = strings.ReplaceAll(r, "+", "%20") - return r +func joinTime(form map[string]string, ctime, mtime int64) { + form["local_mtime"] = strconv.FormatInt(mtime, 10) + form["local_ctime"] = strconv.FormatInt(ctime, 10) } + +func updateObjMd5(obj model.Obj, userAgent, u string) { + object := model.GetRawObject(obj) + if object != nil { + req, _ := http.NewRequest(http.MethodHead, u, nil) + req.Header.Add("User-Agent", userAgent) + resp, _ := base.HttpClient.Do(req) + if resp != nil { + contentMd5 := resp.Header.Get("Content-Md5") + object.HashInfo = utils.NewHashInfo(utils.MD5, contentMd5) + } + } +} + +const ( + DefaultSliceSize int64 = 4 * utils.MB + VipSliceSize = 16 * utils.MB + SVipSliceSize = 32 * utils.MB +) + +func (d *BaiduNetdisk) getSliceSize() int64 { + switch d.vipType { + case 1: + return VipSliceSize + case 2: + return SVipSliceSize + default: + return DefaultSliceSize + } +} + +// func encodeURIComponent(str string) string { +// r := url.QueryEscape(str) +// r = strings.ReplaceAll(r, "+", "%20") +// return r +// } diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 030de7c279d..2a22380e314 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -49,7 +49,11 @@ func (d *Cloudreve) List(ctx context.Context, dir model.Obj, args model.ListArgs } return utils.SliceConvert(r.Objects, func(src Object) (model.Obj, error) { - return objectToObj(src), nil + thumb, err := d.GetThumb(src) + if err != nil { + return nil, err + } + return objectToObj(src, thumb), nil }) } diff --git a/drivers/cloudreve/types.go b/drivers/cloudreve/types.go index 114afd3cfaa..e25673829a8 100644 --- a/drivers/cloudreve/types.go +++ b/drivers/cloudreve/types.go @@ -44,13 +44,16 @@ type Object struct { SourceEnabled bool `json:"source_enabled"` } -func objectToObj(f Object) *model.Object { - return &model.Object{ - ID: f.Id, - Name: f.Name, - Size: int64(f.Size), - Modified: f.Date, - IsFolder: f.Type == "dir", +func objectToObj(f Object, t model.Thumbnail) *model.ObjThumb { + return &model.ObjThumb{ + Object: model.Object{ + ID: f.Id, + Name: f.Name, + Size: int64(f.Size), + Modified: f.Date, + IsFolder: f.Type == "dir", + }, + Thumbnail: t, } } diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index 1b8778be945..ed8794667c1 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -149,3 +149,23 @@ func convertSrc(obj model.Obj) map[string]interface{} { m["items"] = items return m } + +func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { + ua := d.CustomUA + if ua == "" { + ua = base.UserAgent + } + req := base.NoRedirectClient.R() + req.SetHeaders(map[string]string{ + "Cookie": "cloudreve-session=" + d.Cookie, + "Accept": "image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + "User-Agent": ua, + }) + resp, err := req.Execute(http.MethodGet, d.Address+"/api/v3/file/thumb/"+file.Id) + if err != nil { + return model.Thumbnail{}, err + } + return model.Thumbnail{ + Thumbnail: resp.Header().Get("Location"), + }, nil +} diff --git a/drivers/google_drive/types.go b/drivers/google_drive/types.go index 796c13218c7..075459327d9 100644 --- a/drivers/google_drive/types.go +++ b/drivers/google_drive/types.go @@ -5,6 +5,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" log "github.com/sirupsen/logrus" ) @@ -23,12 +24,17 @@ type File struct { Name string `json:"name"` MimeType string `json:"mimeType"` ModifiedTime time.Time `json:"modifiedTime"` + CreatedTime time.Time `json:"createdTime"` Size string `json:"size"` ThumbnailLink string `json:"thumbnailLink"` ShortcutDetails struct { TargetId string `json:"targetId"` TargetMimeType string `json:"targetMimeType"` } `json:"shortcutDetails"` + + MD5Checksum string `json:"md5Checksum"` + SHA1Checksum string `json:"sha1Checksum"` + SHA256Checksum string `json:"sha256Checksum"` } func fileToObj(f File) *model.ObjThumb { @@ -39,10 +45,18 @@ func fileToObj(f File) *model.ObjThumb { ID: f.Id, Name: f.Name, Size: size, + Ctime: f.CreatedTime, Modified: f.ModifiedTime, IsFolder: f.MimeType == "application/vnd.google-apps.folder", + HashInfo: utils.NewHashInfoByMap(map[*utils.HashType]string{ + utils.MD5: f.MD5Checksum, + utils.SHA1: f.SHA1Checksum, + utils.SHA256: f.SHA256Checksum, + }), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: f.ThumbnailLink, }, - Thumbnail: model.Thumbnail{}, } if f.MimeType == "application/vnd.google-apps.shortcut" { obj.ID = f.ShortcutDetails.TargetId diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 5637d00e71d..2c1f13eb8a5 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -5,7 +5,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" "io/ioutil" "net/http" "os" @@ -13,6 +12,8 @@ import ( "strconv" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" @@ -195,7 +196,7 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { } query := map[string]string{ "orderBy": orderBy, - "fields": "files(id,name,mimeType,size,modifiedTime,thumbnailLink,shortcutDetails),nextPageToken", + "fields": "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken", "pageSize": "1000", "q": fmt.Sprintf("'%s' in parents and trashed = false", id), //"includeItemsFromAllDrives": "true", diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index 6a061f1f3bc..28b34837806 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -71,8 +71,8 @@ func (d *OnedriveAPP) _accessToken() error { "grant_type": "client_credentials", "client_id": d.ClientID, "client_secret": d.ClientSecret, - "resource": "https://graph.microsoft.com/", - "scope": "https://graph.microsoft.com/.default", + "resource": onedriveHostMap[d.Region].Api + "/", + "scope": onedriveHostMap[d.Region].Api + "/.default", }).Post(url) if err != nil { return err diff --git a/drivers/webdav/util.go b/drivers/webdav/util.go index 92557c4f553..84eebb2ec3c 100644 --- a/drivers/webdav/util.go +++ b/drivers/webdav/util.go @@ -2,6 +2,7 @@ package webdav import ( "net/http" + "net/http/cookiejar" "github.com/alist-org/alist/v3/drivers/webdav/odrvcookie" "github.com/alist-org/alist/v3/internal/model" @@ -26,6 +27,13 @@ func (d *WebDav) setClient() error { } else { return err } + } else { + cookieJar, err := cookiejar.New(nil) + if err == nil { + c.SetJar(cookieJar) + } else { + return err + } } d.client = c return nil diff --git a/go.mod b/go.mod index 7f066e1c27a..f185aef9b1e 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/SheltonZhu/115driver v1.0.16 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 - github.com/Xhofe/wopan-sdk-go v0.1.1 - github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible + github.com/Xhofe/wopan-sdk-go v0.1.2 + github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.44.327 github.com/blevesearch/bleve/v2 v2.3.10 @@ -24,7 +24,7 @@ require ( github.com/foxxorcat/weiyun-sdk-go v0.1.2 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/go-resty/resty/v2 v2.8.0 + github.com/go-resty/resty/v2 v2.9.1 github.com/go-webauthn/webauthn v0.8.6 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.3.1 @@ -47,10 +47,10 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 - golang.org/x/crypto v0.13.0 + golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/image v0.11.0 - golang.org/x/net v0.15.0 + golang.org/x/net v0.16.0 golang.org/x/oauth2 v0.12.0 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 @@ -186,8 +186,8 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.12.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/api v0.134.0 // indirect diff --git a/go.sum b/go.sum index 49cdc805e4f..0fb9a131f25 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaV github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4/go.mod h1:8pWlL2rpusvx7Xa6yYaIWOJ8bR3gPdFBUT7OystyGOY= github.com/Xhofe/wopan-sdk-go v0.1.1 h1:dSrTxNYclqNuo9libjtC+R6C4RCen/inh/dUXd12vpM= github.com/Xhofe/wopan-sdk-go v0.1.1/go.mod h1:xWcUS7PoFLDD9gy2BK2VQfilEsZngLMz2Vkx3oF2zJY= +github.com/Xhofe/wopan-sdk-go v0.1.2 h1:6Gh4YTT7b7YHN0OoJ33j7Jm9ru/ckuvcDxPnRmH07jc= +github.com/Xhofe/wopan-sdk-go v0.1.2/go.mod h1:ktLYb4t7rnPFq1AshLaPXq5kZER+DkEagT6/i/in0uo= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= @@ -27,6 +29,8 @@ github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible h1:QoRMR0TCctLDqBCMyOu1e github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiwbXTpUEinBpHsN7mG21Rc2k= +github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.1.0 h1:eitZxTPY1krUsxinsng3Qvt/Ud7q/aQmmYRh8p4hyPw= github.com/andreburgaud/crypt2go v1.1.0/go.mod h1:4qhZPzarj1dCIRmCkpdgCklwp+hBq9yEt0zPe9Ayuhc= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= @@ -184,6 +188,8 @@ github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPr github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-resty/resty/v2 v2.8.0 h1:J29d0JFWwSWrDCysnOK/YjsPMLQTx0TvgJEHVGvf2L8= github.com/go-resty/resty/v2 v2.8.0/go.mod h1:UCui0cMHekLrSntoMyofdSTaPpinlRHFtPpizuyDW2w= +github.com/go-resty/resty/v2 v2.9.1 h1:PIgGx4VrHvag0juCJ4dDv3MiFRlDmP0vicBucwf+gLM= +github.com/go-resty/resty/v2 v2.9.1/go.mod h1:4/GYJVjh9nhkhGR6AUNW3XhpDYNUr+Uvy9gV/VGZIy4= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.8.6 h1:bKMtL1qzd2WTFkf1mFTVbreYrwn7dsYmEPjTq6QN90E= @@ -506,6 +512,8 @@ golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= @@ -536,6 +544,8 @@ golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= @@ -573,6 +583,8 @@ golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -584,6 +596,8 @@ golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -599,6 +613,7 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/model/obj.go b/internal/model/obj.go index cb46201fe70..77c0700a35c 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -1,14 +1,15 @@ package model import ( - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/alist-org/alist/v3/pkg/utils" "io" "regexp" "sort" "strings" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + mapset "github.com/deckarep/golang-set/v2" "github.com/maruel/natural" @@ -146,6 +147,20 @@ func GetUrl(obj Obj) (url string, ok bool) { return url, false } +func GetRawObject(obj Obj) *Object { + switch v := obj.(type) { + case *ObjThumbURL: + return &v.Object + case *ObjThumb: + return &v.Object + case *ObjectURL: + return &v.Object + case *Object: + return v + } + return nil +} + // Merge func NewObjMerge() *ObjMerge { return &ObjMerge{ diff --git a/internal/stream/util.go b/internal/stream/util.go index 2cb13daa1cf..7d2b7ef7509 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -3,13 +3,14 @@ package stream import ( "context" "fmt" + "io" + "net/http" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" "github.com/alist-org/alist/v3/pkg/http_range" log "github.com/sirupsen/logrus" - "io" - "net/http" ) func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCloserIF, error) { @@ -40,6 +41,9 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl if len(link.URL) > 0 { response, err := RequestRangedHttp(ctx, link, r.Start, r.Length) if err != nil { + if response == nil { + return nil, fmt.Errorf("http request failure, err:%s", err) + } return nil, fmt.Errorf("http request failure,status: %d err:%s", response.StatusCode, err) } if r.Start == 0 && (r.Length == -1 || r.Length == size) || response.StatusCode == http.StatusPartialContent || diff --git a/pkg/gowebdav/client.go b/pkg/gowebdav/client.go index 6e12289c1ac..2fca0b7f43d 100644 --- a/pkg/gowebdav/client.go +++ b/pkg/gowebdav/client.go @@ -83,6 +83,11 @@ func (c *Client) SetTransport(transport http.RoundTripper) { c.c.Transport = transport } +// SetJar exposes the ability to set a cookie jar to the client. +func (c *Client) SetJar(jar http.CookieJar) { + c.c.Jar = jar +} + // Connect connects to our dav server func (c *Client) Connect() error { rs, err := c.options("/") @@ -351,6 +356,11 @@ func (c *Client) Link(path string) (string, http.Header, error) { return "", nil, newPathErrorErr("Link", path, err) } + if c.c.Jar != nil { + for _, cookie := range c.c.Jar.Cookies(r.URL) { + r.AddCookie(cookie) + } + } for k, vals := range c.headers { for _, v := range vals { r.Header.Add(k, v) diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index bbb8769bcb2..8f8aaa26781 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -184,6 +184,10 @@ type HashInfo struct { h map[*HashType]string `json:"hashInfo"` } +func NewHashInfoByMap(h map[*HashType]string) HashInfo { + return HashInfo{h} +} + func NewHashInfo(ht *HashType, str string) HashInfo { m := make(map[*HashType]string) if ht != nil { diff --git a/pkg/utils/hash/gcid.go b/pkg/utils/hash/gcid.go index 8e1b98d6f4e..f6eccef7aea 100644 --- a/pkg/utils/hash/gcid.go +++ b/pkg/utils/hash/gcid.go @@ -72,11 +72,13 @@ func (h *gcid) Write(p []byte) (n int, err error) { } func (h *gcid) Sum(b []byte) []byte { - if hashm, ok := h.hash.(encoding.BinaryMarshaler); ok { - if hashum, ok := h.hash.(encoding.BinaryUnmarshaler); ok { - tempData, _ := hashm.MarshalBinary() - h.hash.Write(h.hashState.Sum(nil)) - defer hashum.UnmarshalBinary(tempData) + if h.offset != 0 { + if hashm, ok := h.hash.(encoding.BinaryMarshaler); ok { + if hashum, ok := h.hash.(encoding.BinaryUnmarshaler); ok { + tempData, _ := hashm.MarshalBinary() + defer hashum.UnmarshalBinary(tempData) + h.hash.Write(h.hashState.Sum(nil)) + } } } return h.hash.Sum(b) From aba8bc0ec2c12d713d7da00497728739c9c75c59 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 20 Oct 2023 21:15:37 +0800 Subject: [PATCH 09/10] fix: adapt update progress type --- drivers/aliyundrive_open/upload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index 73c37dcfbde..3b224e7d225 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -258,7 +258,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m return nil, err } offset += partSize - up(i * 100 / count) + up(float64(i*100) / float64(count)) } } else { log.Debugf("[aliyundrive_open] rapid upload success, file id: %s", createResp.FileId) From 3b90f591b5d096c609f652819566b3eedd7e279d Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 5 Nov 2023 22:28:31 +0800 Subject: [PATCH 10/10] Squashed commit of the following: commit 65c5ec0c34d5f027a65933fe89af53791747bdd4 Author: itsHenry <2671230065@qq.com> Date: Sat Nov 4 13:35:09 2023 +0800 feat(cloudreve): folder size count and switch (#5457 close #5395) commit a6325967d0de18e6b6c744f06cb1ebaa08ec687e Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Mon Oct 30 15:11:20 2023 +0800 fix(deps): update module github.com/charmbracelet/lipgloss to v0.9.1 (#5234) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit 4dff49470adce36416d8c56594e84868c04d023b Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Mon Oct 30 15:10:36 2023 +0800 fix(deps): update golang.org/x/exp digest to 7918f67 (#5366) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit cc86d6f3d1ff2120669c9dda719b7faabb922f52 Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Sun Oct 29 14:45:55 2023 +0800 fix(deps): update module golang.org/x/net to v0.17.0 [security] (#5370) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit c0f9c8ebafdf8dd2afe5c0b9fba24456819c3155 Author: Andy Hsu Date: Thu Oct 26 19:21:09 2023 +0800 feat: add ignore direct link params (close #5434) --- drivers/cloudreve/driver.go | 8 ++++++++ drivers/cloudreve/meta.go | 11 ++++++----- drivers/cloudreve/types.go | 4 ++++ drivers/cloudreve/util.go | 3 +++ go.mod | 12 ++++++------ go.sum | 10 ++++++++++ internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 1 + internal/op/hook.go | 4 ++++ server/handles/down.go | 8 ++++++-- 10 files changed, 49 insertions(+), 13 deletions(-) diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 2a22380e314..49c2d5f00f2 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -53,6 +53,14 @@ func (d *Cloudreve) List(ctx context.Context, dir model.Obj, args model.ListArgs if err != nil { return nil, err } + if src.Type == "dir" && d.EnableThumbAndFolderSize { + var dprop DirectoryProp + err = d.request(http.MethodGet, "/object/property/"+src.Id+"?is_folder=true", nil, &dprop) + if err != nil { + return nil, err + } + src.Size = dprop.Size + } return objectToObj(src, thumb), nil }) } diff --git a/drivers/cloudreve/meta.go b/drivers/cloudreve/meta.go index d01d54791d3..92c0b9fb1d7 100644 --- a/drivers/cloudreve/meta.go +++ b/drivers/cloudreve/meta.go @@ -9,11 +9,12 @@ type Addition struct { // Usually one of two driver.RootPath // define other - Address string `json:"address" required:"true"` - Username string `json:"username"` - Password string `json:"password"` - Cookie string `json:"cookie"` - CustomUA string `json:"custom_ua"` + Address string `json:"address" required:"true"` + Username string `json:"username"` + Password string `json:"password"` + Cookie string `json:"cookie"` + CustomUA string `json:"custom_ua"` + EnableThumbAndFolderSize bool `json:"enable_thumb_and_folder_size"` } var config = driver.Config{ diff --git a/drivers/cloudreve/types.go b/drivers/cloudreve/types.go index e25673829a8..241d993ebb8 100644 --- a/drivers/cloudreve/types.go +++ b/drivers/cloudreve/types.go @@ -44,6 +44,10 @@ type Object struct { SourceEnabled bool `json:"source_enabled"` } +type DirectoryProp struct { + Size int `json:"size"` +} + func objectToObj(f Object, t model.Thumbnail) *model.ObjThumb { return &model.ObjThumb{ Object: model.Object{ diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index ed8794667c1..284e3289dee 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -151,6 +151,9 @@ func convertSrc(obj model.Obj) map[string]interface{} { } func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { + if !d.Addition.EnableThumbAndFolderSize { + return model.Thumbnail{}, nil + } ua := d.CustomUA if ua == "" { ua = base.UserAgent diff --git a/go.mod b/go.mod index f185aef9b1e..5a6ed9b2390 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.16.1 github.com/charmbracelet/bubbletea v0.24.2 - github.com/charmbracelet/lipgloss v0.7.1 + github.com/charmbracelet/lipgloss v0.9.1 github.com/coreos/go-oidc v2.2.1+incompatible github.com/deckarep/golang-set/v2 v2.3.1 github.com/disintegration/imaging v1.6.2 @@ -48,10 +48,11 @@ require ( github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 golang.org/x/crypto v0.14.0 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/image v0.11.0 - golang.org/x/net v0.16.0 + golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.12.0 + golang.org/x/time v0.3.0 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 gorm.io/driver/sqlite v1.4.4 @@ -137,7 +138,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.0 // indirect @@ -150,7 +151,7 @@ require ( github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/reflow v0.3.0 // indirect - github.com/muesli/termenv v0.15.1 // indirect + github.com/muesli/termenv v0.15.2 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr v0.9.0 // indirect @@ -189,7 +190,6 @@ require ( golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect google.golang.org/api v0.134.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect diff --git a/go.sum b/go.sum index 0fb9a131f25..940eed1d6ed 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,8 @@ github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06 github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E= github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= +github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= +github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= @@ -320,6 +322,8 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -349,6 +353,8 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= @@ -520,6 +526,8 @@ golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= @@ -546,6 +554,8 @@ golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index f5ca599ccd4..21a432ddebd 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -140,6 +140,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.OcrApi, Value: "https://api.nn.ci/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL}, {Key: conf.ForwardDirectLinkParams, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL}, + {Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, // single settings diff --git a/internal/conf/const.go b/internal/conf/const.go index 02a00060a68..eb70602ad3f 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -41,6 +41,7 @@ const ( OcrApi = "ocr_api" FilenameCharMapping = "filename_char_mapping" ForwardDirectLinkParams = "forward_direct_link_params" + IgnoreDirectLinkParams = "ignore_direct_link_params" WebauthnLoginEnabled = "webauthn_login_enabled" // index diff --git a/internal/op/hook.go b/internal/op/hook.go index e37e52df269..23b8e59af2c 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -78,6 +78,10 @@ var settingItemHooks = map[string]SettingItemHook{ log.Debugf("filename char mapping: %+v", conf.FilenameCharMap) return nil }, + conf.IgnoreDirectLinkParams: func(item *model.SettingItem) error { + conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",") + return nil + }, } func RegisterSettingItemHook(key string, hook SettingItemHook) { diff --git a/server/handles/down.go b/server/handles/down.go index e4aec494243..d3d41e85a2b 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -52,7 +52,9 @@ func Down(c *gin.Context) { c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") if setting.GetBool(conf.ForwardDirectLinkParams) { query := c.Request.URL.Query() - query.Del("sign") + for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { + query.Del(v) + } link.URL, err = utils.InjectQuery(link.URL, query) if err != nil { common.ErrorResp(c, err, 500) @@ -95,7 +97,9 @@ func Proxy(c *gin.Context) { } if link.URL != "" && setting.GetBool(conf.ForwardDirectLinkParams) { query := c.Request.URL.Query() - query.Del("sign") + for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { + query.Del(v) + } link.URL, err = utils.InjectQuery(link.URL, query) if err != nil { common.ErrorResp(c, err, 500)