From ffa03bfda11aa18bb899afc1f29e8690fcea1036 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sat, 24 May 2025 13:38:43 +0800 Subject: [PATCH 001/133] feat(cloudreve_v4): add Cloudreve V4 driver (#8470 closes #8328 #8467) * feat(cloudreve_v4): add Cloudreve V4 driver implementation * fix(cloudreve_v4): update request handling to prevent token refresh loop * feat(onedrive): implement retry logic for upload failures * feat(cloudreve): implement retry logic for upload failures * feat(cloudreve_v4): support cloud sorting * fix(cloudreve_v4): improve token handling in Init method * feat(cloudreve_v4): support share * feat(cloudreve): support reference * feat(cloudreve_v4): support version upload * fix(cloudreve_v4): add SetBody in upLocal * fix(cloudreve_v4): update URL structure in Link and FileUrlResp --- drivers/all.go | 1 + drivers/cloudreve/driver.go | 11 + drivers/cloudreve/util.go | 157 ++++++++--- drivers/cloudreve_v4/driver.go | 305 +++++++++++++++++++++ drivers/cloudreve_v4/meta.go | 44 +++ drivers/cloudreve_v4/types.go | 164 ++++++++++++ drivers/cloudreve_v4/util.go | 476 +++++++++++++++++++++++++++++++++ drivers/onedrive/util.go | 33 ++- drivers/onedrive_app/util.go | 33 ++- 9 files changed, 1158 insertions(+), 66 deletions(-) create mode 100644 drivers/cloudreve_v4/driver.go create mode 100644 drivers/cloudreve_v4/meta.go create mode 100644 drivers/cloudreve_v4/types.go create mode 100644 drivers/cloudreve_v4/util.go diff --git a/drivers/all.go b/drivers/all.go index 0b8ce3aa61d..224fb8ddb4b 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -22,6 +22,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_share" _ "github.com/alist-org/alist/v3/drivers/chaoxing" _ "github.com/alist-org/alist/v3/drivers/cloudreve" + _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" _ "github.com/alist-org/alist/v3/drivers/crypt" _ "github.com/alist-org/alist/v3/drivers/doubao" _ "github.com/alist-org/alist/v3/drivers/doubao_share" diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 8c2321b8f40..dcde58c638d 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -18,6 +18,7 @@ import ( type Cloudreve struct { model.Storage Addition + ref *Cloudreve } func (d *Cloudreve) Config() driver.Config { @@ -37,8 +38,18 @@ func (d *Cloudreve) Init(ctx context.Context) error { return d.login() } +func (d *Cloudreve) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*Cloudreve) + if ok { + d.ref = refStorage + return nil + } + return errs.NotSupport +} + func (d *Cloudreve) Drop(ctx context.Context) error { d.Cookie = "" + d.ref = nil return nil } diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index 196d7303337..5054de6cb56 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "encoding/base64" + "encoding/json" "errors" "fmt" "io" "net/http" "strconv" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" @@ -19,7 +21,6 @@ import ( "github.com/alist-org/alist/v3/pkg/cookie" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" - json "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go" ) @@ -35,6 +36,9 @@ func (d *Cloudreve) getUA() string { } func (d *Cloudreve) request(method string, path string, callback base.ReqCallback, out interface{}) error { + if d.ref != nil { + return d.ref.request(method, path, callback, out) + } u := d.Address + "/api/v3" + path req := base.RestyClient.R() req.SetHeaders(map[string]string{ @@ -79,11 +83,11 @@ func (d *Cloudreve) request(method string, path string, callback base.ReqCallbac } if out != nil && r.Data != nil { var marshal []byte - marshal, err = json.Marshal(r.Data) + marshal, err = jsoniter.Marshal(r.Data) if err != nil { return err } - err = json.Unmarshal(marshal, out) + err = jsoniter.Unmarshal(marshal, out) if err != nil { return err } @@ -187,12 +191,9 @@ func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u Up if utils.IsCanceled(ctx) { return ctx.Err() } - utils.Log.Debugf("[Cloudreve-Local] upload: %d", finish) - var byteSize = DEFAULT left := stream.GetSize() - finish - if left < DEFAULT { - byteSize = left - } + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Cloudreve-Local] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) utils.Log.Debug(err, n) @@ -205,9 +206,26 @@ func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u Up req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) req.SetHeader("User-Agent", d.getUA()) req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + req.AddRetryCondition(func(r *resty.Response, err error) bool { + if err != nil { + return true + } + if r.IsError() { + return true + } + var retryResp Resp + jErr := base.RestyClient.JSONUnmarshal(r.Body(), &retryResp) + if jErr != nil { + return true + } + if retryResp.Code != 0 { + return true + } + return false + }) }, nil) if err != nil { - break + return err } finish += byteSize up(float64(finish) * 100 / float64(stream.GetSize())) @@ -222,16 +240,15 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U var finish int64 = 0 var chunk int = 0 DEFAULT := int64(u.ChunkSize) + retryCount := 0 + maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } - utils.Log.Debugf("[Cloudreve-Remote] upload: %d", finish) - var byteSize = DEFAULT left := stream.GetSize() - finish - if left < DEFAULT { - byteSize = left - } + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Cloudreve-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) utils.Log.Debug(err, n) @@ -248,14 +265,43 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Authorization", fmt.Sprint(credential)) req.Header.Set("User-Agent", d.getUA()) - finish += byteSize - res, err := base.HttpClient.Do(req) - if err != nil { - return err + err = func() error { + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return errors.New(res.Status) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }() + if err == nil { + retryCount = 0 + finish += byteSize + up(float64(finish) * 100 / float64(stream.GetSize())) + chunk++ + } else { + retryCount++ + if retryCount > maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< 0 { + src.Size = ds.FolderSummary.Size + } + } + var thumb model.Thumbnail + if d.EnableThumb && src.Type == 0 { + var t FileThumbResp + err := d.request(http.MethodGet, "/file/thumb", func(req *resty.Request) { + req.SetQueryParam("uri", src.Path) + }, &t) + if err == nil && t.URL != "" { + thumb = model.Thumbnail{ + Thumbnail: t.URL, + } + } + } + return &model.ObjThumb{ + Object: model.Object{ + ID: src.ID, + Path: src.Path, + Name: src.Name, + Size: src.Size, + Modified: src.UpdatedAt, + Ctime: src.CreatedAt, + IsFolder: src.Type == 1, + }, + Thumbnail: thumb, + }, nil + }) +} + +func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var url FileUrlResp + err := d.request(http.MethodPost, "/file/url", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{file.GetPath()}, + "download": true, + }) + }, &url) + if err != nil { + return nil, err + } + if len(url.Urls) == 0 { + return nil, errors.New("server returns no url") + } + exp := time.Until(url.Expires) + return &model.Link{ + URL: url.Urls[0].URL, + Expiration: &exp, + }, nil +} + +func (d *CloudreveV4) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "type": "folder", + "uri": parentDir.GetPath() + "/" + dirName, + "error_on_conflict": true, + }) + }, nil) +} + +func (d *CloudreveV4) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.request(http.MethodPost, "/file/move", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{srcObj.GetPath()}, + "dst": dstDir.GetPath(), + "copy": false, + }) + }, nil) +} + +func (d *CloudreveV4) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "new_name": newName, + "uri": srcObj.GetPath(), + }) + }, nil) + +} + +func (d *CloudreveV4) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.request(http.MethodPost, "/file/move", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{srcObj.GetPath()}, + "dst": dstDir.GetPath(), + "copy": true, + }) + }, nil) +} + +func (d *CloudreveV4) Remove(ctx context.Context, obj model.Obj) error { + return d.request(http.MethodDelete, "/file", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{obj.GetPath()}, + "unlink": false, + "skip_soft_delete": true, + }) + }, nil) +} + +func (d *CloudreveV4) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if file.GetSize() == 0 { + // 空文件使用新建文件方法，避免上传卡锁 + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "type": "file", + "uri": dstDir.GetPath() + "/" + file.GetName(), + "error_on_conflict": true, + }) + }, nil) + } + var p StoragePolicy + var r FileResp + var u FileUploadResp + var err error + params := map[string]string{ + "page_size": "10", + "uri": dstDir.GetPath(), + "order_by": "created_at", + "order_direction": "asc", + "page": "0", + } + err = d.request(http.MethodGet, "/file", func(req *resty.Request) { + req.SetQueryParams(params) + }, &r) + if err != nil { + return err + } + p = r.StoragePolicy + body := base.Json{ + "uri": dstDir.GetPath() + "/" + file.GetName(), + "size": file.GetSize(), + "policy_id": p.ID, + "last_modified": file.ModTime().UnixMilli(), + "mime_type": "", + } + if d.EnableVersionUpload { + body["entity_type"] = "version" + } + err = d.request(http.MethodPut, "/file/upload", func(req *resty.Request) { + req.SetBody(body) + }, &u) + if err != nil { + return err + } + if u.StoragePolicy.Relay { + err = d.upLocal(ctx, file, u, up) + } else { + switch u.StoragePolicy.Type { + case "local": + err = d.upLocal(ctx, file, u, up) + case "remote": + err = d.upRemote(ctx, file, u, up) + case "onedrive": + err = d.upOneDrive(ctx, file, u, up) + case "s3": + err = d.upS3(ctx, file, u, up) + default: + return errs.NotImplement + } + } + if err != nil { + // 删除失败的会话 + _ = d.request(http.MethodDelete, "/file/upload", func(req *resty.Request) { + req.SetBody(base.Json{ + "id": u.SessionID, + "uri": u.URI, + }) + }, nil) + return err + } + return nil +} + +func (d *CloudreveV4) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *CloudreveV4) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*CloudreveV4)(nil) diff --git a/drivers/cloudreve_v4/meta.go b/drivers/cloudreve_v4/meta.go new file mode 100644 index 00000000000..bfaa14f81e4 --- /dev/null +++ b/drivers/cloudreve_v4/meta.go @@ -0,0 +1,44 @@ +package cloudreve_v4 + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // driver.RootID + // define other + Address string `json:"address" required:"true"` + Username string `json:"username"` + Password string `json:"password"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + CustomUA string `json:"custom_ua"` + EnableFolderSize bool `json:"enable_folder_size"` + EnableThumb bool `json:"enable_thumb"` + EnableVersionUpload bool `json:"enable_version_upload"` + OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at" default:"name" required:"true"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc" required:"true"` +} + +var config = driver.Config{ + Name: "Cloudreve V4", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "cloudreve://my", + CheckStatus: true, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &CloudreveV4{} + }) +} diff --git a/drivers/cloudreve_v4/types.go b/drivers/cloudreve_v4/types.go new file mode 100644 index 00000000000..e81226d3da5 --- /dev/null +++ b/drivers/cloudreve_v4/types.go @@ -0,0 +1,164 @@ +package cloudreve_v4 + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Object struct { + model.Object + StoragePolicy StoragePolicy +} + +type Resp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data any `json:"data"` +} + +type BasicConfigResp struct { + InstanceID string `json:"instance_id"` + // Title string `json:"title"` + // Themes string `json:"themes"` + // DefaultTheme string `json:"default_theme"` + User struct { + ID string `json:"id"` + // Nickname string `json:"nickname"` + // CreatedAt time.Time `json:"created_at"` + // Anonymous bool `json:"anonymous"` + Group struct { + ID string `json:"id"` + Name string `json:"name"` + Permission string `json:"permission"` + } `json:"group"` + } `json:"user"` + // Logo string `json:"logo"` + // LogoLight string `json:"logo_light"` + // CaptchaReCaptchaKey string `json:"captcha_ReCaptchaKey"` + CaptchaType string `json:"captcha_type"` // support 'normal' only + // AppPromotion bool `json:"app_promotion"` +} + +type SiteLoginConfigResp struct { + LoginCaptcha bool `json:"login_captcha"` + Authn bool `json:"authn"` +} + +type PrepareLoginResp struct { + WebauthnEnabled bool `json:"webauthn_enabled"` + PasswordEnabled bool `json:"password_enabled"` +} + +type CaptchaResp struct { + Image string `json:"image"` + Ticket string `json:"ticket"` +} + +type Token struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + AccessExpires time.Time `json:"access_expires"` + RefreshExpires time.Time `json:"refresh_expires"` +} + +type TokenResponse struct { + User struct { + ID string `json:"id"` + // Email string `json:"email"` + // Nickname string `json:"nickname"` + Status string `json:"status"` + // CreatedAt time.Time `json:"created_at"` + Group struct { + ID string `json:"id"` + Name string `json:"name"` + Permission string `json:"permission"` + // DirectLinkBatchSize int `json:"direct_link_batch_size"` + // TrashRetention int `json:"trash_retention"` + } `json:"group"` + // Language string `json:"language"` + } `json:"user"` + Token Token `json:"token"` +} + +type File struct { + Type int `json:"type"` // 0: file, 1: folder + ID string `json:"id"` + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Size int64 `json:"size"` + Metadata interface{} `json:"metadata"` + Path string `json:"path"` + Capability string `json:"capability"` + Owned bool `json:"owned"` + PrimaryEntity string `json:"primary_entity"` +} + +type StoragePolicy struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + MaxSize int64 `json:"max_size"` + Relay bool `json:"relay,omitempty"` +} + +type Pagination struct { + Page int `json:"page"` + PageSize int `json:"page_size"` + IsCursor bool `json:"is_cursor"` + NextToken string `json:"next_token,omitempty"` +} + +type Props struct { + Capability string `json:"capability"` + MaxPageSize int `json:"max_page_size"` + OrderByOptions []string `json:"order_by_options"` + OrderDirectionOptions []string `json:"order_direction_options"` +} + +type FileResp struct { + Files []File `json:"files"` + Parent File `json:"parent"` + Pagination Pagination `json:"pagination"` + Props Props `json:"props"` + ContextHint string `json:"context_hint"` + MixedType bool `json:"mixed_type"` + StoragePolicy StoragePolicy `json:"storage_policy"` +} + +type FileUrlResp struct { + Urls []struct { + URL string `json:"url"` + } `json:"urls"` + Expires time.Time `json:"expires"` +} + +type FileUploadResp struct { + // UploadID string `json:"upload_id"` + SessionID string `json:"session_id"` + ChunkSize int64 `json:"chunk_size"` + Expires int64 `json:"expires"` + StoragePolicy StoragePolicy `json:"storage_policy"` + URI string `json:"uri"` + CompleteURL string `json:"completeURL,omitempty"` // for S3-like + CallbackSecret string `json:"callback_secret,omitempty"` // for S3-like, OneDrive + UploadUrls []string `json:"upload_urls,omitempty"` // for not-local + Credential string `json:"credential,omitempty"` // for local +} + +type FileThumbResp struct { + URL string `json:"url"` + Expires time.Time `json:"expires"` +} + +type FolderSummaryResp struct { + File + FolderSummary struct { + Size int64 `json:"size"` + Files int64 `json:"files"` + Folders int64 `json:"folders"` + Completed bool `json:"completed"` + CalculatedAt time.Time `json:"calculated_at"` + } `json:"folder_summary"` +} diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go new file mode 100644 index 00000000000..cf2337f279b --- /dev/null +++ b/drivers/cloudreve_v4/util.go @@ -0,0 +1,476 @@ +package cloudreve_v4 + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + jsoniter "github.com/json-iterator/go" +) + +// do others that not defined in Driver interface + +func (d *CloudreveV4) getUA() string { + if d.CustomUA != "" { + return d.CustomUA + } + return base.UserAgent +} + +func (d *CloudreveV4) request(method string, path string, callback base.ReqCallback, out any) error { + if d.ref != nil { + return d.ref.request(method, path, callback, out) + } + u := d.Address + "/api/v4" + path + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "User-Agent": d.getUA(), + }) + if d.AccessToken != "" { + req.SetHeader("Authorization", "Bearer "+d.AccessToken) + } + + var r Resp + req.SetResult(&r) + + if callback != nil { + callback(req) + } + + resp, err := req.Execute(method, u) + if err != nil { + return err + } + if !resp.IsSuccess() { + return errors.New(resp.String()) + } + + if r.Code != 0 { + if r.Code == 401 && d.RefreshToken != "" && path != "/session/token/refresh" { + // try to refresh token + err = d.refreshToken() + if err != nil { + return err + } + return d.request(method, path, callback, out) + } + return errors.New(r.Msg) + } + + if out != nil && r.Data != nil { + var marshal []byte + marshal, err = json.Marshal(r.Data) + if err != nil { + return err + } + err = json.Unmarshal(marshal, out) + if err != nil { + return err + } + } + + return nil +} + +func (d *CloudreveV4) login() error { + var siteConfig SiteLoginConfigResp + err := d.request(http.MethodGet, "/site/config/login", nil, &siteConfig) + if err != nil { + return err + } + if !siteConfig.Authn { + return errors.New("authn not support") + } + var prepareLogin PrepareLoginResp + err = d.request(http.MethodGet, "/session/prepare?email="+d.Addition.Username, nil, &prepareLogin) + if err != nil { + return err + } + if !prepareLogin.PasswordEnabled { + return errors.New("password not enabled") + } + if prepareLogin.WebauthnEnabled { + return errors.New("webauthn not support") + } + for range 5 { + err = d.doLogin(siteConfig.LoginCaptcha) + if err == nil { + break + } + if err.Error() != "CAPTCHA not match." { + break + } + } + return err +} + +func (d *CloudreveV4) doLogin(needCaptcha bool) error { + var err error + loginBody := base.Json{ + "email": d.Username, + "password": d.Password, + } + if needCaptcha { + var config BasicConfigResp + err = d.request(http.MethodGet, "/site/config/basic", nil, &config) + if err != nil { + return err + } + if config.CaptchaType != "normal" { + return fmt.Errorf("captcha type %s not support", config.CaptchaType) + } + var captcha CaptchaResp + err = d.request(http.MethodGet, "/site/captcha", nil, &captcha) + if err != nil { + return err + } + if !strings.HasPrefix(captcha.Image, "data:image/png;base64,") { + return errors.New("can not get captcha") + } + loginBody["ticket"] = captcha.Ticket + i := strings.Index(captcha.Image, ",") + dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(captcha.Image[i+1:])) + vRes, err := base.RestyClient.R().SetMultipartField( + "image", "validateCode.png", "image/png", dec). + Post(setting.GetStr(conf.OcrApi)) + if err != nil { + return err + } + if jsoniter.Get(vRes.Body(), "status").ToInt() != 200 { + return errors.New("ocr error:" + jsoniter.Get(vRes.Body(), "msg").ToString()) + } + captchaCode := jsoniter.Get(vRes.Body(), "result").ToString() + if captchaCode == "" { + return errors.New("ocr error: empty result") + } + loginBody["captcha"] = captchaCode + } + var token TokenResponse + err = d.request(http.MethodPost, "/session/token", func(req *resty.Request) { + req.SetBody(loginBody) + }, &token) + if err != nil { + return err + } + d.AccessToken, d.RefreshToken = token.Token.AccessToken, token.Token.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *CloudreveV4) refreshToken() error { + var token Token + if token.RefreshToken == "" { + if d.Username != "" { + err := d.login() + if err != nil { + return fmt.Errorf("cannot login to get refresh token, error: %s", err) + } + } + return nil + } + err := d.request(http.MethodPost, "/session/token/refresh", func(req *resty.Request) { + req.SetBody(base.Json{ + "refresh_token": d.RefreshToken, + }) + }, &token) + if err != nil { + return err + } + d.AccessToken, d.RefreshToken = token.AccessToken, token.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *CloudreveV4) upLocal(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + if DEFAULT == 0 { + // support relay + DEFAULT = file.GetSize() + } + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[CloudreveV4-Local] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { + req.SetHeader("Content-Type", "application/octet-stream") + req.SetContentLength(true) + req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) + req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + req.AddRetryCondition(func(r *resty.Response, err error) bool { + if err != nil { + return true + } + if r.IsError() { + return true + } + var retryResp Resp + jErr := base.RestyClient.JSONUnmarshal(r.Body(), &retryResp) + if jErr != nil { + return true + } + if retryResp.Code != 0 { + return true + } + return false + }) + }, nil) + if err != nil { + return err + } + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + chunk++ + } + return nil +} + +func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { + uploadUrl := u.UploadUrls[0] + credential := u.Credential + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + retryCount := 0 + maxRetries := 3 + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[CloudreveV4-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.ContentLength = byteSize + // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.Header.Set("Authorization", fmt.Sprint(credential)) + req.Header.Set("User-Agent", d.getUA()) + err = func() error { + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return errors.New(res.Status) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }() + if err == nil { + retryCount = 0 + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + chunk++ + } else { + retryCount++ + if retryCount > maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors", maxRetries) + } + backoff := time.Duration(1<") + for i, etag := range etags { + bodyBuilder.WriteString(fmt.Sprintf( + `%d%s`, + i+1, // PartNumber 从 1 开始 + etag, + )) + } + bodyBuilder.WriteString("") + req, err := http.NewRequest( + "POST", + u.CompleteURL, + strings.NewReader(bodyBuilder.String()), + ) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/xml") + req.Header.Set("User-Agent", d.getUA()) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + return fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(body)) + } + + // 上传成功发送回调请求 + return d.request(http.MethodPost, "/callback/s3/"+u.SessionID+"/"+u.CallbackSecret, func(req *resty.Request) { + req.SetBody("{}") + }, nil) +} diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index e256b7ae262..28ed5ccc3cc 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -8,6 +8,7 @@ import ( "io" "net/http" stdpath "path" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -17,7 +18,6 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" - log "github.com/sirupsen/logrus" ) var onedriveHostMap = map[string]Host{ @@ -204,19 +204,18 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() var finish int64 = 0 DEFAULT := d.ChunkSize * 1024 * 1024 + retryCount := 0 + maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } - log.Debugf("upload: %d", finish) - var byteSize int64 = DEFAULT left := stream.GetSize() - finish - if left < DEFAULT { - byteSize = left - } + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Onedrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) - log.Debug(err, n) + utils.Log.Debug(err, n) if err != nil { return err } @@ -228,19 +227,31 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil req.ContentLength = byteSize // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) - finish += byteSize res, err := base.HttpClient.Do(req) if err != nil { return err } // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - if res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200 { + switch { + case res.StatusCode >= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< Date: Wed, 4 Jun 2025 15:04:06 +0000 Subject: [PATCH 002/133] feat(local): add options to use ffmpeg to generate thumbnail --- drivers/local/driver.go | 15 ++++++ drivers/local/meta.go | 2 + drivers/local/util.go | 114 +++++++++++++++++++++++++++++++++------- 3 files changed, 113 insertions(+), 18 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index faa2b3bd157..0e469cb190e 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -39,6 +39,10 @@ type Local struct { // video thumb position videoThumbPos float64 videoThumbPosIsPercentage bool + thumbPixel int + + // use ffmpeg + useFFmpeg bool } func (d *Local) Config() driver.Config { @@ -65,6 +69,9 @@ func (d *Local) Init(ctx context.Context) error { } d.Addition.RootFolderPath = abs } + + d.useFFmpeg = d.UseFFmpeg + if d.ThumbCacheFolder != "" && !utils.Exists(d.ThumbCacheFolder) { err := os.MkdirAll(d.ThumbCacheFolder, os.FileMode(d.mkdirPerm)) if err != nil { @@ -78,6 +85,14 @@ func (d *Local) Init(ctx context.Context) error { } d.thumbConcurrency = int(v) } + if d.ThumbPixel != "" { + v, err := strconv.ParseUint(d.ThumbPixel, 10, 32) + if err != nil { + return err + } + d.thumbPixel = int(v) + } + if d.thumbConcurrency == 0 { d.thumbTokenBucket = NewNopTokenBucket() } else { diff --git a/drivers/local/meta.go b/drivers/local/meta.go index 14b0404f784..70ce090db5f 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -8,8 +8,10 @@ import ( type Addition struct { driver.RootPath Thumbnail bool `json:"thumbnail" required:"true" help:"enable thumbnail"` + UseFFmpeg bool `json:"use_ffmpeg" required:"true" help:"use ffmpeg to generate thumbnail"` ThumbCacheFolder string `json:"thumb_cache_folder"` ThumbConcurrency string `json:"thumb_concurrency" default:"16" required:"false" help:"Number of concurrent thumbnail generation goroutines. This controls how many thumbnails can be generated in parallel."` + ThumbPixel string `json:"thumb_pixel" default:"320" required:"false" help:"Specifies the target width for image thumbnails in pixels. The height of the thumbnail will be calculated automatically to maintain the original aspect ratio of the image."` VideoThumbPos string `json:"video_thumb_pos" default:"20%" required:"false" help:"The position of the video thumbnail. If the value is a number (integer ot floating point), it represents the time in seconds. If the value ends with '%', it represents the percentage of the video duration."` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` MkdirPerm string `json:"mkdir_perm" default:"777"` diff --git a/drivers/local/util.go b/drivers/local/util.go index 802f60cf627..a473b709dd8 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -36,6 +36,87 @@ func isSymlinkDir(f fs.FileInfo, path string) bool { return false } +// resizeImageToBufferWithFFmpegGo 使用 ffmpeg-go 调整图片大小并输出到内存缓冲区 +func resizeImageToBufferWithFFmpegGo(inputFile string, width int, outputFormat string /* e.g., "image2pipe", "png_pipe", "mjpeg" */) (*bytes.Buffer, error) { + outBuffer := bytes.NewBuffer(nil) + + // Determine codec based on desired output format for piping + // For generic image piping, 'image2' is often used with -f image2pipe + // For specific formats to buffer, you might specify the codec directly + var vcodec string + switch outputFormat { + case "png_pipe": // if you want to ensure PNG format in buffer + vcodec = "png" + case "mjpeg": // if you want to ensure JPEG format in buffer + vcodec = "mjpeg" + // default or "image2pipe" could leave codec choice more to ffmpeg or require -c:v later + } + + outputArgs := ffmpeg.KwArgs{ + "vf": fmt.Sprintf("scale=%d:-1:flags=lanczos,format=yuv444p", width), + "vframes": "1", + "f": outputFormat, // Format for piping (e.g., image2pipe, png_pipe) + } + if vcodec != "" { + outputArgs["vcodec"] = vcodec + } + if outputFormat == "mjpeg" { + outputArgs["q:v"] = "3" + } + + err := ffmpeg.Input(inputFile). + Output("pipe:", outputArgs). // Output to pipe (stdout) + GlobalArgs("-loglevel", "error"). + Silent(true). // Suppress ffmpeg's own console output + WithOutput(outBuffer, os.Stderr). // Capture stdout to outBuffer, stderr to os.Stderr + // ErrorToStdOut(). // Alternative: send ffmpeg's stderr to Go's stdout + Run() + + if err != nil { + return nil, fmt.Errorf("ffmpeg-go failed to resize image %s to buffer: %w", inputFile, err) + } + if outBuffer.Len() == 0 { + return nil, fmt.Errorf("ffmpeg-go produced empty buffer for %s", inputFile) + } + + return outBuffer, nil +} + +func generateThumbnailWithImagingOptimized(imagePath string, targetWidth int, quality int) (*bytes.Buffer, error) { + + file, err := os.Open(imagePath) + if err != nil { + return nil, fmt.Errorf("failed to open image: %w", err) + } + defer file.Close() + + img, err := imaging.Decode(file, imaging.AutoOrientation(true)) + if err != nil { + return nil, fmt.Errorf("failed to decode image: %w", err) + } + + thumbImg := imaging.Resize(img, targetWidth, 0, imaging.Lanczos) + img = nil + + var buf bytes.Buffer + // imaging.Encode + // imaging.PNG, imaging.JPEG, imaging.GIF, imaging.BMP, imaging.TIFF + outputFormat := imaging.JPEG + encodeOptions := []imaging.EncodeOption{imaging.JPEGQuality(quality)} + + // outputFormat := imaging.PNG + // encodeOptions := []imaging.EncodeOption{} + + err = imaging.Encode(&buf, thumbImg, outputFormat, encodeOptions...) + if err != nil { + return nil, fmt.Errorf("failed to encode thumbnail: %w", err) + } + + thumbImg = nil + + return &buf, nil +} + // Get the snapshot of the video func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) { // Run ffprobe to get the video duration @@ -80,7 +161,7 @@ func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) // The "noaccurate_seek" option prevents this error and would also speed up // the seek process. stream := ffmpeg.Input(videoPath, ffmpeg.KwArgs{"ss": ss, "noaccurate_seek": ""}). - Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg"}). + Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg", "vf": fmt.Sprintf("scale=%d:-1:flags=lanczos", d.thumbPixel)}). GlobalArgs("-loglevel", "error").Silent(true). WithOutput(srcBuf, os.Stdout) if err = stream.Run(); err != nil { @@ -125,29 +206,26 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { } srcBuf = videoBuf } else { - imgData, err := os.ReadFile(fullPath) - if err != nil { - return nil, nil, err + if d.useFFmpeg { + imgData, err := resizeImageToBufferWithFFmpegGo(fullPath, d.thumbPixel, "image2pipe") + srcBuf = imgData + if err != nil { + return nil, nil, err + } + } else { + imgData, err := generateThumbnailWithImagingOptimized(fullPath, d.thumbPixel, 70) + srcBuf = imgData + if err != nil { + return nil, nil, err + } } - imgBuf := bytes.NewBuffer(imgData) - srcBuf = imgBuf } - image, err := imaging.Decode(srcBuf, imaging.AutoOrientation(true)) - if err != nil { - return nil, nil, err - } - thumbImg := imaging.Resize(image, 144, 0, imaging.Lanczos) - var buf bytes.Buffer - err = imaging.Encode(&buf, thumbImg, imaging.PNG) - if err != nil { - return nil, nil, err - } if d.ThumbCacheFolder != "" { - err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0666) + err := os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), srcBuf.Bytes(), 0666) if err != nil { return nil, nil, err } } - return &buf, nil, nil + return srcBuf, nil, nil } From 7aeb0ab078c78dad7e1acb531f3cd0fdf78f2d62 Mon Sep 17 00:00:00 2001 From: AlistDev Date: Fri, 27 Jun 2025 16:28:09 +0800 Subject: [PATCH 003/133] fix: update documentation links to point to the new domain And fix 189pc getToken fail --- .github/FUNDING.yml | 2 +- .github/ISSUE_TEMPLATE/bug_report.yml | 8 ++++---- .github/ISSUE_TEMPLATE/feature_request.yml | 2 +- README.md | 8 ++++---- README_cn.md | 10 +++++----- README_ja.md | 10 +++++----- cmd/root.go | 2 +- drivers/189pc/utils.go | 2 +- drivers/onedrive/meta.go | 2 +- internal/bootstrap/data/setting.go | 2 +- internal/model/user.go | 2 +- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index a336406b355..f9e80a5aada 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -10,4 +10,4 @@ liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry -custom: ['https://alist.nn.ci/guide/sponsor.html'] +custom: ['https://alistgo.com/guide/sponsor.html'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 07a8338e5ef..f5cfaedacf8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -16,14 +16,14 @@ body: 您必须勾选以下所有内容，否则您的issue可能会被直接关闭。或者您可以去[讨论区](https://github.com/alist-org/alist/discussions) options: - label: | - I have read the [documentation](https://alist.nn.ci). - 我已经阅读了[文档](https://alist.nn.ci)。 + I have read the [documentation](https://alistgo.com). + 我已经阅读了[文档](https://alistgo.com)。 - label: | I'm sure there are no duplicate issues or discussions. 我确定没有重复的issue或讨论。 - label: | - I'm sure it's due to `AList` and not something else(such as [Network](https://alist.nn.ci/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host) ,`Dependencies` or `Operational`). - 我确定是`AList`的问题，而不是其他原因（例如[网络](https://alist.nn.ci/zh/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host)，`依赖`或`操作`）。 + I'm sure it's due to `AList` and not something else(such as [Network](https://alistgo.com/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host) ,`Dependencies` or `Operational`). + 我确定是`AList`的问题，而不是其他原因（例如[网络](https://alistgo.com/zh/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host)，`依赖`或`操作`）。 - label: | I'm sure this issue is not fixed in the latest version. 我确定这个问题在最新版本中没有被修复。 diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index a16c8f98d24..a118992ce0b 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -7,7 +7,7 @@ body: label: Please make sure of the following things description: You may select more than one, even select all. options: - - label: I have read the [documentation](https://alist.nn.ci). + - label: I have read the [documentation](https://alistgo.com). - label: I'm sure there are no duplicate issues or discussions. - label: I'm sure this feature is not implemented. - label: I'm sure it's a reasonable and popular requirement. diff --git a/README.md b/README.md index 1261839e429..2bd7e812890 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
- logo + logo

🗂️A file list program that supports multiple storages, powered by Gin and Solidjs.

@@ -88,7 +88,7 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing - [x] Dark mode - [x] I18n - [x] Protected routes (password protection and authentication) -- [x] WebDav (see https://alist.nn.ci/guide/webdav.html for details) +- [x] WebDav (see https://alistgo.com/guide/webdav.html for details) - [x] [Docker Deploy](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare Workers proxy - [x] File/Folder package download @@ -112,7 +112,7 @@ Please go to our [discussion forum](https://github.com/alist-org/alist/discussio ## Sponsor AList is an open-source software, if you happen to like this project and want me to keep going, please consider sponsoring me or providing a single donation! Thanks for all the love and support: -https://alist.nn.ci/guide/sponsor.html +https://alistgo.com/guide/sponsor.html ### Special sponsors diff --git a/README_cn.md b/README_cn.md index 5c71ccce4c3..9052e79b0ea 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@
- logo + logo

🗂一个支持多存储的文件列表程序，使用 Gin 和 Solidjs。

@@ -86,7 +86,7 @@ - [x] 黑暗模式 - [x] 国际化 - [x] 受保护的路由（密码保护和身份验证） -- [x] WebDav (具体见 https://alist.nn.ci/zh/guide/webdav.html) +- [x] WebDav (具体见 https://alistgo.com/zh/guide/webdav.html) - [x] [Docker 部署](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare workers 中转 - [x] 文件/文件夹打包下载 @@ -97,7 +97,7 @@ ## 文档 - + ## Demo @@ -109,7 +109,7 @@ ## 赞助 -AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我继续下去，请考虑赞助我或提供一个单一的捐款！感谢所有的爱和支持：https://alist.nn.ci/zh/guide/sponsor.html +AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我继续下去，请考虑赞助我或提供一个单一的捐款！感谢所有的爱和支持：https://alistgo.com/zh/guide/sponsor.html ### 特别赞助 diff --git a/README_ja.md b/README_ja.md index cd4446fab8e..4dcdfd203bf 100644 --- a/README_ja.md +++ b/README_ja.md @@ -1,5 +1,5 @@
- logo + logo

🗂️Gin と Solidjs による、複数のストレージをサポートするファイルリストプログラム。

@@ -87,7 +87,7 @@ - [x] ダークモード - [x] 国際化 - [x] 保護されたルート (パスワード保護と認証) -- [x] WebDav (詳細は https://alist.nn.ci/guide/webdav.html を参照) +- [x] WebDav (詳細は https://alistgo.com/guide/webdav.html を参照) - [x] [Docker デプロイ](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare ワーカープロキシ - [x] ファイル/フォルダパッケージのダウンロード @@ -98,7 +98,7 @@ ## ドキュメント - + ## デモ @@ -111,7 +111,7 @@ ## スポンサー AList はオープンソースのソフトウェアです。もしあなたがこのプロジェクトを気に入ってくださり、続けて欲しいと思ってくださるなら、ぜひスポンサーになってくださるか、1口でも寄付をしてくださるようご検討ください！すべての愛とサポートに感謝します: -https://alist.nn.ci/guide/sponsor.html +https://alistgo.com/guide/sponsor.html ### スペシャルスポンサー diff --git a/cmd/root.go b/cmd/root.go index 59eb989c3a0..cd50529728b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ var RootCmd = &cobra.Command{ Short: "A file list program that supports multiple storage.", Long: `A file list program that supports multiple storage, built with love by Xhofe and friends in Go/Solid.js. -Complete documentation is available at https://alist.nn.ci/`, +Complete documentation is available at https://alistgo.com/`, } func Execute() { diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index c391f7e676f..a8b444cb4b7 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -324,7 +324,7 @@ func (y *Cloud189PC) login() (err error) { _, err = y.client.R(). SetResult(&tokenInfo).SetError(&erron). SetQueryParams(clientSuffix()). - SetQueryParam("redirectURL", url.QueryEscape(loginresp.ToUrl)). + SetQueryParam("redirectURL", loginresp.ToUrl). Post(API_URL + "/getSessionForPC.action") if err != nil { return diff --git a/drivers/onedrive/meta.go b/drivers/onedrive/meta.go index a60e5f33a93..54a7340a942 100644 --- a/drivers/onedrive/meta.go +++ b/drivers/onedrive/meta.go @@ -11,7 +11,7 @@ type Addition struct { IsSharepoint bool `json:"is_sharepoint"` ClientID string `json:"client_id" required:"true"` ClientSecret string `json:"client_secret" required:"true"` - RedirectUri string `json:"redirect_uri" required:"true" default:"https://alist.nn.ci/tool/onedrive/callback"` + RedirectUri string `json:"redirect_uri" required:"true" default:"https://alistgo.com/tool/onedrive/callback"` RefreshToken string `json:"refresh_token" required:"true"` SiteId string `json:"site_id"` ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 407a5c64e17..fe1d9219a08 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -155,7 +155,7 @@ func InitialSettings() []model.SettingItem { ([[:xdigit:]]{1,4}(?::[[:xdigit:]]{1,4}){7}|::|:(?::[[:xdigit:]]{1,4}){1,6}|[[:xdigit:]]{1,4}:(?::[[:xdigit:]]{1,4}){1,5}|(?:[[:xdigit:]]{1,4}:){2}(?::[[:xdigit:]]{1,4}){1,4}|(?:[[:xdigit:]]{1,4}:){3}(?::[[:xdigit:]]{1,4}){1,3}|(?:[[:xdigit:]]{1,4}:){4}(?::[[:xdigit:]]{1,4}){1,2}|(?:[[:xdigit:]]{1,4}:){5}:[[:xdigit:]]{1,4}|(?:[[:xdigit:]]{1,4}:){1,6}:) (?U)access_token=(.*)&`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, - {Key: conf.OcrApi, Value: "https://api.nn.ci/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, + {Key: conf.OcrApi, Value: "https://api.alistgo.com/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL}, {Key: conf.ForwardDirectLinkParams, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL}, {Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL}, diff --git a/internal/model/user.go b/internal/model/user.go index eaa0fed9d09..0f7d3af5064 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -177,5 +177,5 @@ func (u *User) WebAuthnCredentials() []webauthn.Credential { } func (u *User) WebAuthnIcon() string { - return "https://alist.nn.ci/logo.svg" + return "https://alistgo.com/logo.svg" } From b1586612ca75e5a9e55ed79b829a76988584c004 Mon Sep 17 00:00:00 2001 From: Alone Date: Fri, 27 Jun 2025 23:39:23 +0800 Subject: [PATCH 004/133] feat: add ghcr docker image (#8524) --- .github/workflows/release_docker.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 7cd05549f18..1c31b2fd20f 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -18,6 +18,7 @@ env: REGISTRY: 'xhofe/alist' REGISTRY_USERNAME: 'xhofe' REGISTRY_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + GITHUB_CR_REPO: ghcr.io/${{ github.repository }} ARTIFACT_NAME: 'binaries_docker_release' RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64' IMAGE_PUSH: ${{ github.event_name == 'push' }} @@ -114,11 +115,21 @@ jobs: username: ${{ env.REGISTRY_USERNAME }} password: ${{ env.REGISTRY_PASSWORD }} + - name: Login to GHCR + uses: docker/login-action@v3 + with: + logout: true + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }} + images: | + ${{ env.REGISTRY }} + ${{ env.GITHUB_CR_REPO }} tags: ${{ env.IMAGE_IS_PROD == 'true' && '' || env.IMAGE_TAGS_BETA }} flavor: | ${{ env.IMAGE_IS_PROD == 'true' && 'latest=true' || '' }} From 51eeb224657972c6c8d910a6b4368cd702b8c2b4 Mon Sep 17 00:00:00 2001 From: alistgo Date: Fri, 27 Jun 2025 23:58:52 +0800 Subject: [PATCH 005/133] fix: dead link --- .github/workflows/beta_release.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- build.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 485942c4a9b..27e0142ea3f 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -119,7 +119,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 with: - repository: alist-org/desktop-release + repository: AlistGo/desktop-release ref: main persist-credentials: false fetch-depth: 0 @@ -135,4 +135,4 @@ jobs: with: github_token: ${{ secrets.MY_TOKEN }} branch: main - repository: alist-org/desktop-release \ No newline at end of file + repository: AlistGo/desktop-release \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d42019ad06..2257826b064 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,7 +72,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 with: - repository: alist-org/desktop-release + repository: AlistGo/desktop-release ref: main persist-credentials: false fetch-depth: 0 @@ -89,4 +89,4 @@ jobs: with: github_token: ${{ secrets.MY_TOKEN }} branch: main - repository: alist-org/desktop-release \ No newline at end of file + repository: AlistGo/desktop-release \ No newline at end of file diff --git a/build.sh b/build.sh index 2dee8e20773..4045820adfd 100644 --- a/build.sh +++ b/build.sh @@ -93,7 +93,7 @@ BuildDocker() { PrepareBuildDockerMusl() { mkdir -p build/musl-libs - BASE="https://musl.cc/" + BASE="https://github.com/go-cross/musl-toolchain-archive/releases/latest/download/" FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross s390x-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross riscv64-linux-musl-cross powerpc64le-linux-musl-cross) for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" @@ -245,7 +245,7 @@ BuildReleaseFreeBSD() { cgo_cc="clang --target=${CGO_ARGS[$i]} --sysroot=/opt/freebsd/${os_arch}" echo building for freebsd-${os_arch} sudo mkdir -p "/opt/freebsd/${os_arch}" - wget -q https://download.freebsd.org/releases/${os_arch}/14.1-RELEASE/base.txz + wget -q https://download.freebsd.org/releases/${os_arch}/14.3-RELEASE/base.txz sudo tar -xf ./base.txz -C /opt/freebsd/${os_arch} rm base.txz export GOOS=freebsd From fd411866797e404222bf2cecc8129f0231c9a7fa Mon Sep 17 00:00:00 2001 From: AlistDev Date: Mon, 14 Jul 2025 23:04:40 +0800 Subject: [PATCH 006/133] fix: update DriveId assignment to use DeviceID from Addition struct --- drivers/aliyundrive/driver.go | 2 +- drivers/aliyundrive/meta.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index 105e28b2e98..606ff385e81 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -55,7 +55,7 @@ func (d *AliDrive) Init(ctx context.Context) error { if err != nil { return err } - d.DriveId = utils.Json.Get(res, "default_drive_id").ToString() + d.DriveId = d.Addition.DeviceID d.UserID = utils.Json.Get(res, "user_id").ToString() d.cron = cron.NewCron(time.Hour * 2) d.cron.Do(func() { diff --git a/drivers/aliyundrive/meta.go b/drivers/aliyundrive/meta.go index 9aee856908d..a0ae8a5917d 100644 --- a/drivers/aliyundrive/meta.go +++ b/drivers/aliyundrive/meta.go @@ -7,8 +7,8 @@ import ( type Addition struct { driver.RootID - RefreshToken string `json:"refresh_token" required:"true"` - //DeviceID string `json:"device_id" required:"true"` + RefreshToken string `json:"refresh_token" required:"true"` + DeviceID string `json:"device_id" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` RapidUpload bool `json:"rapid_upload"` From 13ea1c14053163206ab37209959b88f6842fcd51 Mon Sep 17 00:00:00 2001 From: AlistDev Date: Wed, 16 Jul 2025 20:39:05 +0800 Subject: [PATCH 007/133] fix: restore user-agent header in HTTP requests --- drivers/123/util.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/123/util.go b/drivers/123/util.go index 7e5a23970c6..b85c9afbac8 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -161,12 +161,12 @@ func (d *Pan123) login() error { } res, err := base.RestyClient.R(). SetHeaders(map[string]string{ - "origin": "https://www.123pan.com", - "referer": "https://www.123pan.com/", - "user-agent": "Dart/2.19(dart:io)-alist", + "origin": "https://www.123pan.com", + "referer": "https://www.123pan.com/", + //"user-agent": "Dart/2.19(dart:io)-alist", "platform": "web", "app-version": "3", - //"user-agent": base.UserAgent, + "user-agent": base.UserAgent, }). SetBody(body).Post(SignIn) if err != nil { @@ -202,7 +202,7 @@ do: "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "authorization": "Bearer " + d.AccessToken, - "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) alist-client", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", "platform": "web", "app-version": "3", //"user-agent": base.UserAgent, From 5e15a360b7aa47b7f8eab273523dfbccf9956b14 Mon Sep 17 00:00:00 2001 From: Sakana Date: Thu, 24 Jul 2025 15:30:12 +0800 Subject: [PATCH 008/133] feat(github_releases): concurrently request the GitHub API (#9211) --- drivers/github_releases/driver.go | 189 ++++++++++++++++++------------ drivers/github_releases/meta.go | 11 +- drivers/github_releases/types.go | 2 +- drivers/github_releases/util.go | 4 +- 4 files changed, 125 insertions(+), 81 deletions(-) diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go index b35aa57a41c..3268dc2fd88 100644 --- a/drivers/github_releases/driver.go +++ b/drivers/github_releases/driver.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "sync" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -36,88 +37,130 @@ func (d *GithubReleases) Drop(ctx context.Context) error { return nil } -func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files := make([]File, 0) - path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) +// processPoint 处理单个挂载点的文件列表 +func (d *GithubReleases) processPoint(point *MountPoint, path string, args model.ListArgs) []File { + var pointFiles []File - for i := range d.points { - point := &d.points[i] + if !d.Addition.ShowAllVersion { // latest + point.RequestLatestRelease(d.GetRequest, args.Refresh) + pointFiles = d.processLatestVersion(point, path) + } else { // all version + point.RequestReleases(d.GetRequest, args.Refresh) + pointFiles = d.processAllVersions(point, path) + } - if !d.Addition.ShowAllVersion { // latest - point.RequestRelease(d.GetRequest, args.Refresh) + return pointFiles +} - if point.Point == path { // 与仓库路径相同 - files = append(files, point.GetLatestRelease()...) - if d.Addition.ShowReadme { - files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) - } - } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 - nextDir := GetNextDir(point.Point, path) - if nextDir == "" { - continue - } +// processLatestVersion 处理最新版本的逻辑 +func (d *GithubReleases) processLatestVersion(point *MountPoint, path string) []File { + var pointFiles []File - hasSameDir := false - for index := range files { - if files[index].GetName() == nextDir { - hasSameDir = true - files[index].Size += point.GetLatestSize() - break - } - } - if !hasSameDir { - files = append(files, File{ - Path: path + "/" + nextDir, - FileName: nextDir, - Size: point.GetLatestSize(), - UpdateAt: point.Release.PublishedAt, - CreateAt: point.Release.CreatedAt, - Type: "dir", - Url: "", - }) - } + if point.Point == path { // 与仓库路径相同 + pointFiles = append(pointFiles, point.GetLatestRelease()...) + if d.Addition.ShowReadme { + files := point.GetOtherFile(d.GetRequest, false) + pointFiles = append(pointFiles, files...) + } + } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + nextDir := GetNextDir(point.Point, path) + if nextDir != "" { + dirFile := File{ + Path: path + "/" + nextDir, + FileName: nextDir, + Size: point.GetLatestSize(), + UpdateAt: point.Release.PublishedAt, + CreateAt: point.Release.CreatedAt, + Type: "dir", + Url: "", } - } else { // all version - point.RequestReleases(d.GetRequest, args.Refresh) + pointFiles = append(pointFiles, dirFile) + } + } - if point.Point == path { // 与仓库路径相同 - files = append(files, point.GetAllVersion()...) - if d.Addition.ShowReadme { - files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) - } - } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 - nextDir := GetNextDir(point.Point, path) - if nextDir == "" { - continue - } + return pointFiles +} - hasSameDir := false - for index := range files { - if files[index].GetName() == nextDir { - hasSameDir = true - files[index].Size += point.GetAllVersionSize() - break - } - } - if !hasSameDir { - files = append(files, File{ - FileName: nextDir, - Path: path + "/" + nextDir, - Size: point.GetAllVersionSize(), - UpdateAt: (*point.Releases)[0].PublishedAt, - CreateAt: (*point.Releases)[0].CreatedAt, - Type: "dir", - Url: "", - }) - } - } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 - tagName := GetNextDir(path, point.Point) - if tagName == "" { - continue - } +// processAllVersions 处理所有版本的逻辑 +func (d *GithubReleases) processAllVersions(point *MountPoint, path string) []File { + var pointFiles []File - files = append(files, point.GetReleaseByTagName(tagName)...) + if point.Point == path { // 与仓库路径相同 + pointFiles = append(pointFiles, point.GetAllVersion()...) + if d.Addition.ShowReadme { + files := point.GetOtherFile(d.GetRequest, false) + pointFiles = append(pointFiles, files...) + } + } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + nextDir := GetNextDir(point.Point, path) + if nextDir != "" { + dirFile := File{ + FileName: nextDir, + Path: path + "/" + nextDir, + Size: point.GetAllVersionSize(), + UpdateAt: (*point.Releases)[0].PublishedAt, + CreateAt: (*point.Releases)[0].CreatedAt, + Type: "dir", + Url: "", } + pointFiles = append(pointFiles, dirFile) + } + } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 + tagName := GetNextDir(path, point.Point) + if tagName != "" { + pointFiles = append(pointFiles, point.GetReleaseByTagName(tagName)...) + } + } + + return pointFiles +} + +// mergeFiles 合并文件列表，处理重复目录 +func (d *GithubReleases) mergeFiles(files *[]File, newFiles []File) { + for _, newFile := range newFiles { + if newFile.Type == "dir" { + hasSameDir := false + for index := range *files { + if (*files)[index].GetName() == newFile.GetName() && (*files)[index].Type == "dir" { + hasSameDir = true + (*files)[index].Size += newFile.Size + break + } + } + if !hasSameDir { + *files = append(*files, newFile) + } + } else { + *files = append(*files, newFile) + } + } +} + +func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files := make([]File, 0) + path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) + + if d.Addition.ConcurrentRequests && d.Addition.Token != "" { // 并发处理 + var mu sync.Mutex + var wg sync.WaitGroup + + for i := range d.points { + wg.Add(1) + go func(point *MountPoint) { + defer wg.Done() + pointFiles := d.processPoint(point, path, args) + + mu.Lock() + d.mergeFiles(&files, pointFiles) + mu.Unlock() + }(&d.points[i]) + } + wg.Wait() + } else { // 串行处理 + for i := range d.points { + point := &d.points[i] + pointFiles := d.processPoint(point, path, args) + d.mergeFiles(&files, pointFiles) } } diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go index 47b84d37927..b54cb3cc608 100644 --- a/drivers/github_releases/meta.go +++ b/drivers/github_releases/meta.go @@ -7,11 +7,12 @@ import ( type Addition struct { driver.RootID - RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"` - ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` - Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` - ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` - GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` + RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"` + ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` + Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` + ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` + ConcurrentRequests bool `json:"concurrent_requests" type:"bool" default:"false" help:"To concurrently request the GitHub API, you must enter a GitHub token"` + GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` } var config = driver.Config{ diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go index b0a9ee619e0..b4562056185 100644 --- a/drivers/github_releases/types.go +++ b/drivers/github_releases/types.go @@ -18,7 +18,7 @@ type MountPoint struct { } // 请求最新版本 -func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) { +func (m *MountPoint) RequestLatestRelease(get func(url string) (*resty.Response, error), refresh bool) { if m.Repo == "" { return } diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go index df846e8a109..097295bf408 100644 --- a/drivers/github_releases/util.go +++ b/drivers/github_releases/util.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" - log "github.com/sirupsen/logrus" ) // 发送 GET 请求 @@ -23,7 +23,7 @@ func (d *GithubReleases) GetRequest(url string) (*resty.Response, error) { return nil, err } if res.StatusCode() != 200 { - log.Warn("failed to get request: ", res.StatusCode(), res.String()) + utils.Log.Warnf("failed to get request: %s %d %s", url, res.StatusCode(), res.String()) } return res, nil } From 00120cba273a131d5e916895e1d4ee324095cdf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sat, 26 Jul 2025 09:51:59 +0800 Subject: [PATCH 009/133] feat: enhance permission control and label management (#9215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 标签管理 * pr检查优化 * feat(role): Implement role management functionality - Add role management routes in `server/router.go` for listing, getting, creating, updating, and deleting roles - Introduce `initRoles()` in `internal/bootstrap/data/data.go` for initializing roles during bootstrap - Create `internal/op/role.go` to handle role operations including caching and singleflight - Implement role handler functions in `server/handles/role.go` for API responses - Define database operations for roles in `internal/db/role.go` - Extend `internal/db/db.go` for role model auto-migration - Design `internal/model/role.go` to represent role structure with ID, name, description, base path, and permissions - Initialize default roles (`admin` and `guest`) in `internal/bootstrap/data/role.go` during startup * refactor(user roles): Support multiple roles for users - Change the `Role` field type from `int` to `[]int` in `drivers/alist_v3/types.go` and `drivers/quqi/types.go`. - Update the `Role` field in `internal/model/user.go` to use a new `Roles` type with JSON and database support. - Modify `IsGuest` and `IsAdmin` methods to check for roles using `Contains` method. - Update `GetUserByRole` method in `internal/db/user.go` to handle multiple roles. - Add `roles.go` to define a new `Roles` type with JSON marshalling and scanning capabilities. - Adjust code in `server/handles/user.go` to compare roles with `utils.SliceEqual`. - Change role initialization for users in `internal/bootstrap/data/dev.go` and `internal/bootstrap/data/user.go`. - Update `Role` handling in `server/handles/task.go`, `server/handles/ssologin.go`, and `server/handles/ldap_login.go`. * feat(user/role): Add path limit check for user and role permissions - Add new permission bit for checking path limits in `user.go` - Implement `CheckPathLimit` method in `User` struct to validate path access - Modify `JoinPath` method in `User` to enforce path limit checks - Update `role.go` to include path limit logic in `Role` struct - Document new permission bit in `Role` and `User` comments for clarity * feat(permission): Add role-based permission handling - Introduce `role_perm.go` for managing user permissions based on roles. - Implement `HasPermission` and `MergeRolePermissions` functions. - Update `webdav.go` to utilize role-based permissions instead of direct user checks. - Modify `fsup.go` to integrate `CanAccessWithRoles` function. - Refactor `fsread.go` to use `common.HasPermission` for permission validation. - Adjust `fsmanage.go` for role-based access control checks. - Enhance `ftp.go` and `sftp.go` to manage FTP access via roles. - Update `fsbatch.go` to employ `MergeRolePermissions` for batch operations. - Replace direct user permission checks with role-based permission handling across various modules. * refactor(user): Replace integer role values with role IDs - Change `GetAdmin()` and `GetGuest()` functions to retrieve role by name and use role ID. - Add patch for version `v3.45.2` to convert legacy integer roles to role IDs. - Update `dev.go` and `user.go` to use role IDs instead of integer values for roles. - Remove redundant code in `role.go` related to guest role creation. - Modify `ssologin.go` and `ldap_login.go` to set user roles to nil instead of using integer roles. - Introduce `convert_roles.go` to handle conversion of legacy roles and ensure role existence in the database. * feat(role_perm): implement support for multiple base paths for roles - Modify role permission checks to support multiple base paths - Update role creation and update functions to handle multiple base paths - Add migration script to convert old base_path to base_paths - Define new Paths type for handling multiple paths in the model - Adjust role model to replace BasePath with BasePaths - Update existing patches to handle roles with multiple base paths - Update bootstrap data to reflect the new base_paths field * feat(role): Restrict modifications to default roles (admin and guest) - Add validation to prevent changes to "admin" and "guest" roles in `UpdateRole` and `DeleteRole` functions. - Introduce `ErrChangeDefaultRole` error in `internal/errs/role.go` to standardize error messaging. - Update role-related API handlers in `server/handles/role.go` to enforce the new restriction. - Enhance comments in `internal/bootstrap/data/role.go` to clarify the significance of default roles. - Ensure consistent error responses for unauthorized role modifications across the application. * 🔄 **refactor(role): Enhance role permission handling** - Replaced `BasePaths` with `PermissionPaths` in `Role` struct for better permission granularity. - Introduced JSON serialization for `PermissionPaths` using `RawPermission` field in `Role` struct. - Implemented `BeforeSave` and `AfterFind` GORM hooks for handling `PermissionPaths` serialization. - Refactored permission calculation logic in `role_perm.go` to work with `PermissionPaths`. - Updated role creation logic to initialize `PermissionPaths` for `admin` and `guest` roles. - Removed deprecated `CheckPathLimit` method from `Role` struct. * fix(model/user/role): update permission settings for admin and role - Change `RawPermission` field in `role.go` to hide JSON representation - Update `Permission` field in `user.go` to `0xFFFF` for full access - Modify `PermissionScopes` in `role.go` to `0xFFFF` for enhanced permissions * 🔒 feat(role-permissions): Enhance role-based access control - Introduce `canReadPathByRole` function in `role_perm.go` to verify path access based on user roles - Modify `CanAccessWithRoles` to include role-based path read check - Add `RoleNames` and `Permissions` to `UserResp` struct in `auth.go` for enhanced user role and permission details - Implement role details aggregation in `auth.go` to populate `RoleNames` and `Permissions` - Update `User` struct in `user.go` to include `RolesDetail` for more detailed role information - Enhance middleware in `auth.go` to load and verify detailed role information for users - Move `guest` user initialization logic in `user.go` to improve code organization and avoid repetition * 🔒 fix(permissions): Add permission checks for archive operations - Add `MergeRolePermissions` and `HasPermission` checks to validate user access for reading archives - Ensure users have `PermReadArchives` before proceeding with `GetNearestMeta` in specific archive paths - Implement permission checks for decompress operations, requiring `PermDecompress` for source paths - Return `PermissionDenied` errors with 403 status if user lacks necessary permissions * 🔒 fix(server): Add permission check for offline download - Add permission merging logic for user roles - Check user has permission for offline download addition - Return error response with "permission denied" if check fails * ✨ feat(role-permission): Implement path-based role permission checks - Add `CheckPathLimitWithRoles` function to validate access based on `PermPathLimit` permission. - Integrate `CheckPathLimitWithRoles` in `offline_download` to enforce path-based access control. - Apply `CheckPathLimitWithRoles` across file system management operations (e.g., creation, movement, deletion). - Ensure `CheckPathLimitWithRoles` is invoked for batch operations and archive-related actions. - Update error handling to return `PermissionDenied` if the path validation fails. - Import `errs` package in `offline_download` for consistent error responses. * ✨ feat(role-permission): Implement path-based role permission checks - Add `CheckPathLimitWithRoles` function to validate access based on `PermPathLimit` permission. - Integrate `CheckPathLimitWithRoles` in `offline_download` to enforce path-based access control. - Apply `CheckPathLimitWithRoles` across file system management operations (e.g., creation, movement, deletion). - Ensure `CheckPathLimitWithRoles` is invoked for batch operations and archive-related actions. - Update error handling to return `PermissionDenied` if the path validation fails. - Import `errs` package in `offline_download` for consistent error responses. * ♻️ refactor(access-control): Update access control logic to use role-based checks - Remove deprecated logic from `CanAccess` function in `check.go`, replacing it with `CanAccessWithRoles` for improved role-based access control. - Modify calls in `search.go` to use `CanAccessWithRoles` for more precise handling of permissions. - Update `fsread.go` to utilize `CanAccessWithRoles`, ensuring accurate access validation based on user roles. - Simplify import statements in `check.go` by removing unused packages to clean up the codebase. * ✨ feat(fs): Improve visibility logic for hidden files - Import `server/common` package to handle permissions more robustly - Update `whetherHide` function to use `MergeRolePermissions` for user-specific path permissions - Replace direct user checks with `HasPermission` for `PermSeeHides` - Enhance logic to ensure `nil` user cases are handled explicitly * 标签管理 * feat(db/auth/user): Enhance role handling and clean permission paths - Comment out role modification checks in `server/handles/user.go` to allow flexible role changes. - Improve permission path handling in `server/handles/auth.go` by normalizing and deduplicating paths. - Introduce `addedPaths` map in `CurrentUser` to prevent duplicate permissions. * feat(storage/db): Implement role permissions path prefix update - Add `UpdateRolePermissionsPathPrefix` function in `role.go` to update role permissions paths. - Modify `storage.go` to call the new function when the mount path is renamed. - Introduce path cleaning and prefix matching logic for accurate path updates. - Ensure roles are updated only if their permission scopes are modified. - Handle potential errors with informative messages during database operations. * feat(role-migration): Implement role conversion and introduce NEWGENERAL role - Add `NEWGENERAL` to the roles enumeration in `user.go` - Create new file `convert_role.go` for migrating legacy roles to new model - Implement `ConvertLegacyRoles` function to handle role conversion with permission scopes - Add `convert_role.go` patch to `all.go` under version `v3.46.0` * feat(role/auth): Add role retrieval by user ID and update path prefixes - Add `GetRolesByUserID` function for efficient role retrieval by user ID - Implement `UpdateUserBasePathPrefix` to update user base paths - Modify `UpdateRolePermissionsPathPrefix` to return modified role IDs - Update `auth.go` middleware to use the new role retrieval function - Refresh role and user caches upon path prefix updates to maintain consistency --------- Co-authored-by: Leslie-Xy <540049476@qq.com> --- drivers/alist_v3/driver.go | 2 +- drivers/alist_v3/types.go | 2 +- drivers/quqi/types.go | 2 +- internal/bootstrap/data/data.go | 1 + internal/bootstrap/data/dev.go | 2 +- internal/bootstrap/data/role.go | 52 ++++++ internal/bootstrap/data/user.go | 50 +++--- internal/bootstrap/patch/all.go | 7 + .../bootstrap/patch/v3_46_0/convert_role.go | 129 ++++++++++++++ internal/db/db.go | 2 +- internal/db/label.go | 79 +++++++++ internal/db/label_file_binding.go | 56 ++++++ internal/db/obj_file.go | 31 ++++ internal/db/role.go | 79 +++++++++ internal/db/user.go | 48 +++++- internal/errs/role.go | 7 + internal/fs/list.go | 10 +- internal/model/label.go | 12 ++ internal/model/label_file_binding.go | 11 ++ internal/model/obj_file.go | 18 ++ internal/model/paths.go | 27 +++ internal/model/role.go | 52 ++++++ internal/model/roles.go | 36 ++++ internal/model/user.go | 38 +++-- internal/op/label.go | 24 +++ internal/op/label_file_binding.go | 159 ++++++++++++++++++ internal/op/role.go | 121 +++++++++++++ internal/op/storage.go | 15 ++ internal/op/user.go | 12 +- server/common/check.go | 33 +--- server/common/role_perm.go | 108 ++++++++++++ server/ftp.go | 4 +- server/ftp/fsmanage.go | 11 +- server/ftp/fsread.go | 6 +- server/ftp/fsup.go | 6 +- server/handles/archive.go | 43 +++-- server/handles/auth.go | 31 +++- server/handles/fsbatch.go | 44 +++-- server/handles/fsmanage.go | 80 ++++++--- server/handles/fsread.go | 66 +++++--- server/handles/label.go | 99 +++++++++++ server/handles/label_file_binding.go | 103 ++++++++++++ server/handles/ldap_login.go | 2 +- server/handles/offline_download.go | 14 +- server/handles/role.go | 101 +++++++++++ server/handles/search.go | 2 +- server/handles/ssologin.go | 2 +- server/handles/task.go | 4 +- server/handles/user.go | 8 +- server/middlewares/auth.go | 23 +++ server/middlewares/fsup.go | 4 +- server/router.go | 20 +++ server/sftp.go | 10 +- server/webdav.go | 31 +++- server/webdav/file.go | 6 +- 55 files changed, 1762 insertions(+), 183 deletions(-) create mode 100644 internal/bootstrap/data/role.go create mode 100644 internal/bootstrap/patch/v3_46_0/convert_role.go create mode 100644 internal/db/label.go create mode 100644 internal/db/label_file_binding.go create mode 100644 internal/db/obj_file.go create mode 100644 internal/db/role.go create mode 100644 internal/errs/role.go create mode 100644 internal/model/label.go create mode 100644 internal/model/label_file_binding.go create mode 100644 internal/model/obj_file.go create mode 100644 internal/model/paths.go create mode 100644 internal/model/role.go create mode 100644 internal/model/roles.go create mode 100644 internal/op/label.go create mode 100644 internal/op/label_file_binding.go create mode 100644 internal/op/role.go create mode 100644 server/common/role_perm.go create mode 100644 server/handles/label.go create mode 100644 server/handles/label_file_binding.go create mode 100644 server/handles/role.go diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index ac7e16a1d16..56f9c01e124 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -56,7 +56,7 @@ func (d *AListV3) Init(ctx context.Context) error { if err != nil { return err } - if resp.Data.Role == model.GUEST { + if utils.SliceContains(resp.Data.Role, model.GUEST) { u := d.Address + "/api/public/settings" res, err := base.RestyClient.R().Get(u) if err != nil { diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go index 1ae7926e078..83ecde8be17 100644 --- a/drivers/alist_v3/types.go +++ b/drivers/alist_v3/types.go @@ -76,7 +76,7 @@ type MeResp struct { Username string `json:"username"` Password string `json:"password"` BasePath string `json:"base_path"` - Role int `json:"role"` + Role []int `json:"role"` Disabled bool `json:"disabled"` Permission int `json:"permission"` SsoId string `json:"sso_id"` diff --git a/drivers/quqi/types.go b/drivers/quqi/types.go index 32557361532..cade93de885 100644 --- a/drivers/quqi/types.go +++ b/drivers/quqi/types.go @@ -83,7 +83,7 @@ type Group struct { Type int `json:"type"` Name string `json:"name"` IsAdministrator int `json:"is_administrator"` - Role int `json:"role"` + Role []int `json:"role"` Avatar string `json:"avatar_url"` IsStick int `json:"is_stick"` Nickname string `json:"nickname"` diff --git a/internal/bootstrap/data/data.go b/internal/bootstrap/data/data.go index c2170d2f479..1f0a5909a58 100644 --- a/internal/bootstrap/data/data.go +++ b/internal/bootstrap/data/data.go @@ -3,6 +3,7 @@ package data import "github.com/alist-org/alist/v3/cmd/flags" func InitData() { + initRoles() initUser() initSettings() initTasks() diff --git a/internal/bootstrap/data/dev.go b/internal/bootstrap/data/dev.go index f6296c9e96a..74097dbd8b7 100644 --- a/internal/bootstrap/data/dev.go +++ b/internal/bootstrap/data/dev.go @@ -26,7 +26,7 @@ func initDevData() { Username: "Noah", Password: "hsu", BasePath: "/data", - Role: 0, + Role: nil, Permission: 512, }) if err != nil { diff --git a/internal/bootstrap/data/role.go b/internal/bootstrap/data/role.go new file mode 100644 index 00000000000..a82fa2afcc3 --- /dev/null +++ b/internal/bootstrap/data/role.go @@ -0,0 +1,52 @@ +package data + +// initRoles creates the default admin and guest roles if missing. +// These roles are essential and must not be modified or removed. + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +func initRoles() { + guestRole, err := op.GetRoleByName("guest") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + guestRole = &model.Role{ + ID: uint(model.GUEST), + Name: "guest", + Description: "Guest", + PermissionScopes: []model.PermissionEntry{ + {Path: "/", Permission: 0}, + }, + } + if err := op.CreateRole(guestRole); err != nil { + utils.Log.Fatalf("[init role] Failed to create guest role: %v", err) + } + } else { + utils.Log.Fatalf("[init role] Failed to get guest role: %v", err) + } + } + + _, err = op.GetRoleByName("admin") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + adminRole := &model.Role{ + ID: uint(model.ADMIN), + Name: "admin", + Description: "Administrator", + PermissionScopes: []model.PermissionEntry{ + {Path: "/", Permission: 0xFFFF}, + }, + } + if err := op.CreateRole(adminRole); err != nil { + utils.Log.Fatalf("[init role] Failed to create admin role: %v", err) + } + } else { + utils.Log.Fatalf("[init role] Failed to get admin role: %v", err) + } + } +} diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index 9c3f8962ad3..6851118668b 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -1,10 +1,10 @@ package data import ( + "github.com/alist-org/alist/v3/internal/db" "os" "github.com/alist-org/alist/v3/cmd/flags" - "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" @@ -14,6 +14,28 @@ import ( ) func initUser() { + guest, err := op.GetGuest() + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + salt := random.String(16) + guestRole, _ := op.GetRoleByName("guest") + guest = &model.User{ + Username: "guest", + PwdHash: model.TwoHashPwd("guest", salt), + Salt: salt, + Role: model.Roles{int(guestRole.ID)}, + BasePath: "/", + Permission: 0, + Disabled: true, + Authn: "[]", + } + if err := db.CreateUser(guest); err != nil { + utils.Log.Fatalf("[init user] Failed to create guest user: %v", err) + } + } else { + utils.Log.Fatalf("[init user] Failed to get guest user: %v", err) + } + } admin, err := op.GetAdmin() adminPassword := random.String(8) envpass := os.Getenv("ALIST_ADMIN_PASSWORD") @@ -25,15 +47,16 @@ func initUser() { if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { salt := random.String(16) + adminRole, _ := op.GetRoleByName("admin") admin = &model.User{ Username: "admin", Salt: salt, PwdHash: model.TwoHashPwd(adminPassword, salt), - Role: model.ADMIN, + Role: model.Roles{int(adminRole.ID)}, BasePath: "/", Authn: "[]", // 0(can see hidden) - 7(can remove) & 12(can read archives) - 13(can decompress archives) - Permission: 0x30FF, + Permission: 0xFFFF, } if err := op.CreateUser(admin); err != nil { panic(err) @@ -44,25 +67,4 @@ func initUser() { utils.Log.Fatalf("[init user] Failed to get admin user: %v", err) } } - guest, err := op.GetGuest() - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - salt := random.String(16) - guest = &model.User{ - Username: "guest", - PwdHash: model.TwoHashPwd("guest", salt), - Salt: salt, - Role: model.GUEST, - BasePath: "/", - Permission: 0, - Disabled: true, - Authn: "[]", - } - if err := db.CreateUser(guest); err != nil { - utils.Log.Fatalf("[init user] Failed to create guest user: %v", err) - } - } else { - utils.Log.Fatalf("[init user] Failed to get guest user: %v", err) - } - } } diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go index b363d12981d..eb679147ec9 100644 --- a/internal/bootstrap/patch/all.go +++ b/internal/bootstrap/patch/all.go @@ -4,6 +4,7 @@ import ( "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_24_0" "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_32_0" "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_41_0" + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_46_0" ) type VersionPatches struct { @@ -32,4 +33,10 @@ var UpgradePatches = []VersionPatches{ v3_41_0.GrantAdminPermissions, }, }, + { + Version: "v3.46.0", + Patches: []func(){ + v3_46_0.ConvertLegacyRoles, + }, + }, } diff --git a/internal/bootstrap/patch/v3_46_0/convert_role.go b/internal/bootstrap/patch/v3_46_0/convert_role.go new file mode 100644 index 00000000000..43799485c12 --- /dev/null +++ b/internal/bootstrap/patch/v3_46_0/convert_role.go @@ -0,0 +1,129 @@ +package v3_46_0 + +import ( + "errors" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "gorm.io/gorm" +) + +// ConvertLegacyRoles migrates old integer role values to a new role model with permission scopes. +func ConvertLegacyRoles() { + guestRole, err := op.GetRoleByName("guest") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + guestRole = &model.Role{ + ID: uint(model.GUEST), + Name: "guest", + Description: "Guest", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0, + }, + }, + } + if err = op.CreateRole(guestRole); err != nil { + utils.Log.Errorf("[convert roles] failed to create guest role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed to get guest role: %v", err) + return + } + } + + adminRole, err := op.GetRoleByName("admin") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + adminRole = &model.Role{ + ID: uint(model.ADMIN), + Name: "admin", + Description: "Administrator", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0x33FF, + }, + }, + } + if err = op.CreateRole(adminRole); err != nil { + utils.Log.Errorf("[convert roles] failed to create admin role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed to get admin role: %v", err) + return + } + } + + generalRole, err := op.GetRoleByName("general") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + generalRole = &model.Role{ + ID: uint(model.NEWGENERAL), + Name: "general", + Description: "General User", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0, + }, + }, + } + if err = op.CreateRole(generalRole); err != nil { + utils.Log.Errorf("[convert roles] failed create general role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed get general role: %v", err) + return + } + } + + users, _, err := op.GetUsers(1, -1) + if err != nil { + utils.Log.Errorf("[convert roles] failed to get users: %v", err) + return + } + + for i := range users { + user := users[i] + if user.Role == nil { + continue + } + changed := false + var roles model.Roles + for _, r := range user.Role { + switch r { + case model.ADMIN: + roles = append(roles, int(adminRole.ID)) + if int(adminRole.ID) != r { + changed = true + } + case model.GUEST: + roles = append(roles, int(guestRole.ID)) + if int(guestRole.ID) != r { + changed = true + } + case model.GENERAL: + roles = append(roles, int(generalRole.ID)) + if int(generalRole.ID) != r { + changed = true + } + default: + roles = append(roles, r) + } + } + if changed { + user.Role = roles + if err := db.UpdateUser(&user); err != nil { + utils.Log.Errorf("[convert roles] failed to update user %s: %v", user.Username, err) + } + } + } + + utils.Log.Infof("[convert roles] completed role conversion for %d users", len(users)) +} diff --git a/internal/db/db.go b/internal/db/db.go index 2cd18050da9..c6491dc984f 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinDing), new(model.ObjFile)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/label.go b/internal/db/label.go new file mode 100644 index 00000000000..fd9842d680c --- /dev/null +++ b/internal/db/label.go @@ -0,0 +1,79 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" + "time" +) + +// GetLabels Get all label from database order by id +func GetLabels(pageIndex, pageSize int) ([]model.Label, int64, error) { + labelDB := db.Model(&model.Label{}) + var count int64 + if err := labelDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get label count") + } + var labels []model.Label + if err := labelDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&labels).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + return labels, count, nil +} + +// GetLabelById Get Label by id, used to update label usually +func GetLabelById(id uint) (*model.Label, error) { + var label model.Label + label.ID = id + if err := db.First(&label).Error; err != nil { + return nil, errors.WithStack(err) + } + return &label, nil +} + +// CreateLabel just insert label to database +func CreateLabel(label model.Label) (uint, error) { + label.CreateTime = time.Now() + err := errors.WithStack(db.Create(&label).Error) + if err != nil { + return label.ID, errors.WithMessage(err, "failed create label in database") + } + return label.ID, nil +} + +// UpdateLabel just update storage in database +func UpdateLabel(label *model.Label) (*model.Label, error) { + label.CreateTime = time.Now() + _, err := GetLabelById(label.ID) + if err != nil { + return nil, errors.WithMessage(err, "failed get old label") + } + err = errors.WithStack(db.Save(label).Error) + if err != nil { + return nil, errors.WithMessage(err, "failed create label in database") + } + return label, nil +} + +// DeleteLabelById just delete label from database by id +func DeleteLabelById(id uint) error { + return errors.WithStack(db.Delete(&model.Label{}, id).Error) +} + +// GetLabelByIds Get label from database order by ids +func GetLabelByIds(ids []uint) ([]model.Label, error) { + labelDB := db.Model(&model.Label{}) + var labels []model.Label + if err := labelDB.Where(ids).Find(&labels).Error; err != nil { + return nil, errors.WithStack(err) + } + return labels, nil +} + +// GetLabelByName Get Label by name +func GetLabelByName(name string) bool { + var label model.Label + result := db.Where("name = ?", name).First(&label) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} diff --git a/internal/db/label_file_binding.go b/internal/db/label_file_binding.go new file mode 100644 index 00000000000..ec722efb979 --- /dev/null +++ b/internal/db/label_file_binding.go @@ -0,0 +1,56 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" + "time" +) + +// GetLabelIds Get all label_ids from database order by file_name +func GetLabelIds(userId uint, fileName string) ([]uint, error) { + labelFileBinDingDB := db.Model(&model.LabelFileBinDing{}) + var labelIds []uint + if err := labelFileBinDingDB.Where("file_name = ?", fileName).Where("user_id = ?", userId).Pluck("label_id", &labelIds).Error; err != nil { + return nil, errors.WithStack(err) + } + return labelIds, nil +} + +func CreateLabelFileBinDing(fileName string, labelId, userId uint) error { + var labelFileBinDing model.LabelFileBinDing + labelFileBinDing.UserId = userId + labelFileBinDing.LabelId = labelId + labelFileBinDing.FileName = fileName + labelFileBinDing.CreateTime = time.Now() + err := errors.WithStack(db.Create(&labelFileBinDing).Error) + if err != nil { + return errors.WithMessage(err, "failed create label in database") + } + return nil +} + +// GetLabelFileBinDingByLabelIdExists Get Label by label_id, used to del label usually +func GetLabelFileBinDingByLabelIdExists(labelId, userId uint) bool { + var labelFileBinDing model.LabelFileBinDing + result := db.Where("label_id = ?", labelId).Where("user_id = ?", userId).First(&labelFileBinDing) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} + +// DelLabelFileBinDingByFileName used to del usually +func DelLabelFileBinDingByFileName(userId uint, fileName string) error { + return errors.WithStack(db.Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinDing{}).Error) +} + +// DelLabelFileBinDingById used to del usually +func DelLabelFileBinDingById(labelId, userId uint, fileName string) error { + return errors.WithStack(db.Where("label_id = ?", labelId).Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinDing{}).Error) +} + +func GetLabelFileBinDingByLabelId(labelIds []uint, userId uint) (result []model.LabelFileBinDing, err error) { + if err := db.Where("label_id in (?)", labelIds).Where("user_id = ?", userId).Find(&result).Error; err != nil { + return nil, errors.WithStack(err) + } + return result, nil +} diff --git a/internal/db/obj_file.go b/internal/db/obj_file.go new file mode 100644 index 00000000000..2bbce9e6dd6 --- /dev/null +++ b/internal/db/obj_file.go @@ -0,0 +1,31 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// GetFileByNameExists Get file by name +func GetFileByNameExists(name string) bool { + var label model.ObjFile + result := db.Where("name = ?", name).First(&label) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} + +// GetFileByName Get file by name +func GetFileByName(name string, userId uint) (objFile model.ObjFile, err error) { + if err = db.Where("name = ?", name).Where("user_id = ?", userId).First(&objFile).Error; err != nil { + return objFile, errors.WithStack(err) + } + return objFile, nil +} + +func CreateObjFile(obj model.ObjFile) error { + err := errors.WithStack(db.Create(&obj).Error) + if err != nil { + return errors.WithMessage(err, "failed create file in database") + } + return nil +} diff --git a/internal/db/role.go b/internal/db/role.go new file mode 100644 index 00000000000..e6d0d9568b4 --- /dev/null +++ b/internal/db/role.go @@ -0,0 +1,79 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "path" + "strings" +) + +func GetRole(id uint) (*model.Role, error) { + var r model.Role + if err := db.First(&r, id).Error; err != nil { + return nil, errors.Wrapf(err, "failed get role") + } + return &r, nil +} + +func GetRoleByName(name string) (*model.Role, error) { + r := model.Role{Name: name} + if err := db.Where(r).First(&r).Error; err != nil { + return nil, errors.Wrapf(err, "failed get role") + } + return &r, nil +} + +func GetRoles(pageIndex, pageSize int) (roles []model.Role, count int64, err error) { + roleDB := db.Model(&model.Role{}) + if err = roleDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get roles count") + } + if err = roleDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&roles).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get find roles") + } + return roles, count, nil +} + +func CreateRole(r *model.Role) error { + return errors.WithStack(db.Create(r).Error) +} + +func UpdateRole(r *model.Role) error { + return errors.WithStack(db.Save(r).Error) +} + +func DeleteRole(id uint) error { + return errors.WithStack(db.Delete(&model.Role{}, id).Error) +} + +func UpdateRolePermissionsPathPrefix(oldPath, newPath string) ([]uint, error) { + var roles []model.Role + var modifiedRoleIDs []uint + + if err := db.Find(&roles).Error; err != nil { + return nil, errors.WithMessage(err, "failed to load roles") + } + + for _, role := range roles { + updated := false + for i, entry := range role.PermissionScopes { + entryPath := path.Clean(entry.Path) + oldPathClean := path.Clean(oldPath) + + if entryPath == oldPathClean { + role.PermissionScopes[i].Path = newPath + updated = true + } else if strings.HasPrefix(entryPath, oldPathClean+"/") { + role.PermissionScopes[i].Path = newPath + entryPath[len(oldPathClean):] + updated = true + } + } + if updated { + if err := UpdateRole(&role); err != nil { + return nil, errors.WithMessagef(err, "failed to update role ID %d", role.ID) + } + modifiedRoleIDs = append(modifiedRoleIDs, role.ID) + } + } + return modifiedRoleIDs, nil +} diff --git a/internal/db/user.go b/internal/db/user.go index 8c9641b2c55..f2b6635a983 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -2,19 +2,26 @@ package db import ( "encoding/base64" - "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-webauthn/webauthn/webauthn" "github.com/pkg/errors" + "gorm.io/gorm" + "path" + "strings" ) func GetUserByRole(role int) (*model.User, error) { - user := model.User{Role: role} - if err := db.Where(user).Take(&user).Error; err != nil { + var users []model.User + if err := db.Find(&users).Error; err != nil { return nil, err } - return &user, nil + for i := range users { + if users[i].Role.Contains(role) { + return &users[i], nil + } + } + return nil, gorm.ErrRecordNotFound } func GetUserByName(username string) (*model.User, error) { @@ -100,3 +107,36 @@ func RemoveAuthn(u *model.User, id string) error { } return UpdateAuthn(u.ID, string(res)) } + +func UpdateUserBasePathPrefix(oldPath, newPath string) ([]string, error) { + var users []model.User + var modifiedUsernames []string + + if err := db.Find(&users).Error; err != nil { + return nil, errors.WithMessage(err, "failed to load users") + } + + oldPathClean := path.Clean(oldPath) + + for _, user := range users { + basePath := path.Clean(user.BasePath) + updated := false + + if basePath == oldPathClean { + user.BasePath = newPath + updated = true + } else if strings.HasPrefix(basePath, oldPathClean+"/") { + user.BasePath = newPath + basePath[len(oldPathClean):] + updated = true + } + + if updated { + if err := UpdateUser(&user); err != nil { + return nil, errors.WithMessagef(err, "failed to update user ID %d", user.ID) + } + modifiedUsernames = append(modifiedUsernames, user.Username) + } + } + + return modifiedUsernames, nil +} diff --git a/internal/errs/role.go b/internal/errs/role.go new file mode 100644 index 00000000000..fbd67404822 --- /dev/null +++ b/internal/errs/role.go @@ -0,0 +1,7 @@ +package errs + +import "errors" + +var ( + ErrChangeDefaultRole = errors.New("cannot modify admin or guest role") +) diff --git a/internal/fs/list.go b/internal/fs/list.go index d4f59cb829f..927b6ead1a4 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -6,6 +6,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -45,8 +46,13 @@ func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) } func whetherHide(user *model.User, meta *model.Meta, path string) bool { - // if is admin, don't hide - if user == nil || user.CanSeeHides() { + // if user is nil, don't hide + if user == nil { + return false + } + perm := common.MergeRolePermissions(user, path) + // if user has see-hides permission, don't hide + if common.HasPermission(perm, common.PermSeeHides) { return false } // if meta is nil, don't hide diff --git a/internal/model/label.go b/internal/model/label.go new file mode 100644 index 00000000000..b397542f5c3 --- /dev/null +++ b/internal/model/label.go @@ -0,0 +1,12 @@ +package model + +import "time" + +type Label struct { + ID uint `json:"id" gorm:"primaryKey"` // unique key + Type int `json:"type"` // use to type + Name string `json:"name"` // use to name + Description string `json:"description"` // use to description + BgColor string `json:"bg_color"` // use to bg_color + CreateTime time.Time `json:"create_time"` +} diff --git a/internal/model/label_file_binding.go b/internal/model/label_file_binding.go new file mode 100644 index 00000000000..3f9ea3b2271 --- /dev/null +++ b/internal/model/label_file_binding.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type LabelFileBinDing struct { + ID uint `json:"id" gorm:"primaryKey"` // unique key + UserId uint `json:"user_id"` // use to user_id + LabelId uint `json:"label_id"` // use to label_id + FileName string `json:"file_name"` // use to file_name + CreateTime time.Time `json:"create_time"` +} diff --git a/internal/model/obj_file.go b/internal/model/obj_file.go new file mode 100644 index 00000000000..0fccd6b5cba --- /dev/null +++ b/internal/model/obj_file.go @@ -0,0 +1,18 @@ +package model + +import "time" + +type ObjFile struct { + Id string `json:"id"` + UserId uint `json:"user_id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` +} diff --git a/internal/model/paths.go b/internal/model/paths.go new file mode 100644 index 00000000000..8403de8e6a2 --- /dev/null +++ b/internal/model/paths.go @@ -0,0 +1,27 @@ +package model + +import ( + "database/sql/driver" + "encoding/json" + "fmt" +) + +type Paths []string + +func (p Paths) Value() (driver.Value, error) { + return json.Marshal([]string(p)) +} + +func (p *Paths) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, (*[]string)(p)) + case string: + return json.Unmarshal([]byte(v), (*[]string)(p)) + case nil: + *p = nil + return nil + default: + return fmt.Errorf("cannot scan %T", value) + } +} diff --git a/internal/model/role.go b/internal/model/role.go new file mode 100644 index 00000000000..ecc9aee29d1 --- /dev/null +++ b/internal/model/role.go @@ -0,0 +1,52 @@ +package model + +import ( + "encoding/json" + + "gorm.io/gorm" +) + +// PermissionEntry defines permission bitmask for a specific path. +type PermissionEntry struct { + Path string `json:"path"` // path prefix, e.g. "/admin" + Permission int32 `json:"permission"` // bitmask permissions +} + +// Role represents a permission template which can be bound to users. +type Role struct { + ID uint `json:"id" gorm:"primaryKey"` + Name string `json:"name" gorm:"unique" binding:"required"` + Description string `json:"description"` + // PermissionScopes stores structured permission list and is ignored by gorm. + PermissionScopes []PermissionEntry `json:"permission_scopes" gorm:"-"` + // RawPermission is the JSON representation of PermissionScopes stored in DB. + RawPermission string `json:"-" gorm:"type:text"` +} + +// BeforeSave GORM hook serializes PermissionScopes into RawPermission. +func (r *Role) BeforeSave(tx *gorm.DB) error { + if len(r.PermissionScopes) == 0 { + r.RawPermission = "" + return nil + } + bs, err := json.Marshal(r.PermissionScopes) + if err != nil { + return err + } + r.RawPermission = string(bs) + return nil +} + +// AfterFind GORM hook deserializes RawPermission into PermissionScopes. +func (r *Role) AfterFind(tx *gorm.DB) error { + if r.RawPermission == "" { + r.PermissionScopes = nil + return nil + } + var scopes []PermissionEntry + if err := json.Unmarshal([]byte(r.RawPermission), &scopes); err != nil { + return err + } + r.PermissionScopes = scopes + return nil +} diff --git a/internal/model/roles.go b/internal/model/roles.go new file mode 100644 index 00000000000..eb626cb93f7 --- /dev/null +++ b/internal/model/roles.go @@ -0,0 +1,36 @@ +package model + +import ( + "database/sql/driver" + "encoding/json" + "fmt" +) + +type Roles []int + +func (r Roles) Value() (driver.Value, error) { + return json.Marshal([]int(r)) +} + +func (r *Roles) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, (*[]int)(r)) + case string: + return json.Unmarshal([]byte(v), (*[]int)(r)) + case nil: + *r = nil + return nil + default: + return fmt.Errorf("cannot scan %T", value) + } +} + +func (r Roles) Contains(role int) bool { + for _, v := range r { + if v == role { + return true + } + } + return false +} diff --git a/internal/model/user.go b/internal/model/user.go index 0f7d3af5064..0b9e576a484 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -17,20 +17,22 @@ const ( GENERAL = iota GUEST // only one exists ADMIN + NEWGENERAL ) const StaticHashSalt = "https://github.com/alist-org/alist" type User struct { - ID uint `json:"id" gorm:"primaryKey"` // unique key - Username string `json:"username" gorm:"unique" binding:"required"` // username - PwdHash string `json:"-"` // password hash - PwdTS int64 `json:"-"` // password timestamp - Salt string `json:"-"` // unique salt - Password string `json:"password"` // password - BasePath string `json:"base_path"` // base path - Role int `json:"role"` // user's role - Disabled bool `json:"disabled"` + ID uint `json:"id" gorm:"primaryKey"` // unique key + Username string `json:"username" gorm:"unique" binding:"required"` // username + PwdHash string `json:"-"` // password hash + PwdTS int64 `json:"-"` // password timestamp + Salt string `json:"-"` // unique salt + Password string `json:"password"` // password + BasePath string `json:"base_path"` // base path + Role Roles `json:"role" gorm:"type:text"` // user's roles + RolesDetail []Role `json:"-" gorm:"-"` + Disabled bool `json:"disabled"` // Determine permissions by bit // 0: can see hidden files // 1: can access without password @@ -46,6 +48,7 @@ type User struct { // 11: ftp/sftp write // 12: can read archives // 13: can decompress archives + // 14: check path limit Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -53,11 +56,11 @@ type User struct { } func (u *User) IsGuest() bool { - return u.Role == GUEST + return u.Role.Contains(GUEST) } func (u *User) IsAdmin() bool { - return u.Role == ADMIN + return u.Role.Contains(ADMIN) } func (u *User) ValidateRawPassword(password string) error { @@ -137,8 +140,19 @@ func (u *User) CanDecompress() bool { return (u.Permission>>13)&1 == 1 } +func (u *User) CheckPathLimit() bool { + return (u.Permission>>14)&1 == 1 +} + func (u *User) JoinPath(reqPath string) (string, error) { - return utils.JoinBasePath(u.BasePath, reqPath) + path, err := utils.JoinBasePath(u.BasePath, reqPath) + if err != nil { + return "", err + } + if u.CheckPathLimit() && !utils.IsSubPath(u.BasePath, path) { + return "", errs.PermissionDenied + } + return path, nil } func StaticHash(password string) string { diff --git a/internal/op/label.go b/internal/op/label.go new file mode 100644 index 00000000000..7e913edfa8d --- /dev/null +++ b/internal/op/label.go @@ -0,0 +1,24 @@ +package op + +import ( + "context" + "github.com/alist-org/alist/v3/internal/db" + "github.com/pkg/errors" +) + +func DeleteLabelById(ctx context.Context, id, userId uint) error { + _, err := db.GetLabelById(id) + if err != nil { + return errors.WithMessage(err, "failed get label") + } + + if db.GetLabelFileBinDingByLabelIdExists(id, userId) { + return errors.New("label have binding relationships") + } + + // delete the label in the database + if err := db.DeleteLabelById(id); err != nil { + return errors.WithMessage(err, "failed delete label in database") + } + return nil +} diff --git a/internal/op/label_file_binding.go b/internal/op/label_file_binding.go new file mode 100644 index 00000000000..79137ed38af --- /dev/null +++ b/internal/op/label_file_binding.go @@ -0,0 +1,159 @@ +package op + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "strconv" + "strings" + "time" +) + +type CreateLabelFileBinDingReq struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + LabelIds string `json:"label_ids"` +} + +type ObjLabelResp struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + LabelList []model.Label `json:"label_list"` +} + +func GetLabelByFileName(userId uint, fileName string) ([]model.Label, error) { + labelIds, err := db.GetLabelIds(userId, fileName) + if err != nil { + return nil, errors.WithMessage(err, "failed get label_file_binding") + } + var labels []model.Label + if len(labelIds) > 0 { + if labels, err = db.GetLabelByIds(labelIds); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + } + return labels, nil +} + +func CreateLabelFileBinDing(req CreateLabelFileBinDingReq, userId uint) error { + if err := db.DelLabelFileBinDingByFileName(userId, req.Name); err != nil { + return errors.WithMessage(err, "failed del label_file_bin_ding in database") + } + if req.LabelIds == "" { + return nil + } + labelMap := strings.Split(req.LabelIds, ",") + for _, value := range labelMap { + labelId, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return fmt.Errorf("invalid label ID '%s': %v", value, err) + } + if err = db.CreateLabelFileBinDing(req.Name, uint(labelId), userId); err != nil { + return errors.WithMessage(err, "failed labels in database") + } + } + if !db.GetFileByNameExists(req.Name) { + objFile := model.ObjFile{ + Id: req.Id, + UserId: userId, + Path: req.Path, + Name: req.Name, + Size: req.Size, + IsDir: req.IsDir, + Modified: req.Modified, + Created: req.Created, + Sign: req.Sign, + Thumb: req.Thumb, + Type: req.Type, + HashInfoStr: req.HashInfoStr, + } + err := db.CreateObjFile(objFile) + if err != nil { + return errors.WithMessage(err, "failed file in database") + } + } + return nil +} + +func GetFileByLabel(userId uint, labelId string) (result []ObjLabelResp, err error) { + labelMap := strings.Split(labelId, ",") + var labelIds []uint + var labelsFile []model.LabelFileBinDing + var labels []model.Label + var labelsFileMap = make(map[string][]model.Label) + var labelsMap = make(map[uint]model.Label) + if labelIds, err = StringSliceToUintSlice(labelMap); err != nil { + return nil, errors.WithMessage(err, "failed string to uint err") + } + //查询标签信息 + if labels, err = db.GetLabelByIds(labelIds); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + for _, val := range labels { + labelsMap[val.ID] = val + } + //查询标签对应文件名列表 + if labelsFile, err = db.GetLabelFileBinDingByLabelId(labelIds, userId); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + for _, value := range labelsFile { + var labelTemp model.Label + labelTemp = labelsMap[value.LabelId] + labelsFileMap[value.FileName] = append(labelsFileMap[value.FileName], labelTemp) + } + for index, v := range labelsFileMap { + objFile, err := db.GetFileByName(index, userId) + if err != nil { + return nil, errors.WithMessage(err, "failed GetFileByName in database") + } + objLabel := ObjLabelResp{ + Id: objFile.Id, + Path: objFile.Path, + Name: objFile.Name, + Size: objFile.Size, + IsDir: objFile.IsDir, + Modified: objFile.Modified, + Created: objFile.Created, + Sign: objFile.Sign, + Thumb: objFile.Thumb, + Type: objFile.Type, + HashInfoStr: objFile.HashInfoStr, + LabelList: v, + } + result = append(result, objLabel) + } + return result, nil +} + +func StringSliceToUintSlice(strSlice []string) ([]uint, error) { + uintSlice := make([]uint, len(strSlice)) + for i, str := range strSlice { + // 使用strconv.ParseUint将字符串转换为uint64 + uint64Value, err := strconv.ParseUint(str, 10, 64) + if err != nil { + return nil, err // 如果转换失败，返回错误 + } + // 将uint64值转换为uint（注意：这里可能存在精度损失，如果uint64值超出了uint的范围） + uintSlice[i] = uint(uint64Value) + } + return uintSlice, nil +} diff --git a/internal/op/role.go b/internal/op/role.go new file mode 100644 index 00000000000..64502f98df4 --- /dev/null +++ b/internal/op/role.go @@ -0,0 +1,121 @@ +package op + +import ( + "fmt" + "time" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/singleflight" + "github.com/alist-org/alist/v3/pkg/utils" +) + +var roleCache = cache.NewMemCache[*model.Role](cache.WithShards[*model.Role](2)) +var roleG singleflight.Group[*model.Role] + +func GetRole(id uint) (*model.Role, error) { + key := fmt.Sprint(id) + if r, ok := roleCache.Get(key); ok { + return r, nil + } + r, err, _ := roleG.Do(key, func() (*model.Role, error) { + _r, err := db.GetRole(id) + if err != nil { + return nil, err + } + roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + return r, err +} + +func GetRoleByName(name string) (*model.Role, error) { + if r, ok := roleCache.Get(name); ok { + return r, nil + } + r, err, _ := roleG.Do(name, func() (*model.Role, error) { + _r, err := db.GetRoleByName(name) + if err != nil { + return nil, err + } + roleCache.Set(name, _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + return r, err +} + +func GetRolesByUserID(userID uint) ([]model.Role, error) { + user, err := GetUserById(userID) + if err != nil { + return nil, err + } + + var roles []model.Role + for _, roleID := range user.Role { + key := fmt.Sprint(roleID) + + if r, ok := roleCache.Get(key); ok { + roles = append(roles, *r) + continue + } + + r, err, _ := roleG.Do(key, func() (*model.Role, error) { + _r, err := db.GetRole(uint(roleID)) + if err != nil { + return nil, err + } + roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + if err != nil { + return nil, err + } + roles = append(roles, *r) + } + + return roles, nil +} + +func GetRoles(pageIndex, pageSize int) ([]model.Role, int64, error) { + return db.GetRoles(pageIndex, pageSize) +} + +func CreateRole(r *model.Role) error { + for i := range r.PermissionScopes { + r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) + } + roleCache.Del(fmt.Sprint(r.ID)) + roleCache.Del(r.Name) + return db.CreateRole(r) +} + +func UpdateRole(r *model.Role) error { + old, err := db.GetRole(r.ID) + if err != nil { + return err + } + if old.Name == "admin" || old.Name == "guest" { + return errs.ErrChangeDefaultRole + } + for i := range r.PermissionScopes { + r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) + } + roleCache.Del(fmt.Sprint(r.ID)) + roleCache.Del(r.Name) + return db.UpdateRole(r) +} + +func DeleteRole(id uint) error { + old, err := db.GetRole(id) + if err != nil { + return err + } + if old.Name == "admin" || old.Name == "guest" { + return errs.ErrChangeDefaultRole + } + roleCache.Del(fmt.Sprint(id)) + roleCache.Del(old.Name) + return db.DeleteRole(id) +} diff --git a/internal/op/storage.go b/internal/op/storage.go index f957f95b596..2ec68aae5e6 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -216,6 +216,21 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) + modifiedRoleIDs, err := db.UpdateRolePermissionsPathPrefix(oldStorage.MountPath, storage.MountPath) + if err != nil { + return errors.WithMessage(err, "failed to update role permissions") + } + for _, id := range modifiedRoleIDs { + roleCache.Del(fmt.Sprint(id)) + } + + modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldStorage.MountPath, storage.MountPath) + if err != nil { + return errors.WithMessage(err, "failed to update user base path") + } + for _, name := range modifiedUsernames { + userCache.Del(name) + } } if err != nil { return errors.WithMessage(err, "failed get storage driver") diff --git a/internal/op/user.go b/internal/op/user.go index 79e73db86ce..e775df63e93 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -18,7 +18,11 @@ var adminUser *model.User func GetAdmin() (*model.User, error) { if adminUser == nil { - user, err := db.GetUserByRole(model.ADMIN) + role, err := GetRoleByName("admin") + if err != nil { + return nil, err + } + user, err := db.GetUserByRole(int(role.ID)) if err != nil { return nil, err } @@ -29,7 +33,11 @@ func GetAdmin() (*model.User, error) { func GetGuest() (*model.User, error) { if guestUser == nil { - user, err := db.GetUserByRole(model.GUEST) + role, err := GetRoleByName("guest") + if err != nil { + return nil, err + } + user, err := db.GetUserByRole(int(role.ID)) if err != nil { return nil, err } diff --git a/server/common/check.go b/server/common/check.go index 78051f4ee1e..34aaa41d93a 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -1,15 +1,11 @@ package common import ( - "path" - "strings" - "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" - "github.com/dlclark/regexp2" ) func IsStorageSignEnabled(rawPath string) bool { @@ -32,30 +28,11 @@ func IsApply(metaPath, reqPath string, applySub bool) bool { } func CanAccess(user *model.User, meta *model.Meta, reqPath string, password string) bool { - // if the reqPath is in hide (only can check the nearest meta) and user can't see hides, can't access - if meta != nil && !user.CanSeeHides() && meta.Hide != "" && - IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path - for _, hide := range strings.Split(meta.Hide, "\n") { - re := regexp2.MustCompile(hide, regexp2.None) - if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { - return false - } - } - } - // if is not guest and can access without password - if user.CanAccessWithoutPassword() { - return true - } - // if meta is nil or password is empty, can access - if meta == nil || meta.Password == "" { - return true - } - // if meta doesn't apply to sub_folder, can access - if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { - return true - } - // validate password - return meta.Password == password + // Deprecated: CanAccess is kept for backward compatibility. + // The logic has been moved to CanAccessWithRoles which performs the + // necessary checks based on role permissions. This wrapper ensures + // older calls still work without relying on user permission bits. + return CanAccessWithRoles(user, meta, reqPath, password) } // ShouldProxy TODO need optimize diff --git a/server/common/role_perm.go b/server/common/role_perm.go new file mode 100644 index 00000000000..1e539d966c5 --- /dev/null +++ b/server/common/role_perm.go @@ -0,0 +1,108 @@ +package common + +import ( + "path" + "strings" + + "github.com/dlclark/regexp2" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + PermSeeHides = iota + PermAccessWithoutPassword + PermAddOfflineDownload + PermWrite + PermRename + PermMove + PermCopy + PermRemove + PermWebdavRead + PermWebdavManage + PermFTPAccess + PermFTPManage + PermReadArchives + PermDecompress + PermPathLimit +) + +func HasPermission(perm int32, bit uint) bool { + return (perm>>bit)&1 == 1 +} + +func MergeRolePermissions(u *model.User, reqPath string) int32 { + if u == nil { + return 0 + } + var perm int32 + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + for _, entry := range role.PermissionScopes { + if utils.IsSubPath(entry.Path, reqPath) { + perm |= entry.Permission + } + } + } + return perm +} + +func CanAccessWithRoles(u *model.User, meta *model.Meta, reqPath, password string) bool { + if !canReadPathByRole(u, reqPath) { + return false + } + perm := MergeRolePermissions(u, reqPath) + if meta != nil && !HasPermission(perm, PermSeeHides) && meta.Hide != "" && + IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { + for _, hide := range strings.Split(meta.Hide, "\n") { + re := regexp2.MustCompile(hide, regexp2.None) + if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { + return false + } + } + } + if HasPermission(perm, PermAccessWithoutPassword) { + return true + } + if meta == nil || meta.Password == "" { + return true + } + if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { + return true + } + return meta.Password == password +} + +func canReadPathByRole(u *model.User, reqPath string) bool { + if u == nil { + return false + } + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + for _, entry := range role.PermissionScopes { + if utils.IsSubPath(entry.Path, reqPath) { + return true + } + } + } + return false +} + +// CheckPathLimitWithRoles checks whether the path is allowed when the user has +// the `PermPathLimit` permission for the target path. When the user does not +// have this permission, the check passes by default. +func CheckPathLimitWithRoles(u *model.User, reqPath string) bool { + perm := MergeRolePermissions(u, reqPath) + if HasPermission(perm, PermPathLimit) { + return canReadPathByRole(u, reqPath) + } + return true +} diff --git a/server/ftp.go b/server/ftp.go index 4d507b684b4..d41063731bf 100644 --- a/server/ftp.go +++ b/server/ftp.go @@ -11,6 +11,7 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/ftp" "math/rand" "net" @@ -130,7 +131,8 @@ func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) return nil, err } } - if userObj.Disabled || !userObj.CanFTPAccess() { + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via FTP") } diff --git a/server/ftp/fsmanage.go b/server/ftp/fsmanage.go index fb03c1b95cb..83e7bae1733 100644 --- a/server/ftp/fsmanage.go +++ b/server/ftp/fsmanage.go @@ -18,7 +18,8 @@ func Mkdir(ctx context.Context, path string) error { if err != nil { return err } - if !user.CanWrite() || !user.CanFTPManage() { + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) || !common.HasPermission(perm, common.PermFTPManage) { meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -34,7 +35,8 @@ func Mkdir(ctx context.Context, path string) error { func Remove(ctx context.Context, path string) error { user := ctx.Value("user").(*model.User) - if !user.CanRemove() || !user.CanFTPManage() { + perm := common.MergeRolePermissions(user, path) + if !common.HasPermission(perm, common.PermRemove) || !common.HasPermission(perm, common.PermFTPManage) { return errs.PermissionDenied } reqPath, err := user.JoinPath(path) @@ -56,13 +58,14 @@ func Rename(ctx context.Context, oldPath, newPath string) error { } srcDir, srcBase := stdpath.Split(srcPath) dstDir, dstBase := stdpath.Split(dstPath) + permSrc := common.MergeRolePermissions(user, srcPath) if srcDir == dstDir { - if !user.CanRename() || !user.CanFTPManage() { + if !common.HasPermission(permSrc, common.PermRename) || !common.HasPermission(permSrc, common.PermFTPManage) { return errs.PermissionDenied } return fs.Rename(ctx, srcPath, dstBase) } else { - if !user.CanFTPManage() || !user.CanMove() || (srcBase != dstBase && !user.CanRename()) { + if !common.HasPermission(permSrc, common.PermFTPManage) || !common.HasPermission(permSrc, common.PermMove) || (srcBase != dstBase && !common.HasPermission(permSrc, common.PermRename)) { return errs.PermissionDenied } if err = fs.Move(ctx, srcPath, dstDir); err != nil { diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index c051a19db21..2ba8cb82abc 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -30,7 +30,7 @@ func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownl } } ctx = context.WithValue(ctx, "meta", meta) - if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { return nil, errs.PermissionDenied } @@ -125,7 +125,7 @@ func Stat(ctx context.Context, path string) (os.FileInfo, error) { } } ctx = context.WithValue(ctx, "meta", meta) - if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { return nil, errs.PermissionDenied } obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) @@ -148,7 +148,7 @@ func List(ctx context.Context, path string) ([]os.FileInfo, error) { } } ctx = context.WithValue(ctx, "meta", meta) - if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { return nil, errs.PermissionDenied } objs, err := fs.List(ctx, reqPath, &fs.ListArgs{}) diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index ee38b1bfb07..9610eea7588 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -35,8 +35,10 @@ func uploadAuth(ctx context.Context, path string) error { return err } } - if !(common.CanAccess(user, meta, path, ctx.Value("meta_pass").(string)) && - ((user.CanFTPManage() && user.CanWrite()) || common.CanWrite(meta, stdpath.Dir(path)))) { + perm := common.MergeRolePermissions(user, path) + if !(common.CanAccessWithRoles(user, meta, path, ctx.Value("meta_pass").(string)) && + ((common.HasPermission(perm, common.PermFTPManage) && common.HasPermission(perm, common.PermWrite)) || + common.CanWrite(meta, stdpath.Dir(path)))) { return errs.PermissionDenied } return nil diff --git a/server/handles/archive.go b/server/handles/archive.go index 550bc3cec9c..0bb8d94a728 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -78,15 +78,20 @@ func FsArchiveMeta(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanReadArchives() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermReadArchives) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -156,15 +161,20 @@ func FsArchiveList(c *gin.Context) { } req.Validate() user := c.MustGet("user").(*model.User) - if !user.CanReadArchives() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermReadArchives) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -242,10 +252,6 @@ func FsArchiveDecompress(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanDecompress() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcPaths := make([]string, 0, len(req.Name)) for _, name := range req.Name { srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) @@ -253,6 +259,10 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } srcPaths = append(srcPaths, srcPath) } dstDir, err := user.JoinPath(req.DstDir) @@ -260,8 +270,17 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } tasks := make([]task.TaskExtensionInfo, 0, len(srcPaths)) for _, srcPath := range srcPaths { + perm := common.MergeRolePermissions(user, srcPath) + if !common.HasPermission(perm, common.PermDecompress) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } t, e := fs.ArchiveDecompress(c, srcPath, dstDir, model.ArchiveDecompressArgs{ ArchiveInnerArgs: model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ diff --git a/server/handles/auth.go b/server/handles/auth.go index 7a2c0fb5376..96a9ba9e20d 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/base64" "image/png" + "path" + "strings" "time" "github.com/Xhofe/go-cache" @@ -89,13 +91,16 @@ func loginHash(c *gin.Context, req *LoginReq) { type UserResp struct { model.User - Otp bool `json:"otp"` + Otp bool `json:"otp"` + RoleNames []string `json:"role_names"` + Permissions []model.PermissionEntry `json:"permissions"` } // CurrentUser get current user by token // if token is empty, return guest user func CurrentUser(c *gin.Context) { user := c.MustGet("user").(*model.User) + userResp := UserResp{ User: *user, } @@ -103,6 +108,30 @@ func CurrentUser(c *gin.Context) { if userResp.OtpSecret != "" { userResp.Otp = true } + + var roleNames []string + permMap := map[string]int32{} + addedPaths := map[string]bool{} + + for _, role := range user.RolesDetail { + roleNames = append(roleNames, role.Name) + for _, entry := range role.PermissionScopes { + cleanPath := path.Clean("/" + strings.TrimPrefix(entry.Path, "/")) + permMap[cleanPath] |= entry.Permission + } + } + userResp.RoleNames = roleNames + + for fullPath, perm := range permMap { + if !addedPaths[fullPath] { + userResp.Permissions = append(userResp.Permissions, model.PermissionEntry{ + Path: fullPath, + Permission: perm, + }) + addedPaths[fullPath] = true + } + } + common.SuccessResp(c, userResp) } diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 3841bff5a34..7ff07c6df28 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -29,20 +29,29 @@ func FsRecursiveMove(c *gin.Context) { } user := c.MustGet("user").(*model.User) - if !user.CanMove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermMove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(srcDir) if err != nil { @@ -149,16 +158,20 @@ func FsBatchRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } - reqPath, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(reqPath) if err != nil { @@ -194,14 +207,19 @@ func FsRegexRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { + reqPath, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CheckPathLimitWithRoles(user, reqPath) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - reqPath, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) return } diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index c527464e2e4..87be6e4106b 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -35,7 +35,12 @@ func FsMkdir(c *gin.Context) { common.ErrorResp(c, err, 403) return } - if !user.CanWrite() { + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) { meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -73,20 +78,29 @@ func FsMove(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanMove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + permMove := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(permMove, common.PermMove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } if !req.Overwrite { for _, name := range req.Names { if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { @@ -116,20 +130,29 @@ func FsCopy(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanCopy() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermCopy) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } if !req.Overwrite { for _, name := range req.Names { if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { @@ -167,15 +190,20 @@ func FsRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } if !req.Overwrite { dstPath := stdpath.Join(stdpath.Dir(reqPath), req.Name) if dstPath != reqPath { @@ -208,15 +236,20 @@ func FsRemove(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRemove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqDir, err := user.JoinPath(req.Dir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqDir) + if !common.HasPermission(perm, common.PermRemove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } for _, name := range req.Names { err := fs.Remove(c, stdpath.Join(reqDir, name)) if err != nil { @@ -240,15 +273,20 @@ func FsRemoveEmptyDirectory(c *gin.Context) { } user := c.MustGet("user").(*model.User) - if !user.CanRemove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermRemove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(srcDir) if err != nil { diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 73bde23b6de..b49f0b646b9 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -48,12 +48,28 @@ type ObjResp struct { } type FsListResp struct { - Content []ObjResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` + Content []ObjLabelResp `json:"content"` + Total int64 `json:"total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + Provider string `json:"provider"` +} + +type ObjLabelResp struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + LabelList []model.Label `json:"label_list"` } func FsList(c *gin.Context) { @@ -77,11 +93,12 @@ func FsList(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } - if !user.CanWrite() && !common.CanWrite(meta, reqPath) && req.Refresh { + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) && !common.CanWrite(meta, reqPath) && req.Refresh { common.ErrorStrResp(c, "Refresh without permission", 403) return } @@ -97,11 +114,11 @@ func FsList(c *gin.Context) { provider = storage.GetStorage().Driver } common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), + Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath), user.ID), Total: int64(total), Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), - Write: user.CanWrite() || common.CanWrite(meta, reqPath), + Write: common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, reqPath), Provider: provider, }) } @@ -135,7 +152,7 @@ func FsDirs(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } @@ -207,11 +224,15 @@ func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { return total, objs[start:end] } -func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { - var resp []ObjResp +func toObjsResp(objs []model.Obj, parent string, encrypt bool, userId uint) []ObjLabelResp { + var resp []ObjLabelResp for _, obj := range objs { + var labels []model.Label + if obj.IsDir() == false { + labels, _ = op.GetLabelByFileName(userId, obj.GetName()) + } thumb, _ := model.GetThumb(obj) - resp = append(resp, ObjResp{ + resp = append(resp, ObjLabelResp{ Id: obj.GetID(), Path: obj.GetPath(), Name: obj.GetName(), @@ -224,6 +245,7 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { Sign: common.Sign(obj, parent, encrypt), Thumb: thumb, Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + LabelList: labels, }) } return resp @@ -236,11 +258,11 @@ type FsGetReq struct { type FsGetResp struct { ObjResp - RawURL string `json:"raw_url"` - Readme string `json:"readme"` - Header string `json:"header"` - Provider string `json:"provider"` - Related []ObjResp `json:"related"` + RawURL string `json:"raw_url"` + Readme string `json:"readme"` + Header string `json:"header"` + Provider string `json:"provider"` + Related []ObjLabelResp `json:"related"` } func FsGet(c *gin.Context) { @@ -263,7 +285,7 @@ func FsGet(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } @@ -347,7 +369,7 @@ func FsGet(c *gin.Context) { Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), Provider: provider, - Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)), + Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath), user.ID), }) } @@ -391,7 +413,7 @@ func FsOther(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, req.Path, req.Password) { + if !common.CanAccessWithRoles(user, meta, req.Path, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } diff --git a/server/handles/label.go b/server/handles/label.go new file mode 100644 index 00000000000..4631124ecbf --- /dev/null +++ b/server/handles/label.go @@ -0,0 +1,99 @@ +package handles + +import ( + "errors" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + "strconv" +) + +func ListLabel(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + log.Debugf("%+v", req) + labels, total, err := db.GetLabels(req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, common.PageResp{ + Content: labels, + Total: total, + }) +} + +func GetLabel(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + label, err := db.GetLabelById(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, label) +} + +func CreateLabel(c *gin.Context) { + var req model.Label + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if db.GetLabelByName(req.Name) { + common.ErrorResp(c, errors.New("label name is exists"), 401) + return + } + if id, err := db.CreateLabel(req); err != nil { + common.ErrorWithDataResp(c, err, 500, gin.H{ + "id": id, + }, true) + } else { + common.SuccessResp(c, gin.H{ + "id": id, + }) + } +} + +func UpdateLabel(c *gin.Context) { + var req model.Label + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if label, err := db.UpdateLabel(&req); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c, label) + } +} + +func DeleteLabel(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + if err = op.DeleteLabelById(c, uint(id), userObj.ID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/label_file_binding.go b/server/handles/label_file_binding.go new file mode 100644 index 00000000000..78af929b34e --- /dev/null +++ b/server/handles/label_file_binding.go @@ -0,0 +1,103 @@ +package handles + +import ( + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "strconv" +) + +type DelLabelFileBinDingReq struct { + FileName string `json:"file_name"` + LabelId string `json:"label_id"` +} + +func GetLabelByFileName(c *gin.Context) { + fileName := c.Query("file_name") + if fileName == "" { + common.ErrorResp(c, errors.New("file_name must not empty"), 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + labels, err := op.GetLabelByFileName(userObj.ID, fileName) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, labels) +} + +func CreateLabelFileBinDing(c *gin.Context) { + var req op.CreateLabelFileBinDingReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.IsDir == true { + common.ErrorStrResp(c, "Unable to bind folder", 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + if err := op.CreateLabelFileBinDing(req, userObj.ID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } else { + common.SuccessResp(c, gin.H{ + "msg": "添加成功！", + }) + } +} + +func DelLabelByFileName(c *gin.Context) { + var req DelLabelFileBinDingReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + labelId, err := strconv.ParseUint(req.LabelId, 10, 64) + if err != nil { + common.ErrorResp(c, fmt.Errorf("invalid label ID '%s': %v", req.LabelId, err), 500, true) + return + } + if err = db.DelLabelFileBinDingById(uint(labelId), userObj.ID, req.FileName); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func GetFileByLabel(c *gin.Context) { + labelId := c.Query("label_id") + if labelId == "" { + common.ErrorResp(c, errors.New("file_name must not empty"), 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + fileList, err := op.GetFileByLabel(userObj.ID, labelId) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, fileList) +} diff --git a/server/handles/ldap_login.go b/server/handles/ldap_login.go index cf3148291b1..2a85dc03d0b 100644 --- a/server/handles/ldap_login.go +++ b/server/handles/ldap_login.go @@ -131,7 +131,7 @@ func ladpRegister(username string) (*model.User, error) { Password: random.String(16), Permission: int32(setting.GetInt(conf.LdapDefaultPermission, 0)), BasePath: setting.GetStr(conf.LdapDefaultDir), - Role: 0, + Role: nil, Disabled: false, } if err := db.CreateUser(user); err != nil { diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 24ff7a05369..8aade9eae57 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -5,6 +5,7 @@ import ( "github.com/alist-org/alist/v3/drivers/pikpak" "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/op" @@ -253,10 +254,6 @@ type AddOfflineDownloadReq struct { func AddOfflineDownload(c *gin.Context) { user := c.MustGet("user").(*model.User) - if !user.CanAddOfflineDownloadTasks() { - common.ErrorStrResp(c, "permission denied", 403) - return - } var req AddOfflineDownloadReq if err := c.ShouldBind(&req); err != nil { @@ -268,6 +265,15 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermAddOfflineDownload) { + common.ErrorStrResp(c, "permission denied", 403) + return + } var tasks []task.TaskExtensionInfo for _, url := range req.Urls { t, err := tool.AddURL(c, &tool.AddURLArgs{ diff --git a/server/handles/role.go b/server/handles/role.go new file mode 100644 index 00000000000..1bf7d4996bd --- /dev/null +++ b/server/handles/role.go @@ -0,0 +1,101 @@ +package handles + +import ( + "strconv" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" +) + +func ListRoles(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + log.Debugf("%+v", req) + roles, total, err := op.GetRoles(req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, common.PageResp{Content: roles, Total: total}) +} + +func GetRole(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, role) +} + +func CreateRole(c *gin.Context) { + var req model.Role + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := op.CreateRole(&req); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c) + } +} + +func UpdateRole(c *gin.Context) { + var req model.Role + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(req.ID) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + if role.Name == "admin" || role.Name == "guest" { + common.ErrorResp(c, errs.ErrChangeDefaultRole, 403) + return + } + if err := op.UpdateRole(&req); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c) + } +} + +func DeleteRole(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + if role.Name == "admin" || role.Name == "guest" { + common.ErrorResp(c, errs.ErrChangeDefaultRole, 403) + return + } + if err := op.DeleteRole(uint(id)); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/search.go b/server/handles/search.go index 8881731bd60..7d421a21e59 100644 --- a/server/handles/search.go +++ b/server/handles/search.go @@ -57,7 +57,7 @@ func Search(c *gin.Context) { if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { continue } - if !common.CanAccess(user, meta, path.Join(node.Parent, node.Name), req.Password) { + if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), req.Password) { continue } filteredNodes = append(filteredNodes, node) diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index 62bd4aaa2bf..eb6599e7a4d 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -154,7 +154,7 @@ func autoRegister(username, userID string, err error) (*model.User, error) { Password: random.String(16), Permission: int32(setting.GetInt(conf.SSODefaultPermission, 0)), BasePath: setting.GetStr(conf.SSODefaultDir), - Role: 0, + Role: nil, Disabled: false, SsoID: userID, } diff --git a/server/handles/task.go b/server/handles/task.go index af7974a9c29..6d49f9e5027 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -18,7 +18,7 @@ type TaskInfo struct { ID string `json:"id"` Name string `json:"name"` Creator string `json:"creator"` - CreatorRole int `json:"creator_role"` + CreatorRole model.Roles `json:"creator_role"` State tache.State `json:"state"` Status string `json:"status"` Progress float64 `json:"progress"` @@ -39,7 +39,7 @@ func getTaskInfo[T task.TaskExtensionInfo](task T) TaskInfo { progress = 100 } creatorName := "" - creatorRole := -1 + var creatorRole model.Roles if task.GetCreator() != nil { creatorName = task.GetCreator().Username creatorRole = task.GetCreator().Role diff --git a/server/handles/user.go b/server/handles/user.go index 4d404a4c652..50eaf969773 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -60,10 +60,10 @@ func UpdateUser(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if user.Role != req.Role { - common.ErrorStrResp(c, "role can not be changed", 400) - return - } + //if !utils.SliceEqual(user.Role, req.Role) { + // common.ErrorStrResp(c, "role can not be changed", 400) + // return + //} if req.Password == "" { req.PwdHash = user.PwdHash req.Salt = user.Salt diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index d65d1ad648a..47e7c0566c9 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -2,6 +2,7 @@ package middlewares import ( "crypto/subtle" + "fmt" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" @@ -68,6 +69,15 @@ func Auth(c *gin.Context) { c.Abort() return } + if len(user.Role) > 0 { + roles, err := op.GetRolesByUserID(user.ID) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("Fail to load roles: %v", err), 500) + c.Abort() + return + } + user.RolesDetail = roles + } c.Set("user", user) log.Debugf("use login token: %+v", user) c.Next() @@ -122,6 +132,19 @@ func Authn(c *gin.Context) { c.Abort() return } + if len(user.Role) > 0 { + var roles []model.Role + for _, roleID := range user.Role { + role, err := op.GetRole(uint(roleID)) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("load role %d failed", roleID), 500) + c.Abort() + return + } + roles = append(roles, *role) + } + user.RolesDetail = roles + } c.Set("user", user) log.Debugf("use login token: %+v", user) c.Next() diff --git a/server/middlewares/fsup.go b/server/middlewares/fsup.go index 2aa7fca6d03..243c22e4131 100644 --- a/server/middlewares/fsup.go +++ b/server/middlewares/fsup.go @@ -35,7 +35,9 @@ func FsUp(c *gin.Context) { return } } - if !(common.CanAccess(user, meta, path, password) && (user.CanWrite() || common.CanWrite(meta, stdpath.Dir(path)))) { + perm := common.MergeRolePermissions(user, path) + if !(common.CanAccessWithRoles(user, meta, path, password) && + (common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, stdpath.Dir(path)))) { common.ErrorResp(c, errs.PermissionDenied, 403) c.Abort() return diff --git a/server/router.go b/server/router.go index 09a0bb44faf..bf43a6258f4 100644 --- a/server/router.go +++ b/server/router.go @@ -120,6 +120,13 @@ func admin(g *gin.RouterGroup) { user.GET("/sshkey/list", handles.ListPublicKeys) user.POST("/sshkey/delete", handles.DeletePublicKey) + role := g.Group("/role") + role.GET("/list", handles.ListRoles) + role.GET("/get", handles.GetRole) + role.POST("/create", handles.CreateRole) + role.POST("/update", handles.UpdateRole) + role.POST("/delete", handles.DeleteRole) + storage := g.Group("/storage") storage.GET("/list", handles.ListStorages) storage.GET("/get", handles.GetStorage) @@ -161,6 +168,19 @@ func admin(g *gin.RouterGroup) { index.POST("/stop", middlewares.SearchIndex, handles.StopIndex) index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex) index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) + + label := g.Group("/label") + label.GET("/list", handles.ListLabel) + label.GET("/get", handles.GetLabel) + label.POST("/create", handles.CreateLabel) + label.POST("/update", handles.UpdateLabel) + label.POST("/delete", handles.DeleteLabel) + + labelFileBinding := g.Group("/label_file_binding") + labelFileBinding.GET("/get", handles.GetLabelByFileName) + labelFileBinding.GET("/get_file_by_label", handles.GetFileByLabel) + labelFileBinding.POST("/create", handles.CreateLabelFileBinDing) + labelFileBinding.POST("/delete", handles.DelLabelByFileName) } func _fs(g *gin.RouterGroup) { diff --git a/server/sftp.go b/server/sftp.go index 42c676e8c17..7d8c7212e9a 100644 --- a/server/sftp.go +++ b/server/sftp.go @@ -8,6 +8,7 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/ftp" "github.com/alist-org/alist/v3/server/sftp" "github.com/pkg/errors" @@ -78,7 +79,8 @@ func (d *SftpDriver) NoClientAuth(conn ssh.ConnMetadata) (*ssh.Permissions, erro if err != nil { return nil, err } - if guest.Disabled || !guest.CanFTPAccess() { + permGuest := common.MergeRolePermissions(guest, guest.BasePath) + if guest.Disabled || !common.HasPermission(permGuest, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via SFTP") } return nil, nil @@ -89,7 +91,8 @@ func (d *SftpDriver) PasswordAuth(conn ssh.ConnMetadata, password []byte) (*ssh. if err != nil { return nil, err } - if userObj.Disabled || !userObj.CanFTPAccess() { + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via SFTP") } passHash := model.StaticHash(string(password)) @@ -104,7 +107,8 @@ func (d *SftpDriver) PublicKeyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*s if err != nil { return nil, err } - if userObj.Disabled || !userObj.CanFTPAccess() { + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via SFTP") } keys, _, err := op.GetSSHPublicKeyByUserId(userObj.ID, 1, -1) diff --git a/server/webdav.go b/server/webdav.go index a735e285527..a65896dfc79 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -3,16 +3,19 @@ package server import ( "context" "crypto/subtle" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/server/middlewares" "net/http" + "net/url" "path" "strings" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/server/middlewares" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/webdav" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -92,7 +95,19 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - if user.Disabled || !user.CanWebdavRead() { + reqPath := c.Param("path") + if reqPath == "" { + reqPath = "/" + } + reqPath, _ = url.PathUnescape(reqPath) + reqPath, err = user.JoinPath(reqPath) + if err != nil { + c.Status(http.StatusForbidden) + c.Abort() + return + } + perm := common.MergeRolePermissions(user, reqPath) + if user.Disabled || !common.HasPermission(perm, common.PermWebdavRead) { if c.Request.Method == "OPTIONS" { c.Set("user", guest) c.Next() @@ -102,27 +117,27 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!user.CanWebdavManage() || !user.CanWrite()) { + if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermWrite)) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "MOVE" && (!user.CanWebdavManage() || (!user.CanMove() && !user.CanRename())) { + if c.Request.Method == "MOVE" && (!common.HasPermission(perm, common.PermWebdavManage) || (!common.HasPermission(perm, common.PermMove) && !common.HasPermission(perm, common.PermRename))) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "COPY" && (!user.CanWebdavManage() || !user.CanCopy()) { + if c.Request.Method == "COPY" && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermCopy)) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "DELETE" && (!user.CanWebdavManage() || !user.CanRemove()) { + if c.Request.Method == "DELETE" && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermRemove)) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "PROPPATCH" && !user.CanWebdavManage() { + if c.Request.Method == "PROPPATCH" && !common.HasPermission(perm, common.PermWebdavManage) { c.Status(http.StatusForbidden) c.Abort() return diff --git a/server/webdav/file.go b/server/webdav/file.go index ac8f5c1cbfb..ab78d26105d 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -14,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" ) // slashClean is equivalent to but slightly more efficient than @@ -34,10 +35,11 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int srcName := path.Base(src) dstName := path.Base(dst) user := ctx.Value("user").(*model.User) - if srcDir != dstDir && !user.CanMove() { + perm := common.MergeRolePermissions(user, src) + if srcDir != dstDir && !common.HasPermission(perm, common.PermMove) { return http.StatusForbidden, nil } - if srcName != dstName && !user.CanRename() { + if srcName != dstName && !common.HasPermission(perm, common.PermRename) { return http.StatusForbidden, nil } if srcDir == dstDir { From f61d13d4330348493e4fa64a3b595999e9b6b100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sat, 26 Jul 2025 15:20:08 +0800 Subject: [PATCH 010/133] refactor(convert_role): Improve role conversion logic for legacy formats (#9219) - Add new imports: `database/sql`, `encoding/json`, and `conf` package in `convert_role.go`. - Simplify permission entry initialization by removing redundant struct formatting. - Update error logging messages for better clarity. - Replace `op.GetUsers` with direct database access for fetching user roles. - Implement role update logic using `rawDb` and handle legacy int role conversion. - Count the number of users whose roles are updated and log completion. - Introduce `IsLegacyRoleDetected` function to check for legacy role formats. - Modify `cmd/common.go` to invoke role conversion if legacy format is detected. --- cmd/common.go | 7 ++ .../bootstrap/patch/v3_46_0/convert_role.go | 107 ++++++++++++++---- 2 files changed, 89 insertions(+), 25 deletions(-) diff --git a/cmd/common.go b/cmd/common.go index 8a73f9b0582..d88a86eb09d 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_46_0" "os" "path/filepath" "strconv" @@ -16,6 +17,12 @@ func Init() { bootstrap.InitConfig() bootstrap.Log() bootstrap.InitDB() + + if v3_46_0.IsLegacyRoleDetected() { + utils.Log.Warnf("Detected legacy role format, executing ConvertLegacyRoles patch early...") + v3_46_0.ConvertLegacyRoles() + } + data.InitData() bootstrap.InitStreamLimit() bootstrap.InitIndex() diff --git a/internal/bootstrap/patch/v3_46_0/convert_role.go b/internal/bootstrap/patch/v3_46_0/convert_role.go index 43799485c12..3aac95b691c 100644 --- a/internal/bootstrap/patch/v3_46_0/convert_role.go +++ b/internal/bootstrap/patch/v3_46_0/convert_role.go @@ -1,7 +1,10 @@ package v3_46_0 import ( + "database/sql" + "encoding/json" "errors" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" @@ -83,47 +86,101 @@ func ConvertLegacyRoles() { } } - users, _, err := op.GetUsers(1, -1) + rawDb := db.GetDb() + table := conf.Conf.Database.TablePrefix + "users" + rows, err := rawDb.Table(table).Select("id, username, role").Rows() if err != nil { utils.Log.Errorf("[convert roles] failed to get users: %v", err) return } + defer rows.Close() - for i := range users { - user := users[i] - if user.Role == nil { + var updatedCount int + for rows.Next() { + var id uint + var username string + var rawRole []byte + + if err := rows.Scan(&id, &username, &rawRole); err != nil { + utils.Log.Warnf("[convert roles] skip user scan err: %v", err) continue } - changed := false - var roles model.Roles - for _, r := range user.Role { + + utils.Log.Debugf("[convert roles] user: %s raw role: %s", username, string(rawRole)) + + if len(rawRole) == 0 { + continue + } + + var oldRoles []int + wasSingleInt := false + if err := json.Unmarshal(rawRole, &oldRoles); err != nil { + var single int + if err := json.Unmarshal(rawRole, &single); err != nil { + utils.Log.Warnf("[convert roles] user %s has invalid role: %s", username, string(rawRole)) + continue + } + oldRoles = []int{single} + wasSingleInt = true + } + + var newRoles model.Roles + for _, r := range oldRoles { switch r { case model.ADMIN: - roles = append(roles, int(adminRole.ID)) - if int(adminRole.ID) != r { - changed = true - } + newRoles = append(newRoles, int(adminRole.ID)) case model.GUEST: - roles = append(roles, int(guestRole.ID)) - if int(guestRole.ID) != r { - changed = true - } + newRoles = append(newRoles, int(guestRole.ID)) case model.GENERAL: - roles = append(roles, int(generalRole.ID)) - if int(generalRole.ID) != r { - changed = true - } + newRoles = append(newRoles, int(generalRole.ID)) default: - roles = append(roles, r) + newRoles = append(newRoles, r) } } - if changed { - user.Role = roles - if err := db.UpdateUser(&user); err != nil { - utils.Log.Errorf("[convert roles] failed to update user %s: %v", user.Username, err) + + if wasSingleInt { + err := rawDb.Table(table).Where("id = ?", id).Update("role", newRoles).Error + if err != nil { + utils.Log.Errorf("[convert roles] failed to update user %s: %v", username, err) + } else { + updatedCount++ + utils.Log.Infof("[convert roles] updated user %s: %v → %v", username, oldRoles, newRoles) } } } - utils.Log.Infof("[convert roles] completed role conversion for %d users", len(users)) + utils.Log.Infof("[convert roles] completed role conversion for %d users", updatedCount) +} + +func IsLegacyRoleDetected() bool { + rawDb := db.GetDb() + table := conf.Conf.Database.TablePrefix + "users" + rows, err := rawDb.Table(table).Select("role").Rows() + if err != nil { + utils.Log.Errorf("[role check] failed to scan user roles: %v", err) + return false + } + defer rows.Close() + + for rows.Next() { + var raw sql.RawBytes + if err := rows.Scan(&raw); err != nil { + continue + } + if len(raw) == 0 { + continue + } + + var roles []int + if err := json.Unmarshal(raw, &roles); err == nil { + continue + } + + var single int + if err := json.Unmarshal(raw, &single); err == nil { + utils.Log.Infof("[role check] detected legacy int role: %d", single) + return true + } + } + return false } From 91cc7529a035d7f78d332c4cf54f501ffb65c2cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sun, 27 Jul 2025 22:25:45 +0800 Subject: [PATCH 011/133] feat(user/role/storage): enhance user and storage operations with additional validations (#9223) - Update `CreateUser` to adjust `BasePath` based on user roles and clean paths. - Modify `UpdateUser` to incorporate role-based path changes. - Add validation in `CreateStorage` and `UpdateStorage` to prevent root mount path. - Prevent changes to admin user's role and username in user handler. - Update `UpdateRole` to modify user base paths when role paths change, and clear user cache accordingly. - Import `errors` package to handle error messages. --- internal/op/role.go | 15 +++++++++++++++ internal/op/storage.go | 8 ++++++++ internal/op/user.go | 31 ++++++++++++++++++++++++++++++- server/handles/user.go | 17 +++++++++++++---- 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/internal/op/role.go b/internal/op/role.go index 64502f98df4..e0f2dc7592c 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -2,6 +2,7 @@ package op import ( "fmt" + "github.com/pkg/errors" "time" "github.com/Xhofe/go-cache" @@ -102,6 +103,20 @@ func UpdateRole(r *model.Role) error { for i := range r.PermissionScopes { r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) } + if len(old.PermissionScopes) > 0 && len(r.PermissionScopes) > 0 && + old.PermissionScopes[0].Path != r.PermissionScopes[0].Path { + + oldPath := old.PermissionScopes[0].Path + newPath := r.PermissionScopes[0].Path + modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath) + if err != nil { + return errors.WithMessage(err, "failed to update user base path when role updated") + } + + for _, name := range modifiedUsernames { + userCache.Del(name) + } + } roleCache.Del(fmt.Sprint(r.ID)) roleCache.Del(r.Name) return db.UpdateRole(r) diff --git a/internal/op/storage.go b/internal/op/storage.go index 2ec68aae5e6..2833afa84ca 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -46,6 +46,11 @@ func GetStorageByMountPath(mountPath string) (driver.Driver, error) { func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + + if storage.MountPath == "/" { + return 0, errors.New("Mount path cannot be '/'") + } + var err error // check driver first driverName := storage.Driver @@ -205,6 +210,9 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { } storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + if storage.MountPath == "/" { + return errors.New("Mount path cannot be '/'") + } err = db.UpdateStorage(&storage) if err != nil { return errors.WithMessage(err, "failed update storage in database") diff --git a/internal/op/user.go b/internal/op/user.go index e775df63e93..30b9d0e6d26 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -78,7 +78,25 @@ func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err err func CreateUser(u *model.User) error { u.BasePath = utils.FixAndCleanPath(u.BasePath) - return db.CreateUser(u) + + err := db.CreateUser(u) + if err != nil { + return err + } + + roles, err := GetRolesByUserID(u.ID) + if err == nil { + for _, role := range roles { + if len(role.PermissionScopes) > 0 { + u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) + break + } + } + _ = db.UpdateUser(u) + userCache.Del(u.Username) + } + + return nil } func DeleteUserById(id uint) error { @@ -106,6 +124,17 @@ func UpdateUser(u *model.User) error { } userCache.Del(old.Username) u.BasePath = utils.FixAndCleanPath(u.BasePath) + if len(u.Role) > 0 { + roles, err := GetRolesByUserID(u.ID) + if err == nil { + for _, role := range roles { + if len(role.PermissionScopes) > 0 { + u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) + break + } + } + } + } return db.UpdateUser(u) } diff --git a/server/handles/user.go b/server/handles/user.go index 50eaf969773..b729d117fdc 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -1,6 +1,7 @@ package handles import ( + "github.com/alist-org/alist/v3/pkg/utils" "strconv" "github.com/alist-org/alist/v3/internal/model" @@ -60,10 +61,18 @@ func UpdateUser(c *gin.Context) { common.ErrorResp(c, err, 500) return } - //if !utils.SliceEqual(user.Role, req.Role) { - // common.ErrorStrResp(c, "role can not be changed", 400) - // return - //} + + if user.Username == "admin" { + if !utils.SliceEqual(user.Role, req.Role) { + common.ErrorStrResp(c, "cannot change role of admin user", 403) + return + } + if user.Username != req.Username { + common.ErrorStrResp(c, "cannot change username of admin user", 403) + return + } + } + if req.Password == "" { req.PwdHash = user.PwdHash req.Salt = user.Salt From 5b8c26510b720a2cd308023dc7cd1c8bd1e9d20c Mon Sep 17 00:00:00 2001 From: qianshi Date: Mon, 28 Jul 2025 23:07:07 +0800 Subject: [PATCH 012/133] feat(user-management): Enhance admin management and role handling - Add `CountEnabledAdminsExcluding` function to count enabled admins excluding a specific user. - Implement `CountUsersByRoleAndEnabledExclude` in `internal/db/user.go` to support exclusion logic. - Refactor role handling with switch-case for better readability in `server/handles/role.go`. - Ensure at least one enabled admin remains when disabling an admin in `server/handles/user.go`. - Maintain guest role name consistency when updating roles in `internal/op/role.go`. --- internal/db/user.go | 11 +++++++++++ internal/op/role.go | 6 +++++- internal/op/user.go | 8 ++++++++ server/handles/role.go | 6 +++++- server/handles/user.go | 13 ++++++++++--- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/internal/db/user.go b/internal/db/user.go index f2b6635a983..9e8ee3fc05e 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -2,6 +2,7 @@ package db import ( "encoding/base64" + "fmt" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-webauthn/webauthn/webauthn" @@ -140,3 +141,13 @@ func UpdateUserBasePathPrefix(oldPath, newPath string) ([]string, error) { return modifiedUsernames, nil } + +func CountUsersByRoleAndEnabledExclude(roleID uint, excludeUserID uint) (int64, error) { + var count int64 + jsonValue := fmt.Sprintf("[%d]", roleID) + err := db.Model(&model.User{}). + Where("disabled = ? AND id != ?", false, excludeUserID). + Where("JSON_CONTAINS(role, ?)", jsonValue). + Count(&count).Error + return count, err +} diff --git a/internal/op/role.go b/internal/op/role.go index 64502f98df4..b7474566438 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -96,8 +96,12 @@ func UpdateRole(r *model.Role) error { if err != nil { return err } - if old.Name == "admin" || old.Name == "guest" { + switch old.Name { + case "admin": return errs.ErrChangeDefaultRole + + case "guest": + r.Name = "guest" } for i := range r.PermissionScopes { r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) diff --git a/internal/op/user.go b/internal/op/user.go index e775df63e93..b9662015902 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -136,3 +136,11 @@ func DelUserCache(username string) error { userCache.Del(username) return nil } + +func CountEnabledAdminsExcluding(userID uint) (int64, error) { + adminRole, err := GetRoleByName("admin") + if err != nil { + return 0, err + } + return db.CountUsersByRoleAndEnabledExclude(adminRole.ID, userID) +} diff --git a/server/handles/role.go b/server/handles/role.go index 1bf7d4996bd..0d071c9f84d 100644 --- a/server/handles/role.go +++ b/server/handles/role.go @@ -66,9 +66,13 @@ func UpdateRole(c *gin.Context) { common.ErrorResp(c, err, 500, true) return } - if role.Name == "admin" || role.Name == "guest" { + switch role.Name { + case "admin": common.ErrorResp(c, errs.ErrChangeDefaultRole, 403) return + + case "guest": + req.Name = "guest" } if err := op.UpdateRole(&req); err != nil { common.ErrorResp(c, err, 500, true) diff --git a/server/handles/user.go b/server/handles/user.go index 50eaf969773..858c2a3bcf0 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -74,9 +74,16 @@ func UpdateUser(c *gin.Context) { if req.OtpSecret == "" { req.OtpSecret = user.OtpSecret } - if req.Disabled && req.IsAdmin() { - common.ErrorStrResp(c, "admin user can not be disabled", 400) - return + if req.Disabled && user.IsAdmin() { + count, err := op.CountEnabledAdminsExcluding(user.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if count == 0 { + common.ErrorStrResp(c, "at least one enabled admin must be kept", 400) + return + } } if err := op.UpdateUser(&req); err != nil { common.ErrorResp(c, err, 500) From 4d7c2a09ce4716cecc541d9e2518d1fdf1d23172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 29 Jul 2025 09:42:34 +0800 Subject: [PATCH 013/133] docs(README): Add API documentation links across multiple languages (#9225) - Add API documentation section to `README.md` with link to Apifox - Add API documentation section to `README_ja.md` with Japanese translation and link to Apifox - Add API documentation section to `README_cn.md` with Chinese translation and link to Apifox --- README.md | 4 ++++ README_cn.md | 4 ++++ README_ja.md | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/README.md b/README.md index 2bd7e812890..5a93997fe40 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,10 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing +## API Documentation (via Apifox): + + + ## Demo diff --git a/README_cn.md b/README_cn.md index 9052e79b0ea..79ed864bc84 100644 --- a/README_cn.md +++ b/README_cn.md @@ -99,6 +99,10 @@ +## API 文档（通过 Apifox 提供） + + + ## Demo diff --git a/README_ja.md b/README_ja.md index 4dcdfd203bf..9291b2acdc2 100644 --- a/README_ja.md +++ b/README_ja.md @@ -100,6 +100,10 @@ +## APIドキュメント（Apifox 提供） + + + ## デモ From 33530554821085e069cf165083e0d7862879cb65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 29 Jul 2025 18:35:47 +0800 Subject: [PATCH 014/133] Update Dockerfile.ci (#9230) chore(docker): Update base image from alpine:edge to alpine:3.20.7 in Dockerfile.ci --- Dockerfile.ci | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index a17aae9fcfd..6075acc639a 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:edge +FROM alpine:3.20.7 ARG TARGETPLATFORM ARG INSTALL_FFMPEG=false @@ -31,4 +31,4 @@ RUN /entrypoint.sh version ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} VOLUME /opt/alist/data/ EXPOSE 5244 5245 -CMD [ "/entrypoint.sh" ] \ No newline at end of file +CMD [ "/entrypoint.sh" ] From 540d6c7064994b009293d0dac42419280a525166 Mon Sep 17 00:00:00 2001 From: Sky_slience <95515853+skysliences@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:48:33 +0800 Subject: [PATCH 015/133] fix(meta): update OAuth token URL and improve default client credentials (#9231) --- drivers/aliyundrive_open/meta.go | 2 +- drivers/baidu_netdisk/meta.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index 03f97f8b795..bb4354ddc11 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -11,7 +11,7 @@ type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` - OauthTokenURL string `json:"oauth_token_url" default:"https://api.nn.ci/alist/ali_open/token"` + OauthTokenURL string `json:"oauth_token_url" default:"https://api.alistgo.com/alist/ali_open/token"` ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 27571056e11..7577c747fe3 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -11,8 +11,8 @@ type Addition struct { OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` DownloadAPI string `json:"download_api" type:"select" options:"official,crack,crack_video" default:"official"` - ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` - ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` + ClientID string `json:"client_id" required:"true" default:"hq9yQ9w9kR4YHj1kyYafLygVocobh7Sf"` + ClientSecret string `json:"client_secret" required:"true" default:"YH2VpZcFJHYNnV6vLfHQXDBhcE7ZChyE"` CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` AccessToken string UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` From 74332e91fb8f3a920d6c59557dddb66f4a2bb188 Mon Sep 17 00:00:00 2001 From: Sky_slience <95515853+skysliences@users.noreply.github.com> Date: Wed, 30 Jul 2025 12:22:02 +0800 Subject: [PATCH 016/133] feat(ui): add new UI configuration option to settings (#9233) * feat(ui): add new UI configuration option to settings * fix(ui): disable new UI feature by default --------- Co-authored-by: Sky_slience --- internal/bootstrap/data/setting.go | 2 ++ internal/conf/const.go | 1 + 2 files changed, 3 insertions(+) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index fe1d9219a08..e00abf2d379 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -103,6 +103,8 @@ func InitialSettings() []model.SettingItem { {Key: conf.AllowIndexed, Value: "false", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, + // newui settings + {Key: conf.UseNewui, Value: "false", Type: conf.TypeBool, Group: model.SITE}, // style settings {Key: conf.Logo, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeText, Group: model.STYLE}, {Key: conf.Favicon, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.STYLE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 5cb8d850bf0..48ac2037fb2 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -16,6 +16,7 @@ const ( AllowIndexed = "allow_indexed" AllowMounted = "allow_mounted" RobotsTxt = "robots_txt" + UseNewui = "use_newui" Logo = "logo" Favicon = "favicon" From 280960ce3e338c41d3e06181b3bf95d6b0ee422f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Wed, 30 Jul 2025 13:15:35 +0800 Subject: [PATCH 017/133] feat(user-db): enhance user management with role-based queries (allow-edit-role-guest) (#9234) - Add `GetUsersByRole` function to fetch users based on their roles. - Extend `UpdateUserBasePathPrefix` to accept optional user lists. - Ensure path cleaning in `UpdateUserBasePathPrefix` for consistency. - Integrate guest role fetching in `auth.go` middleware. - Utilize `GetUsersByRole` in `role.go` for base path modifications. - Remove redundant line in `role.go` role modification logic. --- internal/db/user.go | 33 ++++++++++++++++++++++++++------- internal/op/role.go | 9 +++++++-- server/middlewares/auth.go | 9 +++++++++ 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/internal/db/user.go b/internal/db/user.go index 9e8ee3fc05e..8f1c28b92f2 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "gorm.io/gorm" "path" + "slices" "strings" ) @@ -25,6 +26,20 @@ func GetUserByRole(role int) (*model.User, error) { return nil, gorm.ErrRecordNotFound } +func GetUsersByRole(roleID int) ([]model.User, error) { + var users []model.User + if err := db.Find(&users).Error; err != nil { + return nil, err + } + var result []model.User + for _, u := range users { + if slices.Contains(u.Role, roleID) { + result = append(result, u) + } + } + return result, nil +} + func GetUserByName(username string) (*model.User, error) { user := model.User{Username: username} if err := db.Where(user).First(&user).Error; err != nil { @@ -109,25 +124,29 @@ func RemoveAuthn(u *model.User, id string) error { return UpdateAuthn(u.ID, string(res)) } -func UpdateUserBasePathPrefix(oldPath, newPath string) ([]string, error) { +func UpdateUserBasePathPrefix(oldPath, newPath string, usersOpt ...[]model.User) ([]string, error) { var users []model.User var modifiedUsernames []string - if err := db.Find(&users).Error; err != nil { - return nil, errors.WithMessage(err, "failed to load users") - } - oldPathClean := path.Clean(oldPath) + if len(usersOpt) > 0 { + users = usersOpt[0] + } else { + if err := db.Find(&users).Error; err != nil { + return nil, errors.WithMessage(err, "failed to load users") + } + } + for _, user := range users { basePath := path.Clean(user.BasePath) updated := false if basePath == oldPathClean { - user.BasePath = newPath + user.BasePath = path.Clean(newPath) updated = true } else if strings.HasPrefix(basePath, oldPathClean+"/") { - user.BasePath = newPath + basePath[len(oldPathClean):] + user.BasePath = path.Clean(newPath + basePath[len(oldPathClean):]) updated = true } diff --git a/internal/op/role.go b/internal/op/role.go index b312f8c79e0..c719c6f4cdf 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -100,7 +100,6 @@ func UpdateRole(r *model.Role) error { switch old.Name { case "admin": return errs.ErrChangeDefaultRole - case "guest": r.Name = "guest" } @@ -112,7 +111,13 @@ func UpdateRole(r *model.Role) error { oldPath := old.PermissionScopes[0].Path newPath := r.PermissionScopes[0].Path - modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath) + + users, err := db.GetUsersByRole(int(r.ID)) + if err != nil { + return errors.WithMessage(err, "failed to get users by role") + } + + modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath, users) if err != nil { return errors.WithMessage(err, "failed to update user base path when role updated") } diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 47e7c0566c9..c0743c9ce9d 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -41,6 +41,15 @@ func Auth(c *gin.Context) { c.Abort() return } + if len(guest.Role) > 0 { + roles, err := op.GetRolesByUserID(guest.ID) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("Fail to load guest roles: %v", err), 500) + c.Abort() + return + } + guest.RolesDetail = roles + } c.Set("user", guest) log.Debugf("use empty token: %+v", guest) c.Next() From 394a18cbd96b581cab75a780be04aff84451fd46 Mon Sep 17 00:00:00 2001 From: Sky_slience <95515853+skysliences@users.noreply.github.com> Date: Wed, 30 Jul 2025 16:55:32 +0800 Subject: [PATCH 018/133] Fix 123 download (#9235) * fix(driver): handle additional HTTP status code 210 for URL redirection * fix(driver): 123 download url error --------- Co-authored-by: Sky_slience --- drivers/123/driver.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 32c053e22ab..a8af2b6fd01 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -113,6 +113,8 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) log.Debugln("res code: ", res.StatusCode()) if res.StatusCode() == 302 { link.URL = res.Header().Get("location") + } else if res.StatusCode() == 210 { + link.URL = downloadUrl } else if res.StatusCode() < 300 { link.URL = utils.Json.Get(res.Body(), "data", "redirect_url").ToString() } From ae90fb579bded772f1a7744195ed4e0409a8fadb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sun, 3 Aug 2025 09:26:23 +0800 Subject: [PATCH 019/133] feat(log): enhance log formatter to respect NO_COLOR env variable (#9239) - Adjust log formatter to disable colors when NO_COLOR or ALIST_NO_COLOR environment variables are set. - Reorganize formatter settings for better readability. --- internal/bootstrap/log.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/bootstrap/log.go b/internal/bootstrap/log.go index 00411e5e189..b4f4af08f1b 100644 --- a/internal/bootstrap/log.go +++ b/internal/bootstrap/log.go @@ -14,10 +14,14 @@ import ( func init() { formatter := logrus.TextFormatter{ - ForceColors: true, - EnvironmentOverrideColors: true, - TimestampFormat: "2006-01-02 15:04:05", - FullTimestamp: true, + TimestampFormat: "2006-01-02 15:04:05", + FullTimestamp: true, + } + if os.Getenv("NO_COLOR") != "" || os.Getenv("ALIST_NO_COLOR") == "1" { + formatter.DisableColors = true + } else { + formatter.ForceColors = true + formatter.EnvironmentOverrideColors = true } logrus.SetFormatter(&formatter) utils.Log.SetFormatter(&formatter) From 46de9e9ebb4bed894eae82270cbf6137cf790dd8 Mon Sep 17 00:00:00 2001 From: Sky_slience <95515853+skysliences@users.noreply.github.com> Date: Sun, 3 Aug 2025 20:00:09 +0800 Subject: [PATCH 020/133] fix(driver): 123 download and modify request headers on the frontend (#9236) Co-authored-by: Sky_slience --- drivers/123/driver.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index a8af2b6fd01..32c053e22ab 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -113,8 +113,6 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) log.Debugln("res code: ", res.StatusCode()) if res.StatusCode() == 302 { link.URL = res.Header().Get("location") - } else if res.StatusCode() == 210 { - link.URL = downloadUrl } else if res.StatusCode() < 300 { link.URL = utils.Json.Get(res.Body(), "data", "redirect_url").ToString() } From 52da07e8a768277e0145457396401ce1b4758e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 4 Aug 2025 11:56:57 +0800 Subject: [PATCH 021/133] feat(123_open): add new driver support for 123 Open (#9246) - Implement new driver for 123 Open service, enabling file operations such as listing, uploading, moving, and removing files. - Introduce token management for authentication and authorization. - Add API integration for various file operations and actions. - Include utility functions for handling API requests and responses. - Register the new driver in the existing drivers' list. --- drivers/123_open/api.go | 191 +++++++++++++++++++++++++ drivers/123_open/driver.go | 277 ++++++++++++++++++++++++++++++++++++ drivers/123_open/meta.go | 33 +++++ drivers/123_open/token.go | 85 +++++++++++ drivers/123_open/types.go | 70 +++++++++ drivers/123_open/upload.go | 282 +++++++++++++++++++++++++++++++++++++ drivers/123_open/util.go | 20 +++ drivers/all.go | 1 + internal/errs/driver.go | 1 + internal/model/obj.go | 15 ++ 10 files changed, 975 insertions(+) create mode 100644 drivers/123_open/api.go create mode 100644 drivers/123_open/driver.go create mode 100644 drivers/123_open/meta.go create mode 100644 drivers/123_open/token.go create mode 100644 drivers/123_open/types.go create mode 100644 drivers/123_open/upload.go create mode 100644 drivers/123_open/util.go diff --git a/drivers/123_open/api.go b/drivers/123_open/api.go new file mode 100644 index 00000000000..1d2a6f164ff --- /dev/null +++ b/drivers/123_open/api.go @@ -0,0 +1,191 @@ +package _123Open + +import ( + "fmt" + "github.com/go-resty/resty/v2" + "net/http" +) + +const ( + // baseurl + ApiBaseURL = "https://open-api.123pan.com" + + // auth + ApiToken = "/api/v1/access_token" + + // file list + ApiFileList = "/api/v2/file/list" + + // direct link + ApiGetDirectLink = "/api/v1/direct-link/url" + + // mkdir + ApiMakeDir = "/upload/v1/file/mkdir" + + // remove + ApiRemove = "/api/v1/file/trash" + + // upload + ApiUploadDomainURL = "/upload/v2/file/domain" + ApiSingleUploadURL = "/upload/v2/file/single/create" + ApiCreateUploadURL = "/upload/v2/file/create" + ApiUploadSliceURL = "/upload/v2/file/slice" + ApiUploadCompleteURL = "/upload/v2/file/upload_complete" + + // move + ApiMove = "/api/v1/file/move" + + // rename + ApiRename = "/api/v1/file/name" +) + +type Response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type TokenResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data TokenData `json:"data"` +} + +type TokenData struct { + AccessToken string `json:"accessToken"` + ExpiredAt string `json:"expiredAt"` +} + +type FileListResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data FileListData `json:"data"` +} + +type FileListData struct { + LastFileId int64 `json:"lastFileId"` + FileList []File `json:"fileList"` +} + +type DirectLinkResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data DirectLinkData `json:"data"` +} + +type DirectLinkData struct { + URL string `json:"url"` +} + +type MakeDirRequest struct { + Name string `json:"name"` + ParentID int64 `json:"parentID"` +} + +type MakeDirResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data MakeDirData `json:"data"` +} + +type MakeDirData struct { + DirID int64 `json:"dirID"` +} + +type RemoveRequest struct { + FileIDs []int64 `json:"fileIDs"` +} + +type UploadCreateResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadCreateData `json:"data"` +} + +type UploadCreateData struct { + FileID int64 `json:"fileId"` + Reuse bool `json:"reuse"` + PreuploadID string `json:"preuploadId"` + SliceSize int64 `json:"sliceSize"` + Servers []string `json:"servers"` +} + +type UploadUrlResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadUrlData `json:"data"` +} + +type UploadUrlData struct { + PresignedURL string `json:"presignedUrl"` +} + +type UploadCompleteResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadCompleteData `json:"data"` +} + +type UploadCompleteData struct { + FileID int `json:"fileID"` + Completed bool `json:"completed"` +} + +func (d *Open123) Request(endpoint string, method string, setup func(*resty.Request), result any) (*resty.Response, error) { + client := resty.New() + token, err := d.tm.getToken() + if err != nil { + return nil, err + } + + req := client.R(). + SetHeader("Authorization", "Bearer "+token). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", "application/json"). + SetResult(result) + + if setup != nil { + setup(req) + } + + switch method { + case http.MethodGet: + return req.Get(ApiBaseURL + endpoint) + case http.MethodPost: + return req.Post(ApiBaseURL + endpoint) + case http.MethodPut: + return req.Put(ApiBaseURL + endpoint) + default: + return nil, fmt.Errorf("unsupported method: %s", method) + } +} + +func (d *Open123) RequestTo(fullURL string, method string, setup func(*resty.Request), result any) (*resty.Response, error) { + client := resty.New() + + token, err := d.tm.getToken() + if err != nil { + return nil, err + } + + req := client.R(). + SetHeader("Authorization", "Bearer "+token). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", "application/json"). + SetResult(result) + + if setup != nil { + setup(req) + } + + switch method { + case http.MethodGet: + return req.Get(fullURL) + case http.MethodPost: + return req.Post(fullURL) + case http.MethodPut: + return req.Put(fullURL) + default: + return nil, fmt.Errorf("unsupported method: %s", method) + } +} diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go new file mode 100644 index 00000000000..39ed146eb03 --- /dev/null +++ b/drivers/123_open/driver.go @@ -0,0 +1,277 @@ +package _123Open + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "net/http" + "strconv" +) + +type Open123 struct { + model.Storage + Addition + + UploadThread int + tm *tokenManager +} + +func (d *Open123) Config() driver.Config { + return config +} + +func (d *Open123) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Open123) Init(ctx context.Context) error { + d.tm = newTokenManager(d.ClientID, d.ClientSecret) + + if _, err := d.tm.getToken(); err != nil { + return fmt.Errorf("token 初始化失败: %w", err) + } + + return nil +} + +func (d *Open123) Drop(ctx context.Context) error { + return nil +} + +func (d *Open123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + parentFileId, err := strconv.ParseInt(dir.GetID(), 10, 64) + if err != nil { + return nil, err + } + + fileLastId := int64(0) + var results []File + + for fileLastId != -1 { + files, err := d.getFiles(parentFileId, 100, fileLastId) + if err != nil { + return nil, err + } + for _, f := range files.Data.FileList { + if f.Trashed == 0 { + results = append(results, f) + } + } + fileLastId = files.Data.LastFileId + } + + objs := make([]model.Obj, 0, len(results)) + for _, f := range results { + objs = append(objs, f) + } + return objs, nil +} + +func (d *Open123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.LinkIsDir + } + + fileID := file.GetID() + + var result DirectLinkResp + url := fmt.Sprintf("%s?fileID=%s", ApiGetDirectLink, fileID) + _, err := d.Request(url, http.MethodGet, nil, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("get link failed: %s", result.Message) + } + + return &model.Link{ + URL: result.Data.URL, + }, nil +} + +func (d *Open123) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + parentID, err := strconv.ParseInt(parentDir.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid parent ID: %w", err) + } + + var result MakeDirResp + reqBody := MakeDirRequest{ + Name: dirName, + ParentID: parentID, + } + + _, err = d.Request(ApiMakeDir, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("mkdir failed: %s", result.Message) + } + + newDir := File{ + FileId: result.Data.DirID, + FileName: dirName, + Type: 1, + ParentFileId: int(parentID), + Size: 0, + Trashed: 0, + } + return newDir, nil +} + +func (d *Open123) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcID, err := strconv.ParseInt(srcObj.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid src file ID: %w", err) + } + dstID, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid dest dir ID: %w", err) + } + + var result Response[any] + reqBody := map[string]interface{}{ + "fileIDs": []int64{srcID}, + "toParentFileID": dstID, + } + + _, err = d.Request(ApiMove, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("move failed: %s", result.Message) + } + + files, err := d.getFiles(dstID, 100, 0) + if err != nil { + return nil, fmt.Errorf("move succeed but failed to get target dir: %w", err) + } + for _, f := range files.Data.FileList { + if f.FileId == srcID { + return f, nil + } + } + return nil, fmt.Errorf("move succeed but file not found in target dir") +} + +func (d *Open123) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + srcID, err := strconv.ParseInt(srcObj.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid file ID: %w", err) + } + + var result Response[any] + reqBody := map[string]interface{}{ + "fileId": srcID, + "fileName": newName, + } + + _, err = d.Request(ApiRename, http.MethodPut, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("rename failed: %s", result.Message) + } + + parentID := 0 + if file, ok := srcObj.(File); ok { + parentID = file.ParentFileId + } + files, err := d.getFiles(int64(parentID), 100, 0) + if err != nil { + return nil, fmt.Errorf("rename succeed but failed to get parent dir: %w", err) + } + for _, f := range files.Data.FileList { + if f.FileId == srcID { + return f, nil + } + } + return nil, fmt.Errorf("rename succeed but file not found in parent dir") +} + +func (d *Open123) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { + idStr := obj.GetID() + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + return fmt.Errorf("invalid file ID: %w", err) + } + + var result Response[any] + reqBody := RemoveRequest{ + FileIDs: []int64{id}, + } + + _, err = d.Request(ApiRemove, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return err + } + if result.Code != 0 { + return fmt.Errorf("remove failed: %s", result.Message) + } + + return nil +} + +func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + etag := file.GetHash().GetHash(utils.MD5) + + if len(etag) < utils.MD5.Width { + up = model.UpdateProgressWithRange(up, 50, 100) + _, etag, err = stream.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return err + } + } + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + if err != nil { + return err + } + if createResp.Data.Reuse { + return nil + } + + return d.Upload(ctx, file, parentFileId, createResp, up) +} + +func (d *Open123) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotSupport +} + +func (d *Open123) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Open123) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotSupport +} + +func (d *Open123) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +//func (d *Open123) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Open123)(nil) diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go new file mode 100644 index 00000000000..d99bb75ba2c --- /dev/null +++ b/drivers/123_open/meta.go @@ -0,0 +1,33 @@ +package _123Open + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + + ClientID string `json:"client_id" required:"true" label:"Client ID"` + ClientSecret string `json:"client_secret" required:"true" label:"Client Secret"` +} + +var config = driver.Config{ + Name: "123 Open", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Open123{} + }) +} diff --git a/drivers/123_open/token.go b/drivers/123_open/token.go new file mode 100644 index 00000000000..435c0b0de67 --- /dev/null +++ b/drivers/123_open/token.go @@ -0,0 +1,85 @@ +package _123Open + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" +) + +const tokenURL = ApiBaseURL + ApiToken + +type tokenManager struct { + clientID string + clientSecret string + + mu sync.Mutex + accessToken string + expireTime time.Time +} + +func newTokenManager(clientID, clientSecret string) *tokenManager { + return &tokenManager{ + clientID: clientID, + clientSecret: clientSecret, + } +} + +func (tm *tokenManager) getToken() (string, error) { + tm.mu.Lock() + defer tm.mu.Unlock() + + if tm.accessToken != "" && time.Now().Before(tm.expireTime.Add(-5*time.Minute)) { + return tm.accessToken, nil + } + + reqBody := map[string]string{ + "clientID": tm.clientID, + "clientSecret": tm.clientSecret, + } + body, _ := json.Marshal(reqBody) + req, err := http.NewRequest("POST", tokenURL, bytes.NewBuffer(body)) + if err != nil { + return "", err + } + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var result TokenResp + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", err + } + + if result.Code != 0 { + return "", fmt.Errorf("get token failed: %s", result.Message) + } + + tm.accessToken = result.Data.AccessToken + expireAt, err := time.Parse(time.RFC3339, result.Data.ExpiredAt) + if err != nil { + return "", fmt.Errorf("parse expire time failed: %w", err) + } + tm.expireTime = expireAt + + return tm.accessToken, nil +} + +func (tm *tokenManager) buildHeaders() (http.Header, error) { + token, err := tm.getToken() + if err != nil { + return nil, err + } + header := http.Header{} + header.Set("Authorization", "Bearer "+token) + header.Set("Platform", "open_platform") + header.Set("Content-Type", "application/json") + return header, nil +} diff --git a/drivers/123_open/types.go b/drivers/123_open/types.go new file mode 100644 index 00000000000..afece279e60 --- /dev/null +++ b/drivers/123_open/types.go @@ -0,0 +1,70 @@ +package _123Open + +import ( + "fmt" + "github.com/alist-org/alist/v3/pkg/utils" + "time" +) + +type File struct { + FileName string `json:"filename"` + Size int64 `json:"size"` + CreateAt string `json:"createAt"` + UpdateAt string `json:"updateAt"` + FileId int64 `json:"fileId"` + Type int `json:"type"` + Etag string `json:"etag"` + S3KeyFlag string `json:"s3KeyFlag"` + ParentFileId int `json:"parentFileId"` + Category int `json:"category"` + Status int `json:"status"` + Trashed int `json:"trashed"` +} + +func (f File) GetID() string { + return fmt.Sprint(f.FileId) +} + +func (f File) GetName() string { + return f.FileName +} + +func (f File) GetSize() int64 { + return f.Size +} + +func (f File) IsDir() bool { + return f.Type == 1 +} + +func (f File) GetModified() string { + return f.UpdateAt +} + +func (f File) GetThumb() string { + return "" +} + +func (f File) ModTime() time.Time { + t, err := time.Parse("2006-01-02 15:04:05", f.UpdateAt) + if err != nil { + return time.Time{} + } + return t +} + +func (f File) CreateTime() time.Time { + t, err := time.Parse("2006-01-02 15:04:05", f.CreateAt) + if err != nil { + return time.Time{} + } + return t +} + +func (f File) GetHash() utils.HashInfo { + return utils.NewHashInfo(utils.MD5, f.Etag) +} + +func (f File) GetPath() string { + return "" +} diff --git a/drivers/123_open/upload.go b/drivers/123_open/upload.go new file mode 100644 index 00000000000..76e8ead46f8 --- /dev/null +++ b/drivers/123_open/upload.go @@ -0,0 +1,282 @@ +package _123Open + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "golang.org/x/sync/errgroup" + "io" + "mime/multipart" + "net/http" + "runtime" + "strconv" + "time" +) + +func (d *Open123) create(parentFileID int64, filename, etag string, size int64, duplicate int, containDir bool) (*UploadCreateResp, error) { + var resp UploadCreateResp + + _, err := d.Request(ApiCreateUploadURL, http.MethodPost, func(req *resty.Request) { + body := base.Json{ + "parentFileID": parentFileID, + "filename": filename, + "etag": etag, + "size": size, + } + if duplicate > 0 { + body["duplicate"] = duplicate + } + if containDir { + body["containDir"] = true + } + req.SetBody(body) + }, &resp) + + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Open123) GetUploadDomains() ([]string, error) { + var resp struct { + Code int `json:"code"` + Message string `json:"message"` + Data []string `json:"data"` + } + + _, err := d.Request(ApiUploadDomainURL, http.MethodGet, nil, &resp) + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("get upload domain failed: %s", resp.Message) + } + return resp.Data, nil +} + +func (d *Open123) UploadSingle(ctx context.Context, createResp *UploadCreateResp, file model.FileStreamer, parentID int64) error { + domain := createResp.Data.Servers[0] + + etag := file.GetHash().GetHash(utils.MD5) + if len(etag) < utils.MD5.Width { + _, _, err := stream.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return err + } + } + + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) + if err != nil { + return err + } + reader = driver.NewLimitedUploadStream(ctx, reader) + + var b bytes.Buffer + mw := multipart.NewWriter(&b) + mw.WriteField("parentFileID", fmt.Sprint(parentID)) + mw.WriteField("filename", file.GetName()) + mw.WriteField("etag", etag) + mw.WriteField("size", fmt.Sprint(file.GetSize())) + fw, _ := mw.CreateFormFile("file", file.GetName()) + _, err = io.Copy(fw, reader) + mw.Close() + + req, err := http.NewRequestWithContext(ctx, "POST", domain+ApiSingleUploadURL, &b) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+d.tm.accessToken) + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", mw.FormDataContentType()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + var result struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + FileID int64 `json:"fileID"` + Completed bool `json:"completed"` + } `json:"data"` + } + body, _ := io.ReadAll(resp.Body) + if err := json.Unmarshal(body, &result); err != nil { + return fmt.Errorf("unmarshal response error: %v, body: %s", err, string(body)) + } + if result.Code != 0 { + return fmt.Errorf("upload failed: %s", result.Message) + } + if !result.Data.Completed || result.Data.FileID == 0 { + return fmt.Errorf("upload incomplete or missing fileID") + } + return nil +} + +func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, parentID int64, createResp *UploadCreateResp, up driver.UpdateProgress) error { + if cacher, ok := file.(interface{ CacheFullInTempFile() (model.File, error) }); ok { + if _, err := cacher.CacheFullInTempFile(); err != nil { + return err + } + } + + size := file.GetSize() + chunkSize := createResp.Data.SliceSize + uploadNums := (size + chunkSize - 1) / chunkSize + uploadDomain := createResp.Data.Servers[0] + + if d.UploadThread <= 0 { + cpuCores := runtime.NumCPU() + threads := cpuCores * 2 + if threads < 4 { + threads = 4 + } + if threads > 16 { + threads = 16 + } + d.UploadThread = threads + fmt.Printf("[Upload] Auto set upload concurrency: %d (CPU cores=%d)\n", d.UploadThread, cpuCores) + } + + fmt.Printf("[Upload] File size: %d bytes, chunk size: %d bytes, total slices: %d, concurrency: %d\n", + size, chunkSize, uploadNums, d.UploadThread) + + if size <= 1<<30 { + return d.UploadSingle(ctx, createResp, file, parentID) + } + + if createResp.Data.Reuse { + up(100) + return nil + } + + client := resty.New() + semaphore := make(chan struct{}, d.UploadThread) + threadG, _ := errgroup.WithContext(ctx) + + var progressArr = make([]int64, uploadNums) + + for partIndex := int64(0); partIndex < uploadNums; partIndex++ { + partIndex := partIndex + semaphore <- struct{}{} + + threadG.Go(func() error { + defer func() { <-semaphore }() + offset := partIndex * chunkSize + length := min(chunkSize, size-offset) + partNumber := partIndex + 1 + + fmt.Printf("[Slice %d] Starting read from offset %d, length %d\n", partNumber, offset, length) + reader, err := file.RangeRead(http_range.Range{Start: offset, Length: length}) + if err != nil { + return fmt.Errorf("[Slice %d] RangeRead error: %v", partNumber, err) + } + + buf := make([]byte, length) + n, err := io.ReadFull(reader, buf) + if err != nil && err != io.EOF { + return fmt.Errorf("[Slice %d] Read error: %v", partNumber, err) + } + buf = buf[:n] + hash := md5.Sum(buf) + sliceMD5Str := hex.EncodeToString(hash[:]) + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + writer.WriteField("preuploadID", createResp.Data.PreuploadID) + writer.WriteField("sliceNo", strconv.FormatInt(partNumber, 10)) + writer.WriteField("sliceMD5", sliceMD5Str) + partName := fmt.Sprintf("%s.part%d", file.GetName(), partNumber) + fw, _ := writer.CreateFormFile("slice", partName) + fw.Write(buf) + writer.Close() + + resp, err := client.R(). + SetHeader("Authorization", "Bearer "+d.tm.accessToken). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", writer.FormDataContentType()). + SetBody(body.Bytes()). + Post(uploadDomain + ApiUploadSliceURL) + + if err != nil { + return fmt.Errorf("[Slice %d] Upload HTTP error: %v", partNumber, err) + } + if resp.StatusCode() != 200 { + return fmt.Errorf("[Slice %d] Upload failed with status: %s, resp: %s", partNumber, resp.Status(), resp.String()) + } + + progressArr[partIndex] = length + var totalUploaded int64 = 0 + for _, v := range progressArr { + totalUploaded += v + } + if up != nil { + percent := float64(totalUploaded) / float64(size) * 100 + up(percent) + } + + fmt.Printf("[Slice %d] MD5: %s\n", partNumber, sliceMD5Str) + fmt.Printf("[Slice %d] Upload finished\n", partNumber) + return nil + }) + } + + if err := threadG.Wait(); err != nil { + return err + } + + var completeResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Completed bool `json:"completed"` + FileID int64 `json:"fileID"` + } `json:"data"` + } + + for { + reqBody := fmt.Sprintf(`{"preuploadID":"%s"}`, createResp.Data.PreuploadID) + req, err := http.NewRequestWithContext(ctx, "POST", uploadDomain+ApiUploadCompleteURL, bytes.NewBufferString(reqBody)) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+d.tm.accessToken) + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + if err := json.Unmarshal(body, &completeResp); err != nil { + return fmt.Errorf("completion response unmarshal error: %v, body: %s", err, string(body)) + } + if completeResp.Code != 0 { + return fmt.Errorf("completion API returned error code %d: %s", completeResp.Code, completeResp.Message) + } + if completeResp.Data.Completed && completeResp.Data.FileID != 0 { + fmt.Printf("[Upload] Upload completed successfully. FileID: %d\n", completeResp.Data.FileID) + break + } + time.Sleep(time.Second) + } + up(100) + return nil +} diff --git a/drivers/123_open/util.go b/drivers/123_open/util.go new file mode 100644 index 00000000000..429a5e5dda8 --- /dev/null +++ b/drivers/123_open/util.go @@ -0,0 +1,20 @@ +package _123Open + +import ( + "fmt" + "net/http" +) + +func (d *Open123) getFiles(parentFileId int64, limit int, lastFileId int64) (*FileListResp, error) { + var result FileListResp + url := fmt.Sprintf("%s?parentFileId=%d&limit=%d&lastFileId=%d", ApiFileList, parentFileId, limit, lastFileId) + + _, err := d.Request(url, http.MethodGet, nil, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("list error: %s", result.Message) + } + return &result, nil +} diff --git a/drivers/all.go b/drivers/all.go index 224fb8ddb4b..a8c8620989e 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -6,6 +6,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/115_share" _ "github.com/alist-org/alist/v3/drivers/123" _ "github.com/alist-org/alist/v3/drivers/123_link" + _ "github.com/alist-org/alist/v3/drivers/123_open" _ "github.com/alist-org/alist/v3/drivers/123_share" _ "github.com/alist-org/alist/v3/drivers/139" _ "github.com/alist-org/alist/v3/drivers/189" diff --git a/internal/errs/driver.go b/internal/errs/driver.go index 4b6b5cac48e..7f67c0e2c2d 100644 --- a/internal/errs/driver.go +++ b/internal/errs/driver.go @@ -4,4 +4,5 @@ import "errors" var ( EmptyToken = errors.New("empty token") + LinkIsDir = errors.New("link is dir") ) diff --git a/internal/model/obj.go b/internal/model/obj.go index f0fce7a133a..93fa7a96475 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -55,6 +55,21 @@ type FileStreamer interface { type UpdateProgress func(percentage float64) +// Reference implementation from OpenListTeam: +// https://github.com/OpenListTeam/OpenList/blob/a703b736c9346c483bae56905a39bc07bf781cff/internal/model/obj.go#L58 +func UpdateProgressWithRange(inner UpdateProgress, start, end float64) UpdateProgress { + return func(p float64) { + if p < 0 { + p = 0 + } + if p > 100 { + p = 100 + } + scaled := start + (end-start)*(p/100.0) + inner(scaled) + } +} + type URL interface { URL() string } From 85fe4e5bb3455e70ec3b9114d3370c35019d702c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 4 Aug 2025 12:02:45 +0800 Subject: [PATCH 022/133] feat(alist_v3): add IntSlice type for JSON unmarshalling (#9247) - Add `IntSlice` type to handle both single int and array in JSON. - Modify `MeResp` struct to use `IntSlice` for `Role` field. - Import `encoding/json` for JSON operations. --- drivers/alist_v3/types.go | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go index 83ecde8be17..3e8e2f71eac 100644 --- a/drivers/alist_v3/types.go +++ b/drivers/alist_v3/types.go @@ -1,6 +1,7 @@ package alist_v3 import ( + "encoding/json" "time" "github.com/alist-org/alist/v3/internal/model" @@ -72,15 +73,15 @@ type LoginResp struct { } type MeResp struct { - Id int `json:"id"` - Username string `json:"username"` - Password string `json:"password"` - BasePath string `json:"base_path"` - Role []int `json:"role"` - Disabled bool `json:"disabled"` - Permission int `json:"permission"` - SsoId string `json:"sso_id"` - Otp bool `json:"otp"` + Id int `json:"id"` + Username string `json:"username"` + Password string `json:"password"` + BasePath string `json:"base_path"` + Role IntSlice `json:"role"` + Disabled bool `json:"disabled"` + Permission int `json:"permission"` + SsoId string `json:"sso_id"` + Otp bool `json:"otp"` } type ArchiveMetaReq struct { @@ -168,3 +169,17 @@ type DecompressReq struct { PutIntoNewDir bool `json:"put_into_new_dir"` SrcDir string `json:"src_dir"` } + +type IntSlice []int + +func (s *IntSlice) UnmarshalJSON(data []byte) error { + if len(data) > 0 && data[0] == '[' { + return json.Unmarshal(data, (*[]int)(s)) + } + var single int + if err := json.Unmarshal(data, &single); err != nil { + return err + } + *s = []int{single} + return nil +} From 6b2d81eede823fd7e5e56d4da78da815a4fe1434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Wed, 6 Aug 2025 16:31:36 +0800 Subject: [PATCH 023/133] feat(user): enhance path management and role handling (#9249) - Add `GetUsersByRole` function for fetching users by role. - Introduce `GetAllBasePathsFromRoles` to aggregate paths from roles. - Refine path handling in `pkg/utils/path.go` for normalization. - Comment out base path prefix updates to simplify role operations. --- internal/model/user.go | 35 +++++++++++++++++++++++++++++++++-- internal/op/role.go | 41 ++++++++++++++++++++--------------------- internal/op/storage.go | 20 ++++++++++++++------ internal/op/user.go | 26 +++++++++++++++----------- pkg/utils/path.go | 7 +++++++ 5 files changed, 89 insertions(+), 40 deletions(-) diff --git a/internal/model/user.go b/internal/model/user.go index 0b9e576a484..221747a407e 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -149,9 +149,21 @@ func (u *User) JoinPath(reqPath string) (string, error) { if err != nil { return "", err } - if u.CheckPathLimit() && !utils.IsSubPath(u.BasePath, path) { - return "", errs.PermissionDenied + + if u.CheckPathLimit() { + basePaths := GetAllBasePathsFromRoles(u) + match := false + for _, base := range basePaths { + if utils.IsSubPath(base, path) { + match = true + break + } + } + if !match { + return "", errs.PermissionDenied + } } + return path, nil } @@ -193,3 +205,22 @@ func (u *User) WebAuthnCredentials() []webauthn.Credential { func (u *User) WebAuthnIcon() string { return "https://alistgo.com/logo.svg" } + +// GetAllBasePathsFromRoles returns all permission paths from user's roles +func GetAllBasePathsFromRoles(u *User) []string { + basePaths := make([]string, 0) + seen := make(map[string]struct{}) + + for _, role := range u.RolesDetail { + for _, entry := range role.PermissionScopes { + if entry.Path == "" { + continue + } + if _, ok := seen[entry.Path]; !ok { + basePaths = append(basePaths, entry.Path) + seen[entry.Path] = struct{}{} + } + } + } + return basePaths +} diff --git a/internal/op/role.go b/internal/op/role.go index c719c6f4cdf..e24ba5ab1a8 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -2,7 +2,6 @@ package op import ( "fmt" - "github.com/pkg/errors" "time" "github.com/Xhofe/go-cache" @@ -106,26 +105,26 @@ func UpdateRole(r *model.Role) error { for i := range r.PermissionScopes { r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) } - if len(old.PermissionScopes) > 0 && len(r.PermissionScopes) > 0 && - old.PermissionScopes[0].Path != r.PermissionScopes[0].Path { - - oldPath := old.PermissionScopes[0].Path - newPath := r.PermissionScopes[0].Path - - users, err := db.GetUsersByRole(int(r.ID)) - if err != nil { - return errors.WithMessage(err, "failed to get users by role") - } - - modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath, users) - if err != nil { - return errors.WithMessage(err, "failed to update user base path when role updated") - } - - for _, name := range modifiedUsernames { - userCache.Del(name) - } - } + //if len(old.PermissionScopes) > 0 && len(r.PermissionScopes) > 0 && + // old.PermissionScopes[0].Path != r.PermissionScopes[0].Path { + // + // oldPath := old.PermissionScopes[0].Path + // newPath := r.PermissionScopes[0].Path + // + // users, err := db.GetUsersByRole(int(r.ID)) + // if err != nil { + // return errors.WithMessage(err, "failed to get users by role") + // } + // + // modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath, users) + // if err != nil { + // return errors.WithMessage(err, "failed to update user base path when role updated") + // } + // + // for _, name := range modifiedUsernames { + // userCache.Del(name) + // } + //} roleCache.Del(fmt.Sprint(r.ID)) roleCache.Del(r.Name) return db.UpdateRole(r) diff --git a/internal/op/storage.go b/internal/op/storage.go index 2833afa84ca..5ab1da1840a 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -232,12 +232,20 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { roleCache.Del(fmt.Sprint(id)) } - modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldStorage.MountPath, storage.MountPath) - if err != nil { - return errors.WithMessage(err, "failed to update user base path") - } - for _, name := range modifiedUsernames { - userCache.Del(name) + //modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldStorage.MountPath, storage.MountPath) + //if err != nil { + // return errors.WithMessage(err, "failed to update user base path") + //} + for _, id := range modifiedRoleIDs { + roleCache.Del(fmt.Sprint(id)) + + users, err := db.GetUsersByRole(int(id)) + if err != nil { + return errors.WithMessage(err, "failed to get users by role") + } + for _, user := range users { + userCache.Del(user.Username) + } } } if err != nil { diff --git a/internal/op/user.go b/internal/op/user.go index 942daae65d1..44b19db3508 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -50,6 +50,10 @@ func GetUserByRole(role int) (*model.User, error) { return db.GetUserByRole(role) } +func GetUsersByRole(role int) ([]model.User, error) { + return db.GetUsersByRole(role) +} + func GetUserByName(username string) (*model.User, error) { if username == "" { return nil, errs.EmptyUsername @@ -124,17 +128,17 @@ func UpdateUser(u *model.User) error { } userCache.Del(old.Username) u.BasePath = utils.FixAndCleanPath(u.BasePath) - if len(u.Role) > 0 { - roles, err := GetRolesByUserID(u.ID) - if err == nil { - for _, role := range roles { - if len(role.PermissionScopes) > 0 { - u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) - break - } - } - } - } + //if len(u.Role) > 0 { + // roles, err := GetRolesByUserID(u.ID) + // if err == nil { + // for _, role := range roles { + // if len(role.PermissionScopes) > 0 { + // u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) + // break + // } + // } + // } + //} return db.UpdateUser(u) } diff --git a/pkg/utils/path.go b/pkg/utils/path.go index 135f8e4ebca..6f3a55fc3d3 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -88,6 +88,13 @@ func JoinBasePath(basePath, reqPath string) (string, error) { strings.Contains(reqPath, "/../") { return "", errs.RelativePath } + + reqPath = FixAndCleanPath(reqPath) + + if strings.HasPrefix(reqPath, "/") { + return reqPath, nil + } + return stdpath.Join(FixAndCleanPath(basePath), FixAndCleanPath(reqPath)), nil } From aea3ba1499d56cb3de39da7f57c973197801af8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 Aug 2025 08:09:00 -0700 Subject: [PATCH 024/133] feat: add tag backup and fix bugs (#9265) * feat(label): enhance label file binding and router setup (feat/add-tag-backup) - Add `GetLabelsByFileNamesPublic` to retrieve labels using file names. - Refactor router setup for label and file binding routes. - Improve `toObjsResp` for efficient label retrieval by file names. - Comment out unnecessary user ID parameter in `toObjsResp`. * feat(label): enhance label file binding and router setup - Add `GetLabelsByFileNamesPublic` for label retrieval by file names. - Refactor router setup for label and file binding routes. - Improve `toObjsResp` for efficient label retrieval by file names. - Comment out unnecessary user ID parameter in `toObjsResp`. * refactor(db): comment out debug print in GetLabelIds (#feat/add-tag-backup) - Comment out debug print statement in GetLabelIds to clean up logs. - Enhance code readability by removing unnecessary debug output. * feat(label-file-binding): add batch creation and improve label ID handling - Introduced `CreateLabelFileBinDingBatch` API for batch label binding. - Added `collectLabelIDs` helper function to handle label ID parsing. - Enhanced label ID handling to support varied delimiters and input formats. - Refactored `CreateLabelFileBinDing` logic for improved code readability. - Updated router to include `POST /label_file_binding/create_batch`. --- internal/db/db.go | 2 +- internal/db/label_file_binding.go | 148 ++++++++++++++++++++++++-- internal/errs/role.go | 2 +- internal/model/label_file_binding.go | 2 +- internal/op/label_file_binding.go | 58 +++++++++-- server/handles/fsread.go | 20 +++- server/handles/label_file_binding.go | 149 ++++++++++++++++++++++++++- server/handles/user.go | 8 +- server/router.go | 20 +++- 9 files changed, 375 insertions(+), 34 deletions(-) diff --git a/internal/db/db.go b/internal/db/db.go index c6491dc984f..0d8ab42130a 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinDing), new(model.ObjFile)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/label_file_binding.go b/internal/db/label_file_binding.go index ec722efb979..4dda80f2c42 100644 --- a/internal/db/label_file_binding.go +++ b/internal/db/label_file_binding.go @@ -1,15 +1,18 @@ package db import ( + "fmt" "github.com/alist-org/alist/v3/internal/model" "github.com/pkg/errors" "gorm.io/gorm" + "gorm.io/gorm/clause" "time" ) // GetLabelIds Get all label_ids from database order by file_name func GetLabelIds(userId uint, fileName string) ([]uint, error) { - labelFileBinDingDB := db.Model(&model.LabelFileBinDing{}) + //fmt.Printf(">>> [GetLabelIds] userId: %d, fileName: %s\n", userId, fileName) + labelFileBinDingDB := db.Model(&model.LabelFileBinding{}) var labelIds []uint if err := labelFileBinDingDB.Where("file_name = ?", fileName).Where("user_id = ?", userId).Pluck("label_id", &labelIds).Error; err != nil { return nil, errors.WithStack(err) @@ -18,7 +21,7 @@ func GetLabelIds(userId uint, fileName string) ([]uint, error) { } func CreateLabelFileBinDing(fileName string, labelId, userId uint) error { - var labelFileBinDing model.LabelFileBinDing + var labelFileBinDing model.LabelFileBinding labelFileBinDing.UserId = userId labelFileBinDing.LabelId = labelId labelFileBinDing.FileName = fileName @@ -32,7 +35,7 @@ func CreateLabelFileBinDing(fileName string, labelId, userId uint) error { // GetLabelFileBinDingByLabelIdExists Get Label by label_id, used to del label usually func GetLabelFileBinDingByLabelIdExists(labelId, userId uint) bool { - var labelFileBinDing model.LabelFileBinDing + var labelFileBinDing model.LabelFileBinding result := db.Where("label_id = ?", labelId).Where("user_id = ?", userId).First(&labelFileBinDing) exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) return exists @@ -40,17 +43,150 @@ func GetLabelFileBinDingByLabelIdExists(labelId, userId uint) bool { // DelLabelFileBinDingByFileName used to del usually func DelLabelFileBinDingByFileName(userId uint, fileName string) error { - return errors.WithStack(db.Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinDing{}).Error) + return errors.WithStack(db.Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinding{}).Error) } // DelLabelFileBinDingById used to del usually func DelLabelFileBinDingById(labelId, userId uint, fileName string) error { - return errors.WithStack(db.Where("label_id = ?", labelId).Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinDing{}).Error) + return errors.WithStack(db.Where("label_id = ?", labelId).Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinding{}).Error) } -func GetLabelFileBinDingByLabelId(labelIds []uint, userId uint) (result []model.LabelFileBinDing, err error) { +func GetLabelFileBinDingByLabelId(labelIds []uint, userId uint) (result []model.LabelFileBinding, err error) { if err := db.Where("label_id in (?)", labelIds).Where("user_id = ?", userId).Find(&result).Error; err != nil { return nil, errors.WithStack(err) } return result, nil } + +func GetLabelBindingsByFileNamesPublic(fileNames []string) (map[string][]uint, error) { + var binds []model.LabelFileBinding + if err := db.Where("file_name IN ?", fileNames).Find(&binds).Error; err != nil { + return nil, errors.WithStack(err) + } + out := make(map[string][]uint, len(fileNames)) + seen := make(map[string]struct{}, len(binds)) + for _, b := range binds { + key := fmt.Sprintf("%s-%d", b.FileName, b.LabelId) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out[b.FileName] = append(out[b.FileName], b.LabelId) + } + return out, nil +} + +func GetLabelsByFileNamesPublic(fileNames []string) (map[string][]model.Label, error) { + bindMap, err := GetLabelBindingsByFileNamesPublic(fileNames) + if err != nil { + return nil, err + } + + idSet := make(map[uint]struct{}) + for _, ids := range bindMap { + for _, id := range ids { + idSet[id] = struct{}{} + } + } + if len(idSet) == 0 { + return make(map[string][]model.Label, 0), nil + } + allIDs := make([]uint, 0, len(idSet)) + for id := range idSet { + allIDs = append(allIDs, id) + } + labels, err := GetLabelByIds(allIDs) // 你已有的函数 + if err != nil { + return nil, err + } + + labelByID := make(map[uint]model.Label, len(labels)) + for _, l := range labels { + labelByID[l.ID] = l + } + + out := make(map[string][]model.Label, len(bindMap)) + for fname, ids := range bindMap { + for _, id := range ids { + if lab, ok := labelByID[id]; ok { + out[fname] = append(out[fname], lab) + } + } + } + return out, nil +} + +func ListLabelFileBinDing(userId uint, labelIDs []uint, fileName string, page, pageSize int) ([]model.LabelFileBinding, int64, error) { + q := db.Model(&model.LabelFileBinding{}).Where("user_id = ?", userId) + + if len(labelIDs) > 0 { + q = q.Where("label_id IN ?", labelIDs) + } + if fileName != "" { + q = q.Where("file_name LIKE ?", "%"+fileName+"%") + } + + var total int64 + if err := q.Count(&total).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + + var rows []model.LabelFileBinding + if err := q. + Order("id DESC"). + Offset((page - 1) * pageSize). + Limit(pageSize). + Find(&rows).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + return rows, total, nil +} + +func RestoreLabelFileBindings(bindings []model.LabelFileBinding, keepIDs bool, override bool) error { + if len(bindings) == 0 { + return nil + } + tx := db.Begin() + + if override { + type key struct { + uid uint + name string + } + toDel := make(map[key]struct{}, len(bindings)) + for i := range bindings { + k := key{uid: bindings[i].UserId, name: bindings[i].FileName} + toDel[k] = struct{}{} + } + for k := range toDel { + if err := tx.Where("user_id = ? AND file_name = ?", k.uid, k.name). + Delete(&model.LabelFileBinding{}).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } + } + + for i := range bindings { + b := bindings[i] + if !keepIDs { + b.ID = 0 + } + if b.CreateTime.IsZero() { + b.CreateTime = time.Now() + } + if override { + if err := tx.Create(&b).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } else { + if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&b).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } + } + + return errors.WithStack(tx.Commit().Error) +} diff --git a/internal/errs/role.go b/internal/errs/role.go index fbd67404822..a818ea21264 100644 --- a/internal/errs/role.go +++ b/internal/errs/role.go @@ -3,5 +3,5 @@ package errs import "errors" var ( - ErrChangeDefaultRole = errors.New("cannot modify admin or guest role") + ErrChangeDefaultRole = errors.New("cannot modify admin role") ) diff --git a/internal/model/label_file_binding.go b/internal/model/label_file_binding.go index 3f9ea3b2271..af57fed4d88 100644 --- a/internal/model/label_file_binding.go +++ b/internal/model/label_file_binding.go @@ -2,7 +2,7 @@ package model import "time" -type LabelFileBinDing struct { +type LabelFileBinding struct { ID uint `json:"id" gorm:"primaryKey"` // unique key UserId uint `json:"user_id"` // use to user_id LabelId uint `json:"label_id"` // use to label_id diff --git a/internal/op/label_file_binding.go b/internal/op/label_file_binding.go index 79137ed38af..2802f0c0b38 100644 --- a/internal/op/label_file_binding.go +++ b/internal/op/label_file_binding.go @@ -23,6 +23,7 @@ type CreateLabelFileBinDingReq struct { Type int `json:"type"` HashInfoStr string `json:"hashinfo"` LabelIds string `json:"label_ids"` + LabelIDs []uint64 `json:"labelIdList"` } type ObjLabelResp struct { @@ -54,23 +55,29 @@ func GetLabelByFileName(userId uint, fileName string) ([]model.Label, error) { return labels, nil } +func GetLabelsByFileNamesPublic(fileNames []string) (map[string][]model.Label, error) { + return db.GetLabelsByFileNamesPublic(fileNames) +} + func CreateLabelFileBinDing(req CreateLabelFileBinDingReq, userId uint) error { if err := db.DelLabelFileBinDingByFileName(userId, req.Name); err != nil { return errors.WithMessage(err, "failed del label_file_bin_ding in database") } - if req.LabelIds == "" { + + ids, err := collectLabelIDs(req) + if err != nil { + return err + } + if len(ids) == 0 { return nil } - labelMap := strings.Split(req.LabelIds, ",") - for _, value := range labelMap { - labelId, err := strconv.ParseUint(value, 10, 64) - if err != nil { - return fmt.Errorf("invalid label ID '%s': %v", value, err) - } - if err = db.CreateLabelFileBinDing(req.Name, uint(labelId), userId); err != nil { + + for _, id := range ids { + if err = db.CreateLabelFileBinDing(req.Name, uint(id), userId); err != nil { return errors.WithMessage(err, "failed labels in database") } } + if !db.GetFileByNameExists(req.Name) { objFile := model.ObjFile{ Id: req.Id, @@ -86,8 +93,7 @@ func CreateLabelFileBinDing(req CreateLabelFileBinDingReq, userId uint) error { Type: req.Type, HashInfoStr: req.HashInfoStr, } - err := db.CreateObjFile(objFile) - if err != nil { + if err := db.CreateObjFile(objFile); err != nil { return errors.WithMessage(err, "failed file in database") } } @@ -97,7 +103,7 @@ func CreateLabelFileBinDing(req CreateLabelFileBinDingReq, userId uint) error { func GetFileByLabel(userId uint, labelId string) (result []ObjLabelResp, err error) { labelMap := strings.Split(labelId, ",") var labelIds []uint - var labelsFile []model.LabelFileBinDing + var labelsFile []model.LabelFileBinding var labels []model.Label var labelsFileMap = make(map[string][]model.Label) var labelsMap = make(map[uint]model.Label) @@ -157,3 +163,33 @@ func StringSliceToUintSlice(strSlice []string) ([]uint, error) { } return uintSlice, nil } + +func RestoreLabelFileBindings(bindings []model.LabelFileBinding, keepIDs bool, override bool) error { + return db.RestoreLabelFileBindings(bindings, keepIDs, override) +} + +func collectLabelIDs(req CreateLabelFileBinDingReq) ([]uint64, error) { + if len(req.LabelIDs) > 0 { + return req.LabelIDs, nil + } + s := strings.TrimSpace(req.LabelIds) + if s == "" { + return nil, nil + } + replacer := strings.NewReplacer("，", ",", "、", ",", "；", ",", ";", ",") + s = replacer.Replace(s) + parts := strings.Split(s, ",") + ids := make([]uint64, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + id, err := strconv.ParseUint(p, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid label ID '%s': %v", p, err) + } + ids = append(ids, id) + } + return ids, nil +} diff --git a/server/handles/fsread.go b/server/handles/fsread.go index b49f0b646b9..cc403c4aeb1 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -114,7 +114,7 @@ func FsList(c *gin.Context) { provider = storage.GetStorage().Driver } common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath), user.ID), + Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), Total: int64(total), Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), @@ -224,12 +224,22 @@ func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { return total, objs[start:end] } -func toObjsResp(objs []model.Obj, parent string, encrypt bool, userId uint) []ObjLabelResp { +func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjLabelResp { var resp []ObjLabelResp + + names := make([]string, 0, len(objs)) + for _, obj := range objs { + if !obj.IsDir() { + names = append(names, obj.GetName()) + } + } + + labelsByName, _ := op.GetLabelsByFileNamesPublic(names) + for _, obj := range objs { var labels []model.Label - if obj.IsDir() == false { - labels, _ = op.GetLabelByFileName(userId, obj.GetName()) + if !obj.IsDir() { + labels = labelsByName[obj.GetName()] } thumb, _ := model.GetThumb(obj) resp = append(resp, ObjLabelResp{ @@ -369,7 +379,7 @@ func FsGet(c *gin.Context) { Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), Provider: provider, - Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath), user.ID), + Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)), }) } diff --git a/server/handles/label_file_binding.go b/server/handles/label_file_binding.go index 78af929b34e..04f0c105fc2 100644 --- a/server/handles/label_file_binding.go +++ b/server/handles/label_file_binding.go @@ -8,7 +8,9 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" + "net/url" "strconv" + "strings" ) type DelLabelFileBinDingReq struct { @@ -16,18 +18,36 @@ type DelLabelFileBinDingReq struct { LabelId string `json:"label_id"` } +type pageResp[T any] struct { + Content []T `json:"content"` + Total int64 `json:"total"` +} + +type restoreLabelBindingsReq struct { + KeepIDs bool `json:"keep_ids"` + Override bool `json:"override"` + Bindings []model.LabelFileBinding `json:"bindings"` +} + func GetLabelByFileName(c *gin.Context) { fileName := c.Query("file_name") if fileName == "" { common.ErrorResp(c, errors.New("file_name must not empty"), 400) return } + decodedFileName, err := url.QueryUnescape(fileName) + if err != nil { + common.ErrorResp(c, errors.New("invalid file_name"), 400) + return + } + fmt.Println(">>> 原始 fileName:", fileName) + fmt.Println(">>> 解码后 fileName:", decodedFileName) userObj, ok := c.Value("user").(*model.User) if !ok { common.ErrorStrResp(c, "user invalid", 401) return } - labels, err := op.GetLabelByFileName(userObj.ID, fileName) + labels, err := op.GetLabelByFileName(userObj.ID, decodedFileName) if err != nil { common.ErrorResp(c, err, 500, true) return @@ -101,3 +121,130 @@ func GetFileByLabel(c *gin.Context) { } common.SuccessResp(c, fileList) } + +func ListLabelFileBinding(c *gin.Context) { + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + + pageStr := c.DefaultQuery("page", "1") + sizeStr := c.DefaultQuery("page_size", "50") + page, err := strconv.Atoi(pageStr) + if err != nil || page <= 0 { + page = 1 + } + pageSize, err := strconv.Atoi(sizeStr) + if err != nil || pageSize <= 0 || pageSize > 200 { + pageSize = 50 + } + + fileName := c.Query("file_name") + labelIDStr := c.Query("label_id") + var labelIDs []uint + if labelIDStr != "" { + parts := strings.Split(labelIDStr, ",") + for _, p := range parts { + if p == "" { + continue + } + id64, err := strconv.ParseUint(strings.TrimSpace(p), 10, 64) + if err != nil { + common.ErrorResp(c, fmt.Errorf("invalid label_id '%s': %v", p, err), 400) + return + } + labelIDs = append(labelIDs, uint(id64)) + } + } + + list, total, err := db.ListLabelFileBinDing(userObj.ID, labelIDs, fileName, page, pageSize) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, pageResp[model.LabelFileBinding]{ + Content: list, + Total: total, + }) +} + +func RestoreLabelFileBinding(c *gin.Context) { + var req restoreLabelBindingsReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if len(req.Bindings) == 0 { + common.ErrorStrResp(c, "empty bindings", 400) + return + } + + if u, ok := c.Value("user").(*model.User); ok { + for i := range req.Bindings { + if req.Bindings[i].UserId == 0 { + req.Bindings[i].UserId = u.ID + } + } + } + + for i := range req.Bindings { + b := req.Bindings[i] + if b.UserId == 0 || b.LabelId == 0 || strings.TrimSpace(b.FileName) == "" { + common.ErrorStrResp(c, "invalid binding: user_id/label_id/file_name required", 400) + return + } + } + + if err := op.RestoreLabelFileBindings(req.Bindings, req.KeepIDs, req.Override); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, gin.H{ + "msg": fmt.Sprintf("restored %d rows", len(req.Bindings)), + }) +} + +func CreateLabelFileBinDingBatch(c *gin.Context) { + var req struct { + Items []op.CreateLabelFileBinDingReq `json:"items" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil || len(req.Items) == 0 { + common.ErrorResp(c, err, 400) + return + } + + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + + type perResult struct { + Name string `json:"name"` + Ok bool `json:"ok"` + ErrMsg string `json:"errMsg,omitempty"` + } + results := make([]perResult, 0, len(req.Items)) + succeed := 0 + + for _, item := range req.Items { + if item.IsDir { + results = append(results, perResult{Name: item.Name, Ok: false, ErrMsg: "Unable to bind folder"}) + continue + } + if err := op.CreateLabelFileBinDing(item, userObj.ID); err != nil { + results = append(results, perResult{Name: item.Name, Ok: false, ErrMsg: err.Error()}) + continue + } + succeed++ + results = append(results, perResult{Name: item.Name, Ok: true}) + } + + common.SuccessResp(c, gin.H{ + "total": len(req.Items), + "succeed": succeed, + "failed": len(req.Items) - succeed, + "results": results, + }) +} diff --git a/server/handles/user.go b/server/handles/user.go index d5eebba4780..b4c152c5a86 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -67,10 +67,10 @@ func UpdateUser(c *gin.Context) { common.ErrorStrResp(c, "cannot change role of admin user", 403) return } - if user.Username != req.Username { - common.ErrorStrResp(c, "cannot change username of admin user", 403) - return - } + //if user.Username != req.Username { + // common.ErrorStrResp(c, "cannot change username of admin user", 403) + // return + //} } if req.Password == "" { diff --git a/server/router.go b/server/router.go index bf43a6258f4..72546f4eb6b 100644 --- a/server/router.go +++ b/server/router.go @@ -92,6 +92,8 @@ func Init(e *gin.Engine) { _fs(auth.Group("/fs")) _task(auth.Group("/task", middlewares.AuthNotGuest)) + _label(auth.Group("/label")) + _labelFileBinding(auth.Group("/label_file_binding")) admin(auth.Group("/admin", middlewares.AuthAdmin)) if flags.Debug || flags.Dev { debug(g.Group("/debug")) @@ -170,17 +172,17 @@ func admin(g *gin.RouterGroup) { index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) label := g.Group("/label") - label.GET("/list", handles.ListLabel) - label.GET("/get", handles.GetLabel) label.POST("/create", handles.CreateLabel) label.POST("/update", handles.UpdateLabel) label.POST("/delete", handles.DeleteLabel) labelFileBinding := g.Group("/label_file_binding") - labelFileBinding.GET("/get", handles.GetLabelByFileName) - labelFileBinding.GET("/get_file_by_label", handles.GetFileByLabel) + labelFileBinding.GET("/list", handles.ListLabelFileBinding) labelFileBinding.POST("/create", handles.CreateLabelFileBinDing) + labelFileBinding.POST("/create_batch", handles.CreateLabelFileBinDingBatch) labelFileBinding.POST("/delete", handles.DelLabelByFileName) + labelFileBinding.POST("/restore", handles.RestoreLabelFileBinding) + } func _fs(g *gin.RouterGroup) { @@ -216,6 +218,16 @@ func _task(g *gin.RouterGroup) { handles.SetupTaskRoute(g) } +func _label(g *gin.RouterGroup) { + g.GET("/list", handles.ListLabel) + g.GET("/get", handles.GetLabel) +} + +func _labelFileBinding(g *gin.RouterGroup) { + g.GET("/get", handles.GetLabelByFileName) + g.GET("/get_file_by_label", handles.GetFileByLabel) +} + func Cors(r *gin.Engine) { config := cors.DefaultConfig() // config.AllowAllOrigins = true From fcfb3369d17150e6b09ed13f1da1b70caa0b3e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 Aug 2025 08:10:55 -0700 Subject: [PATCH 025/133] fix: webdav error location (#9266) * feat: improve WebDAV permission handling and user role fetching - Added logic to handle root permissions in WebDAV requests. - Improved the user role fetching mechanism. - Enhanced path checks and permission scopes in role_perm.go. - Set FetchRole function to avoid import cycles between modules. * fix(webdav): resolve connection reset issue by encoding paths - Adjust path encoding in webdav.go to prevent connection reset. - Utilize utils.EncodePath for correct path formatting. - Ensure proper handling of directory paths with trailing slash. * fix(webdav): resolve connection reset issue by encoding paths - Adjust path encoding in webdav.go to prevent connection reset. - Utilize utils.FixAndCleanPath for correct path formatting. - Ensure proper handling of directory paths with trailing slash. * fix: resolve webdav handshake error in permission checks - Updated role permission logic to handle bidirectional subpaths. - This adjustment fixes the issue where remote host terminates the handshake due to improper path matching. * fix: resolve webdav handshake error in permission checks (fix/fix-webdav-error) - Updated role permission logic to handle bidirectional subpaths, fixing handshake termination by remote host due to path mismatch. - Refactored function naming for consistency and clarity. - Enhanced filtering of objects based on user permissions. * fix: resolve webdav handshake error in permission checks - Updated role permission logic to handle bidirectional subpaths, fixing handshake termination by remote host due to path mismatch. - Refactored function naming for consistency and clarity. - Enhanced filtering of objects based on user permissions. --- internal/model/user.go | 18 ++++++- internal/op/role.go | 4 ++ server/common/role_perm.go | 41 ++++++++++++--- server/handles/fsread.go | 18 ++++++- server/webdav.go | 6 ++- server/webdav/file.go | 4 ++ server/webdav/webdav.go | 101 ++++++++++++++++++++++++++++++++++++- 7 files changed, 180 insertions(+), 12 deletions(-) diff --git a/internal/model/user.go b/internal/model/user.go index 221747a407e..8ea1ef1aaff 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -145,12 +145,15 @@ func (u *User) CheckPathLimit() bool { } func (u *User) JoinPath(reqPath string) (string, error) { + if reqPath == "/" { + return utils.FixAndCleanPath(u.BasePath), nil + } path, err := utils.JoinBasePath(u.BasePath, reqPath) if err != nil { return "", err } - if u.CheckPathLimit() { + if path != "/" && u.CheckPathLimit() { basePaths := GetAllBasePathsFromRoles(u) match := false for _, base := range basePaths { @@ -206,12 +209,23 @@ func (u *User) WebAuthnIcon() string { return "https://alistgo.com/logo.svg" } +// FetchRole is used to load role details by id. It should be set by the op package +// to avoid an import cycle between model and op. +var FetchRole func(uint) (*Role, error) + // GetAllBasePathsFromRoles returns all permission paths from user's roles func GetAllBasePathsFromRoles(u *User) []string { basePaths := make([]string, 0) seen := make(map[string]struct{}) - for _, role := range u.RolesDetail { + for _, rid := range u.Role { + if FetchRole == nil { + continue + } + role, err := FetchRole(uint(rid)) + if err != nil || role == nil { + continue + } for _, entry := range role.PermissionScopes { if entry.Path == "" { continue diff --git a/internal/op/role.go b/internal/op/role.go index e24ba5ab1a8..4d187506417 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -15,6 +15,10 @@ import ( var roleCache = cache.NewMemCache[*model.Role](cache.WithShards[*model.Role](2)) var roleG singleflight.Group[*model.Role] +func init() { + model.FetchRole = GetRole +} + func GetRole(id uint) (*model.Role, error) { key := fmt.Sprint(id) if r, ok := roleCache.Get(key); ok { diff --git a/server/common/role_perm.go b/server/common/role_perm.go index 1e539d966c5..36dedf98c5e 100644 --- a/server/common/role_perm.go +++ b/server/common/role_perm.go @@ -43,17 +43,23 @@ func MergeRolePermissions(u *model.User, reqPath string) int32 { if err != nil { continue } - for _, entry := range role.PermissionScopes { - if utils.IsSubPath(entry.Path, reqPath) { + if reqPath == "/" || utils.PathEqual(reqPath, u.BasePath) { + for _, entry := range role.PermissionScopes { perm |= entry.Permission } + } else { + for _, entry := range role.PermissionScopes { + if utils.IsSubPath(entry.Path, reqPath) { + perm |= entry.Permission + } + } } } return perm } func CanAccessWithRoles(u *model.User, meta *model.Meta, reqPath, password string) bool { - if !canReadPathByRole(u, reqPath) { + if !CanReadPathByRole(u, reqPath) { return false } perm := MergeRolePermissions(u, reqPath) @@ -78,7 +84,30 @@ func CanAccessWithRoles(u *model.User, meta *model.Meta, reqPath, password strin return meta.Password == password } -func canReadPathByRole(u *model.User, reqPath string) bool { +func CanReadPathByRole(u *model.User, reqPath string) bool { + if u == nil { + return false + } + if reqPath == "/" || utils.PathEqual(reqPath, u.BasePath) { + return len(u.Role) > 0 + } + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + for _, entry := range role.PermissionScopes { + if utils.PathEqual(entry.Path, reqPath) || utils.IsSubPath(entry.Path, reqPath) || utils.IsSubPath(reqPath, entry.Path) { + return true + } + } + } + return false +} + +// HasChildPermission checks whether any child path under reqPath grants the +// specified permission bit. +func HasChildPermission(u *model.User, reqPath string, bit uint) bool { if u == nil { return false } @@ -88,7 +117,7 @@ func canReadPathByRole(u *model.User, reqPath string) bool { continue } for _, entry := range role.PermissionScopes { - if utils.IsSubPath(entry.Path, reqPath) { + if utils.IsSubPath(reqPath, entry.Path) && HasPermission(entry.Permission, bit) { return true } } @@ -102,7 +131,7 @@ func canReadPathByRole(u *model.User, reqPath string) bool { func CheckPathLimitWithRoles(u *model.User, reqPath string) bool { perm := MergeRolePermissions(u, reqPath) if HasPermission(perm, PermPathLimit) { - return canReadPathByRole(u, reqPath) + return CanReadPathByRole(u, reqPath) } return true } diff --git a/server/handles/fsread.go b/server/handles/fsread.go index cc403c4aeb1..8cf3c9b0290 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -107,7 +107,14 @@ func FsList(c *gin.Context) { common.ErrorResp(c, err, 500) return } - total, objs := pagination(objs, &req.PageReq) + filtered := make([]model.Obj, 0, len(objs)) + for _, obj := range objs { + childPath := stdpath.Join(reqPath, obj.GetName()) + if common.CanReadPathByRole(user, childPath) { + filtered = append(filtered, obj) + } + } + total, objs := pagination(filtered, &req.PageReq) provider := "unknown" storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) if err == nil { @@ -161,7 +168,14 @@ func FsDirs(c *gin.Context) { common.ErrorResp(c, err, 500) return } - dirs := filterDirs(objs) + visible := make([]model.Obj, 0, len(objs)) + for _, obj := range objs { + childPath := stdpath.Join(reqPath, obj.GetName()) + if common.CanReadPathByRole(user, childPath) { + visible = append(visible, obj) + } + } + dirs := filterDirs(visible) common.SuccessResp(c, dirs) } diff --git a/server/webdav.go b/server/webdav.go index a65896dfc79..582c469d73a 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -95,6 +95,9 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } + if roles, err := op.GetRolesByUserID(user.ID); err == nil { + user.RolesDetail = roles + } reqPath := c.Param("path") if reqPath == "" { reqPath = "/" @@ -107,7 +110,8 @@ func WebDAVAuth(c *gin.Context) { return } perm := common.MergeRolePermissions(user, reqPath) - if user.Disabled || !common.HasPermission(perm, common.PermWebdavRead) { + webdavRead := common.HasPermission(perm, common.PermWebdavRead) + if user.Disabled || (!webdavRead && (c.Request.Method != "PROPFIND" || !common.HasChildPermission(user, reqPath, common.PermWebdavRead))) { if c.Request.Method == "OPTIONS" { c.Set("user", guest) c.Next() diff --git a/server/webdav/file.go b/server/webdav/file.go index ab78d26105d..419c7b07207 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -94,6 +94,7 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn depth = 0 } meta, _ := op.GetNearestMeta(name) + user := ctx.Value("user").(*model.User) // Read directory names. objs, err := fs.List(context.WithValue(ctx, "meta", meta), name, &fs.ListArgs{}) //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) @@ -108,6 +109,9 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn for _, fileInfo := range objs { filename := path.Join(name, fileInfo.GetName()) + if !common.CanReadPathByRole(user, filename) { + continue + } if err != nil { if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { return err diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index f22e15aadb9..dde73559f8a 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -648,6 +648,98 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status mw := multistatusWriter{w: w} + if utils.PathEqual(reqPath, user.BasePath) { + hasRootPerm := false + for _, role := range user.RolesDetail { + for _, entry := range role.PermissionScopes { + if utils.PathEqual(entry.Path, user.BasePath) { + hasRootPerm = true + break + } + } + if hasRootPerm { + break + } + } + if !hasRootPerm { + basePaths := model.GetAllBasePathsFromRoles(user) + type infoItem struct { + path string + info model.Obj + } + infos := []infoItem{{reqPath, fi}} + seen := make(map[string]struct{}) + for _, p := range basePaths { + if !utils.IsSubPath(user.BasePath, p) { + continue + } + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(p), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + dir := strings.Split(rel, "/")[0] + if dir == "" { + continue + } + if _, ok := seen[dir]; ok { + continue + } + seen[dir] = struct{}{} + sp := utils.FixAndCleanPath(path.Join(user.BasePath, dir)) + info, err := fs.Get(ctx, sp, &fs.GetArgs{}) + if err != nil { + continue + } + infos = append(infos, infoItem{sp, info}) + } + for _, item := range infos { + var pstats []Propstat + if pf.Propname != nil { + pnames, err := propnames(ctx, h.LockSystem, item.info) + if err != nil { + return http.StatusInternalServerError, err + } + pstat := Propstat{Status: http.StatusOK} + for _, xmlname := range pnames { + pstat.Props = append(pstat.Props, Property{XMLName: xmlname}) + } + pstats = append(pstats, pstat) + } else if pf.Allprop != nil { + pstats, err = allprop(ctx, h.LockSystem, item.info, pf.Prop) + if err != nil { + return http.StatusInternalServerError, err + } + } else { + pstats, err = props(ctx, h.LockSystem, item.info, pf.Prop) + if err != nil { + return http.StatusInternalServerError, err + } + } + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(item.path), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + href := utils.EncodePath(path.Join("/", h.Prefix, rel), true) + if href != "/" && item.info.IsDir() { + href += "/" + } + if err := mw.write(makePropstatResponse(href, pstats)); err != nil { + return http.StatusInternalServerError, err + } + } + if err := mw.close(); err != nil { + return http.StatusInternalServerError, err + } + return 0, nil + } + } + walkFn := func(reqPath string, info model.Obj, err error) error { if err != nil { return err @@ -671,7 +763,14 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status if err != nil { return err } - href := path.Join(h.Prefix, strings.TrimPrefix(reqPath, user.BasePath)) + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(reqPath), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + href := utils.EncodePath(path.Join("/", h.Prefix, rel), true) if href != "/" && info.IsDir() { href += "/" } From 97d4f79b96330ad782c9aa8f122b43d9a37fd6b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sat, 16 Aug 2025 05:55:17 -0700 Subject: [PATCH 026/133] fix: resolve webdav decode issue (#9268) * fix: resolve webdav handshake error in permission checks - Updated role permission logic to handle bidirectional subpaths, fixing handshake termination by remote host due to path mismatch. - Refactored function naming for consistency and clarity. - Enhanced filtering of objects based on user permissions. - Modified `makePropstatResponse` to preserve encoded href paths. - Added test for `makePropstatResponse` to ensure href encoding. * Delete server/webdav/makepropstatresponse_test.go * ci(workflow): set GOPROXY for Go builds on GitHub Actions - Use `GOPROXY=https://proxy.golang.org,direct` to speed up module downloads - Mitigates network flakiness (e.g., checksum DB timeouts/rate limits) - `,direct` provides fallback for private/unproxyable modules - No build logic changes; only affects dependency resolution across all matrix targets --------- Co-authored-by: AlistGo --- .github/workflows/build.yml | 2 ++ server/webdav/webdav.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2c934e7a5e..cf6eff39e48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,8 @@ jobs: - android-arm64 name: Build runs-on: ${{ matrix.platform }} + env: + GOPROXY: https://proxy.golang.org,direct steps: - name: Checkout diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index dde73559f8a..93211e8a77e 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -833,7 +833,7 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu func makePropstatResponse(href string, pstats []Propstat) *response { resp := response{ - Href: []string{(&url.URL{Path: href}).EscapedPath()}, + Href: []string{href}, Propstat: make([]propstat, 0, len(pstats)), } for _, p := range pstats { From eca500861a8b74a2780789f595bc5f5a492b64ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 18 Aug 2025 16:38:21 +0800 Subject: [PATCH 027/133] feat: add user registration endpoint and role-based default settings (#9277) * feat(setting): add role-based default and registration settings (closed #feat/register-and-statistics) - Added `AllowRegister` and `DefaultRole` settings to site configuration. - Integrated dynamic role options for `DefaultRole` using `op.GetRoles`. - Updated `setting.go` handlers to manage `DefaultRole` options dynamically. - Modified `const.go` to include new site settings constants. - Updated dependencies in `go.mod` and `go.sum` to support new functionality. * feat(register-and-statistics): add user registration endpoint - Added `POST /auth/register` endpoint to support user registration. - Implemented registration logic in `auth.go` with dynamic role assignment. - Integrated settings `AllowRegister` and `DefaultRole` for registration flow. - Updated imports to include new modules: `conf`, `setting`. - Adjusted user creation logic to use `DefaultRole` setting dynamically. * feat(register-and-statistics): add user registration endpoint (#register-and-statistics) - Added `POST /auth/register` endpoint to support user registration. - Implemented registration logic in `auth.go` with dynamic role assignment. - Integrated `AllowRegister` and `DefaultRole` settings for registration flow. - Updated imports to include new modules: `conf`, `setting`. - Adjusted user creation logic to use `DefaultRole` dynamically. * feat(register-and-statistics): enhance role management logic (#register-and-statistics) - Refactored CreateRole and UpdateRole functions to handle default role. - Added dynamic role assignment logic in 'role.go' using conf settings. - Improved request handling in 'handles/role.go' with structured data. - Implemented default role logic in 'db/role.go' to update non-default roles. - Modified 'model/role.go' to include a 'Default' field for role management. * feat(register-and-statistics): enhance role management logic - Refactor CreateRole and UpdateRole to handle default roles. - Add dynamic role assignment using conf settings in 'role.go'. - Improve request handling with structured data in 'handles/role.go'. - Implement default role logic in 'db/role.go' for non-default roles. - Modify 'model/role.go' to include 'Default' field for role management. * feat(register-and-statistics): improve role handling logic - Switch from role names to role IDs for better consistency. - Update logic to prioritize "guest" for default role ID. - Adjust `DefaultRole` setting to use role IDs. - Refactor `getRoleOptions` to return role IDs as a comma-separated string. * feat(register-and-statistics): improve role handling logic --- go.mod | 9 ++---- go.sum | 13 ++++++-- internal/bootstrap/data/setting.go | 18 +++++++++++ internal/conf/const.go | 16 ++++++---- internal/db/role.go | 20 ++++++++++-- internal/model/role.go | 1 + internal/op/hook.go | 13 ++++++++ internal/op/role.go | 51 ++++++++++++++++++++++++++++-- server/handles/auth.go | 31 ++++++++++++++++++ server/handles/role.go | 20 +++++++++--- server/handles/setting.go | 30 ++++++++++++++++++ server/handles/user.go | 3 ++ server/router.go | 1 + 13 files changed, 202 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index e8afe0e7a62..5bc953fb4de 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/alist-org/alist/v3 go 1.23.4 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 @@ -79,11 +81,7 @@ require ( gorm.io/gorm v1.25.11 ) -require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 // indirect -) +require github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect require ( github.com/STARRY-S/zip v0.2.1 // indirect @@ -109,7 +107,6 @@ require ( github.com/ipfs/boxo v0.12.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/klauspost/pgzip v1.2.6 // indirect - github.com/kr/text v0.2.0 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 diff --git a/go.sum b/go.sum index 6fbaeb2b3ef..a9faa92fa69 100644 --- a/go.sum +++ b/go.sum @@ -21,10 +21,16 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= +github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -172,7 +178,6 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -398,6 +403,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/larksuite/oapi-sdk-go/v3 v3.3.1 h1:DLQQEgHUAGZB6RVlceB1f6A94O206exxW2RIMH+gMUc= github.com/larksuite/oapi-sdk-go/v3 v3.3.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= @@ -492,6 +499,8 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -739,8 +748,6 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= -golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index e00abf2d379..39c3b1be571 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -2,6 +2,7 @@ package data import ( "strconv" + "strings" "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" @@ -91,6 +92,21 @@ func InitialSettings() []model.SettingItem { } else { token = random.Token() } + roles, _, err := op.GetRoles(1, model.MaxInt) + if err != nil { + utils.Log.Fatalf("failed get roles: %+v", err) + } + roleNames := make([]string, len(roles)) + defaultRoleID := "" + for i, role := range roles { + roleNames[i] = role.Name + if role.Name == "guest" { + defaultRoleID = strconv.Itoa(int(role.ID)) + } + } + if defaultRoleID == "" && len(roles) > 0 { + defaultRoleID = strconv.Itoa(int(roles[0].ID)) + } initialSettingItems = []model.SettingItem{ // site settings {Key: conf.VERSION, Value: conf.Version, Type: conf.TypeString, Group: model.SITE, Flag: model.READONLY}, @@ -103,6 +119,8 @@ func InitialSettings() []model.SettingItem { {Key: conf.AllowIndexed, Value: "false", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, + {Key: conf.AllowRegister, Value: "false", Type: conf.TypeBool, Group: model.SITE}, + {Key: conf.DefaultRole, Value: defaultRoleID, Type: conf.TypeSelect, Options: strings.Join(roleNames, ","), Group: model.SITE}, // newui settings {Key: conf.UseNewui, Value: "false", Type: conf.TypeBool, Group: model.SITE}, // style settings diff --git a/internal/conf/const.go b/internal/conf/const.go index 48ac2037fb2..0bf0cd67f7e 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -10,13 +10,15 @@ const ( const ( // site - VERSION = "version" - SiteTitle = "site_title" - Announcement = "announcement" - AllowIndexed = "allow_indexed" - AllowMounted = "allow_mounted" - RobotsTxt = "robots_txt" - UseNewui = "use_newui" + VERSION = "version" + SiteTitle = "site_title" + Announcement = "announcement" + AllowIndexed = "allow_indexed" + AllowMounted = "allow_mounted" + RobotsTxt = "robots_txt" + AllowRegister = "allow_register" + DefaultRole = "default_role" + UseNewui = "use_newui" Logo = "logo" Favicon = "favicon" diff --git a/internal/db/role.go b/internal/db/role.go index e6d0d9568b4..ae62a8ed898 100644 --- a/internal/db/role.go +++ b/internal/db/role.go @@ -35,11 +35,27 @@ func GetRoles(pageIndex, pageSize int) (roles []model.Role, count int64, err err } func CreateRole(r *model.Role) error { - return errors.WithStack(db.Create(r).Error) + if err := db.Create(r).Error; err != nil { + return errors.WithStack(err) + } + if r.Default { + if err := db.Model(&model.Role{}).Where("id <> ?", r.ID).Update("default", false).Error; err != nil { + return errors.WithStack(err) + } + } + return nil } func UpdateRole(r *model.Role) error { - return errors.WithStack(db.Save(r).Error) + if err := db.Save(r).Error; err != nil { + return errors.WithStack(err) + } + if r.Default { + if err := db.Model(&model.Role{}).Where("id <> ?", r.ID).Update("default", false).Error; err != nil { + return errors.WithStack(err) + } + } + return nil } func DeleteRole(id uint) error { diff --git a/internal/model/role.go b/internal/model/role.go index ecc9aee29d1..87855551ddb 100644 --- a/internal/model/role.go +++ b/internal/model/role.go @@ -17,6 +17,7 @@ type Role struct { ID uint `json:"id" gorm:"primaryKey"` Name string `json:"name" gorm:"unique" binding:"required"` Description string `json:"description"` + Default bool `json:"default" gorm:"default:false"` // PermissionScopes stores structured permission list and is ignored by gorm. PermissionScopes []PermissionEntry `json:"permission_scopes" gorm:"-"` // RawPermission is the JSON representation of PermissionScopes stored in DB. diff --git a/internal/op/hook.go b/internal/op/hook.go index 23b8e59af2c..08ea4603e5c 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -2,6 +2,7 @@ package op import ( "regexp" + "strconv" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -82,6 +83,18 @@ var settingItemHooks = map[string]SettingItemHook{ conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",") return nil }, + conf.DefaultRole: func(item *model.SettingItem) error { + v := strings.TrimSpace(item.Value) + if v == "" { + return nil + } + r, err := GetRoleByName(v) + if err != nil { + return err + } + item.Value = strconv.Itoa(int(r.ID)) + return nil + }, } func RegisterSettingItemHook(key string, hook SettingItemHook) { diff --git a/internal/op/role.go b/internal/op/role.go index 4d187506417..5c9aad06f41 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -2,9 +2,11 @@ package op import ( "fmt" + "strconv" "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -50,6 +52,23 @@ func GetRoleByName(name string) (*model.Role, error) { return r, err } +func GetDefaultRoleID() int { + item, err := GetSettingItemByKey(conf.DefaultRole) + if err == nil && item != nil && item.Value != "" { + if id, err := strconv.Atoi(item.Value); err == nil && id != 0 { + return id + } + if r, err := db.GetRoleByName(item.Value); err == nil { + return int(r.ID) + } + } + var r model.Role + if err := db.GetDb().Where("`default` = ?", true).First(&r).Error; err == nil { + return int(r.ID) + } + return int(model.GUEST) +} + func GetRolesByUserID(userID uint) ([]model.Role, error) { user, err := GetUserById(userID) if err != nil { @@ -92,7 +111,21 @@ func CreateRole(r *model.Role) error { } roleCache.Del(fmt.Sprint(r.ID)) roleCache.Del(r.Name) - return db.CreateRole(r) + if err := db.CreateRole(r); err != nil { + return err + } + if r.Default { + roleCache.Clear() + item, err := GetSettingItemByKey(conf.DefaultRole) + if err != nil { + return err + } + item.Value = strconv.Itoa(int(r.ID)) + if err := SaveSettingItem(item); err != nil { + return err + } + } + return nil } func UpdateRole(r *model.Role) error { @@ -131,7 +164,21 @@ func UpdateRole(r *model.Role) error { //} roleCache.Del(fmt.Sprint(r.ID)) roleCache.Del(r.Name) - return db.UpdateRole(r) + if err := db.UpdateRole(r); err != nil { + return err + } + if r.Default { + roleCache.Clear() + item, err := GetSettingItemByKey(conf.DefaultRole) + if err != nil { + return err + } + item.Value = strconv.Itoa(int(r.ID)) + if err := SaveSettingItem(item); err != nil { + return err + } + } + return nil } func DeleteRole(id uint) error { diff --git a/server/handles/auth.go b/server/handles/auth.go index 96a9ba9e20d..26447ddd594 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -9,8 +9,10 @@ import ( "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" @@ -89,6 +91,35 @@ func loginHash(c *gin.Context, req *LoginReq) { loginCache.Del(ip) } +type RegisterReq struct { + Username string `json:"username" binding:"required"` + Password string `json:"password" binding:"required"` +} + +// Register a new user +func Register(c *gin.Context) { + if !setting.GetBool(conf.AllowRegister) { + common.ErrorStrResp(c, "registration is disabled", 403) + return + } + var req RegisterReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := &model.User{ + Username: req.Username, + Role: model.Roles{op.GetDefaultRoleID()}, + Authn: "[]", + } + user.SetPassword(req.Password) + if err := op.CreateUser(user); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + type UserResp struct { model.User Otp bool `json:"otp"` diff --git a/server/handles/role.go b/server/handles/role.go index 0d071c9f84d..17271a530de 100644 --- a/server/handles/role.go +++ b/server/handles/role.go @@ -44,7 +44,7 @@ func GetRole(c *gin.Context) { func CreateRole(c *gin.Context) { var req model.Role - if err := c.ShouldBind(&req); err != nil { + if err := c.ShouldBindJSON(&req); err != nil { common.ErrorResp(c, err, 400) return } @@ -56,8 +56,14 @@ func CreateRole(c *gin.Context) { } func UpdateRole(c *gin.Context) { - var req model.Role - if err := c.ShouldBind(&req); err != nil { + var req struct { + ID uint `json:"id"` + Name string `json:"name" binding:"required"` + Description string `json:"description"` + PermissionScopes []model.PermissionEntry `json:"permission_scopes"` + Default *bool `json:"default"` + } + if err := c.ShouldBindJSON(&req); err != nil { common.ErrorResp(c, err, 400) return } @@ -74,7 +80,13 @@ func UpdateRole(c *gin.Context) { case "guest": req.Name = "guest" } - if err := op.UpdateRole(&req); err != nil { + role.Name = req.Name + role.Description = req.Description + role.PermissionScopes = req.PermissionScopes + if req.Default != nil { + role.Default = *req.Default + } + if err := op.UpdateRole(role); err != nil { common.ErrorResp(c, err, 500, true) } else { common.SuccessResp(c) diff --git a/server/handles/setting.go b/server/handles/setting.go index f778b1803c5..3ce5fcbf94f 100644 --- a/server/handles/setting.go +++ b/server/handles/setting.go @@ -14,6 +14,18 @@ import ( "github.com/gin-gonic/gin" ) +func getRoleOptions() string { + roles, _, err := op.GetRoles(1, model.MaxInt) + if err != nil { + return "" + } + names := make([]string, len(roles)) + for i, r := range roles { + names[i] = r.Name + } + return strings.Join(names, ",") +} + func ResetToken(c *gin.Context) { token := random.Token() item := model.SettingItem{Key: "token", Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} @@ -34,6 +46,12 @@ func GetSetting(c *gin.Context) { common.ErrorResp(c, err, 400) return } + if item.Key == conf.DefaultRole { + copy := *item + copy.Options = getRoleOptions() + common.SuccessResp(c, copy) + return + } common.SuccessResp(c, item) } else { items, err := op.GetSettingItemInKeys(strings.Split(keys, ",")) @@ -41,6 +59,12 @@ func GetSetting(c *gin.Context) { common.ErrorResp(c, err, 400) return } + for i := range items { + if items[i].Key == conf.DefaultRole { + items[i].Options = getRoleOptions() + break + } + } common.SuccessResp(c, items) } } @@ -88,6 +112,12 @@ func ListSettings(c *gin.Context) { common.ErrorResp(c, err, 400) return } + for i := range settings { + if settings[i].Key == conf.DefaultRole { + settings[i].Options = getRoleOptions() + break + } + } common.SuccessResp(c, settings) } diff --git a/server/handles/user.go b/server/handles/user.go index b4c152c5a86..01368beec6e 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -36,6 +36,9 @@ func CreateUser(c *gin.Context) { common.ErrorResp(c, err, 400) return } + if len(req.Role) == 0 { + req.Role = model.Roles{op.GetDefaultRoleID()} + } if req.IsAdmin() || req.IsGuest() { common.ErrorStrResp(c, "admin or guest user can not be created", 400, true) return diff --git a/server/router.go b/server/router.go index 72546f4eb6b..e8902f7a648 100644 --- a/server/router.go +++ b/server/router.go @@ -61,6 +61,7 @@ func Init(e *gin.Engine) { api.POST("/auth/login", handles.Login) api.POST("/auth/login/hash", handles.LoginHash) api.POST("/auth/login/ldap", handles.LoginLdap) + api.POST("/auth/register", handles.Register) auth.GET("/me", handles.CurrentUser) auth.POST("/me/update", handles.UpdateCurrent) auth.GET("/me/sshkey/list", handles.ListMyPublicKey) From 74e384175b25a0a1968c1aa2e75b720b23104727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 19 Aug 2025 00:53:52 +0800 Subject: [PATCH 028/133] fix(lanzou): correct comment parsing logic in lanzou driver (#9278) - Adjusted logic to skip incrementing index when exiting comments. - Added checks to continue loop if inside a single-line or block comment. - Prevents erroneous parsing and retains intended comment exclusion. --- drivers/lanzou/help.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index c3f5c6bb5bc..b3d69006791 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -94,6 +94,7 @@ func RemoveJSComment(data string) string { } if inComment && v == '*' && i+1 < len(data) && data[i+1] == '/' { inComment = false + i++ continue } if v == '/' && i+1 < len(data) { @@ -108,6 +109,9 @@ func RemoveJSComment(data string) string { continue } } + if inComment || inSingleLineComment { + continue + } result.WriteByte(v) } From a9fcd51bc4f0e95127bf486690a10023da05241d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 19 Aug 2025 15:01:32 +0800 Subject: [PATCH 029/133] fix: ensure DefaultRole stores role ID while exposing role name in APIs (#9279) * fix(setting): ensure DefaultRole stores role ID while exposing role name in APIs - Simplified initial settings to use `model.GUEST` as the default role ID instead of querying roles at startup. - Updated `GetSetting`, `ListSettings` handlers to: - Convert stored role ID into the corresponding role name when returning data. - Preserve dynamic role options for selection. - Removed unused `strings` import and role preloading logic from `InitialSettings`. - This change avoids DB dependency during initialization while keeping consistent role display for frontend clients. * fix(setting): ensure DefaultRole stores role ID while exposing role name in APIs (fix/settings-get-role) - Simplify initial settings to use `model.GUEST` as the default role ID instead of querying roles at startup. - Update `GetSetting`, `ListSettings` handlers to: - Convert stored role ID into the corresponding role name when returning data. - Preserve dynamic role options for selection. - Remove unused `strings` import and role preloading logic from `InitialSettings`. - Avoid DB dependency during initialization while keeping consistent role display for frontend clients. --- internal/bootstrap/data/setting.go | 19 ++------------ internal/op/hook.go | 8 +++--- server/handles/setting.go | 40 +++++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 39c3b1be571..17a63af21c2 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -2,7 +2,6 @@ package data import ( "strconv" - "strings" "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" @@ -92,21 +91,7 @@ func InitialSettings() []model.SettingItem { } else { token = random.Token() } - roles, _, err := op.GetRoles(1, model.MaxInt) - if err != nil { - utils.Log.Fatalf("failed get roles: %+v", err) - } - roleNames := make([]string, len(roles)) - defaultRoleID := "" - for i, role := range roles { - roleNames[i] = role.Name - if role.Name == "guest" { - defaultRoleID = strconv.Itoa(int(role.ID)) - } - } - if defaultRoleID == "" && len(roles) > 0 { - defaultRoleID = strconv.Itoa(int(roles[0].ID)) - } + defaultRoleID := strconv.Itoa(model.GUEST) initialSettingItems = []model.SettingItem{ // site settings {Key: conf.VERSION, Value: conf.Version, Type: conf.TypeString, Group: model.SITE, Flag: model.READONLY}, @@ -120,7 +105,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, {Key: conf.AllowRegister, Value: "false", Type: conf.TypeBool, Group: model.SITE}, - {Key: conf.DefaultRole, Value: defaultRoleID, Type: conf.TypeSelect, Options: strings.Join(roleNames, ","), Group: model.SITE}, + {Key: conf.DefaultRole, Value: defaultRoleID, Type: conf.TypeSelect, Group: model.SITE}, // newui settings {Key: conf.UseNewui, Value: "false", Type: conf.TypeBool, Group: model.SITE}, // style settings diff --git a/internal/op/hook.go b/internal/op/hook.go index 08ea4603e5c..f08966c4ade 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -88,12 +88,12 @@ var settingItemHooks = map[string]SettingItemHook{ if v == "" { return nil } - r, err := GetRoleByName(v) + id, err := strconv.Atoi(v) if err != nil { - return err + return errors.WithStack(err) } - item.Value = strconv.Itoa(int(r.ID)) - return nil + _, err = GetRole(uint(id)) + return err }, } diff --git a/server/handles/setting.go b/server/handles/setting.go index 3ce5fcbf94f..e0dbb490033 100644 --- a/server/handles/setting.go +++ b/server/handles/setting.go @@ -19,9 +19,12 @@ func getRoleOptions() string { if err != nil { return "" } - names := make([]string, len(roles)) - for i, r := range roles { - names[i] = r.Name + names := make([]string, 0, len(roles)) + for _, r := range roles { + if r.Name == "admin" || r.Name == "guest" { + continue + } + names = append(names, r.Name) } return strings.Join(names, ",") } @@ -49,6 +52,11 @@ func GetSetting(c *gin.Context) { if item.Key == conf.DefaultRole { copy := *item copy.Options = getRoleOptions() + if id, err := strconv.Atoi(copy.Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + copy.Value = r.Name + } + } common.SuccessResp(c, copy) return } @@ -61,6 +69,11 @@ func GetSetting(c *gin.Context) { } for i := range items { if items[i].Key == conf.DefaultRole { + if id, err := strconv.Atoi(items[i].Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + items[i].Value = r.Name + } + } items[i].Options = getRoleOptions() break } @@ -75,6 +88,22 @@ func SaveSettings(c *gin.Context) { common.ErrorResp(c, err, 400) return } + + for i := range req { + if req[i].Key == conf.DefaultRole { + role, err := op.GetRoleByName(req[i].Value) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if role.Name == "admin" || role.Name == "guest" { + common.ErrorStrResp(c, "cannot set admin or guest as default role", 400) + return + } + req[i].Value = strconv.Itoa(int(role.ID)) + } + } + if err := op.SaveSettingItems(req); err != nil { common.ErrorResp(c, err, 500) } else { @@ -114,6 +143,11 @@ func ListSettings(c *gin.Context) { } for i := range settings { if settings[i].Key == conf.DefaultRole { + if id, err := strconv.Atoi(settings[i].Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + settings[i].Value = r.Name + } + } settings[i].Options = getRoleOptions() break } From d7723c378f67f13e38d9b5f5acd75ea07741c964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 25 Aug 2025 19:46:10 +0800 Subject: [PATCH 030/133] chore(deps): Upgrade 115driver to v1.1.1 (#9283) - Upgraded `github.com/SheltonZhu/115driver` from v1.0.34 to v1.1.1 - Updated the corresponding version verification information in `go.sum` --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5bc953fb4de..9923c489eef 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 - github.com/SheltonZhu/115driver v1.0.34 + github.com/SheltonZhu/115driver v1.1.1 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 diff --git a/go.sum b/go.sum index a9faa92fa69..e47d96234ae 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,8 @@ github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4 github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.0.34 h1:zhMLp4vgq7GksqvSxQQDOVfK6EOHldQl4b2n8tnZ+EE= -github.com/SheltonZhu/115driver v1.0.34/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= +github.com/SheltonZhu/115driver v1.1.1 h1:9EMhe2ZJflGiAaZbYInw2jqxTcqZNF+DtVDsEy70aFU= +github.com/SheltonZhu/115driver v1.1.1/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= From 3319f6ea6ad4e2a388e27f4d231a4aa5459aad4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 25 Aug 2025 19:46:24 +0800 Subject: [PATCH 031/133] feat(search): Optimized search result filtering and paging logic (#9287) - Introduced the `filteredNodes` list to optimize the node filtering process - Filtered results based on the page limit during paging - Modified search logic to ensure nodes are within the user's base path - Added access permission checks for node metadata - Adjusted paging logic to avoid redundant node retrieval --- server/handles/search.go | 43 +++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/server/handles/search.go b/server/handles/search.go index 7d421a21e59..832fc94fb07 100644 --- a/server/handles/search.go +++ b/server/handles/search.go @@ -43,28 +43,39 @@ func Search(c *gin.Context) { common.ErrorResp(c, err, 400) return } - nodes, total, err := search.Search(c, req.SearchReq) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - var filteredNodes []model.SearchNode - for _, node := range nodes { - if !strings.HasPrefix(node.Parent, user.BasePath) { - continue + var ( + filteredNodes []model.SearchNode + ) + for len(filteredNodes) < req.PerPage { + nodes, _, err := search.Search(c, req.SearchReq) + if err != nil { + common.ErrorResp(c, err, 500) + return } - meta, err := op.GetNearestMeta(node.Parent) - if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { - continue + if len(nodes) == 0 { + break } - if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), req.Password) { - continue + for _, node := range nodes { + if !strings.HasPrefix(node.Parent, user.BasePath) { + continue + } + meta, err := op.GetNearestMeta(node.Parent) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + continue + } + if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), req.Password) { + continue + } + filteredNodes = append(filteredNodes, node) + if len(filteredNodes) >= req.PerPage { + break + } } - filteredNodes = append(filteredNodes, node) + req.Page++ } common.SuccessResp(c, common.PageResp{ Content: utils.MustSliceConvert(filteredNodes, nodeToSearchResp), - Total: total, + Total: int64(len(filteredNodes)), }) } From c64f899a636b66d12056c6ae3fcab6c21b0cb146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 25 Aug 2025 19:46:38 +0800 Subject: [PATCH 032/133] feat: implement session management (#9286) * feat(auth): Added device session management - Added the `handleSession` function to manage user device sessions and verify client identity - Updated `auth.go` to call `handleSession` for device handling when a user logs in - Added the `Session` model to database migrations - Added `device.go` and `session.go` files to handle device session logic - Updated `settings.go` to add device-related configuration items, such as the maximum number of devices, device eviction policy, and session TTL * feat(session): Adds session management features - Added `SessionInactive` error type in `device.go` - Added session-related APIs in `router.go` to support listing and evicting sessions - Added `ListSessionsByUser`, `ListSessions`, and `MarkInactive` methods in `session.go` - Returns an appropriate error when the session state is `SessionInactive` * feat(auth): Marks the device session as invalid. - Import the `session` package into the `auth` module to handle device session status. - Add a check in the login logic. If `device_key` is obtained, call `session.MarkInactive` to mark the device session as invalid. - Store the invalid status in the context variable `session_inactive` for subsequent middleware checks. - Add a check in the session refresh logic to abort the process if the current session has been marked invalid. * feat(auth, session): Added device information processing and session management changes - Updated device handling logic in `auth.go` to pass user agent and IP information - Adjusted database queries in `session.go` to optimize session query fields and add `user_agent` and `ip` fields - Modified the `Handle` method to add `ua` and `ip` parameters to store the user agent and IP address - Added the `SessionResp` structure to return a session response containing `user_agent` and `ip` - Updated the `/admin/user/create` and `/webdav` endpoints to pass the user agent and IP address to the device handler --- internal/bootstrap/data/setting.go | 3 + internal/conf/const.go | 3 + internal/db/db.go | 2 +- internal/db/session.go | 65 +++++++++++++++++++ internal/device/session.go | 67 +++++++++++++++++++ internal/errs/device.go | 8 +++ internal/model/session.go | 16 +++++ internal/session/session.go | 8 +++ pkg/utils/mask.go | 30 +++++++++ server/handles/auth.go | 8 +++ server/handles/session.go | 92 +++++++++++++++++++++++++++ server/middlewares/auth.go | 30 ++++++++- server/middlewares/session_refresh.go | 26 ++++++++ server/router.go | 7 ++ server/webdav.go | 17 +++++ 15 files changed, 378 insertions(+), 4 deletions(-) create mode 100644 internal/db/session.go create mode 100644 internal/device/session.go create mode 100644 internal/errs/device.go create mode 100644 internal/model/session.go create mode 100644 internal/session/session.go create mode 100644 pkg/utils/mask.go create mode 100644 server/handles/session.go create mode 100644 server/middlewares/session_refresh.go diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 17a63af21c2..bbb633e33a6 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -165,6 +165,9 @@ func InitialSettings() []model.SettingItem { {Key: conf.ForwardDirectLinkParams, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL}, {Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, + {Key: conf.MaxDevices, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL}, + {Key: conf.DeviceEvictPolicy, Value: "deny", Type: conf.TypeSelect, Options: "deny,evict_oldest", Group: model.GLOBAL}, + {Key: conf.DeviceSessionTTL, Value: "86400", Type: conf.TypeNumber, Group: model.GLOBAL}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 0bf0cd67f7e..1a5581633a0 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -48,6 +48,9 @@ const ( ForwardDirectLinkParams = "forward_direct_link_params" IgnoreDirectLinkParams = "ignore_direct_link_params" WebauthnLoginEnabled = "webauthn_login_enabled" + MaxDevices = "max_devices" + DeviceEvictPolicy = "device_evict_policy" + DeviceSessionTTL = "device_session_ttl" // index SearchIndex = "search_index" diff --git a/internal/db/db.go b/internal/db/db.go index 0d8ab42130a..4577059d4f8 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile), new(model.Session)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/session.go b/internal/db/session.go new file mode 100644 index 00000000000..8db9fa69f8b --- /dev/null +++ b/internal/db/session.go @@ -0,0 +1,65 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm/clause" +) + +func GetSession(userID uint, deviceKey string) (*model.Session, error) { + s := model.Session{UserID: userID, DeviceKey: deviceKey} + if err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where(&s).First(&s).Error; err != nil { + return nil, errors.Wrap(err, "failed find session") + } + return &s, nil +} + +func CreateSession(s *model.Session) error { + return errors.WithStack(db.Create(s).Error) +} + +func UpsertSession(s *model.Session) error { + return errors.WithStack(db.Clauses(clause.OnConflict{UpdateAll: true}).Create(s).Error) +} + +func DeleteSession(userID uint, deviceKey string) error { + return errors.WithStack(db.Where("user_id = ? AND device_key = ?", userID, deviceKey).Delete(&model.Session{}).Error) +} + +func CountSessionsByUser(userID uint) (int64, error) { + var count int64 + err := db.Model(&model.Session{}).Where("user_id = ?", userID).Count(&count).Error + return count, errors.WithStack(err) +} + +func DeleteSessionsBefore(ts int64) error { + return errors.WithStack(db.Where("last_active < ?", ts).Delete(&model.Session{}).Error) +} + +func GetOldestSession(userID uint) (*model.Session, error) { + var s model.Session + if err := db.Where("user_id = ?", userID).Order("last_active ASC").First(&s).Error; err != nil { + return nil, errors.Wrap(err, "failed get oldest session") + } + return &s, nil +} + +func UpdateSessionLastActive(userID uint, deviceKey string, lastActive int64) error { + return errors.WithStack(db.Model(&model.Session{}).Where("user_id = ? AND device_key = ?", userID, deviceKey).Update("last_active", lastActive).Error) +} + +func ListSessionsByUser(userID uint) ([]model.Session, error) { + var sessions []model.Session + err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where("user_id = ? AND status = ?", userID, model.SessionActive).Find(&sessions).Error + return sessions, errors.WithStack(err) +} + +func ListSessions() ([]model.Session, error) { + var sessions []model.Session + err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where("status = ?", model.SessionActive).Find(&sessions).Error + return sessions, errors.WithStack(err) +} + +func MarkInactive(sessionID string) error { + return errors.WithStack(db.Model(&model.Session{}).Where("device_key = ?", sessionID).Update("status", model.SessionInactive).Error) +} diff --git a/internal/device/session.go b/internal/device/session.go new file mode 100644 index 00000000000..d407c858dfa --- /dev/null +++ b/internal/device/session.go @@ -0,0 +1,67 @@ +package device + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// Handle verifies device sessions for a user and upserts current session. +func Handle(userID uint, deviceKey, ua, ip string) error { + ttl := setting.GetInt(conf.DeviceSessionTTL, 86400) + if ttl > 0 { + _ = db.DeleteSessionsBefore(time.Now().Unix() - int64(ttl)) + } + + ip = utils.MaskIP(ip) + + now := time.Now().Unix() + sess, err := db.GetSession(userID, deviceKey) + if err == nil { + if sess.Status == model.SessionInactive { + return errors.WithStack(errs.SessionInactive) + } + sess.LastActive = now + sess.Status = model.SessionActive + sess.UserAgent = ua + sess.IP = ip + return db.UpsertSession(sess) + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, err := db.CountSessionsByUser(userID) + if err != nil { + return err + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + oldest, err := db.GetOldestSession(userID) + if err == nil { + _ = db.DeleteSession(userID, oldest.DeviceKey) + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + + s := &model.Session{UserID: userID, DeviceKey: deviceKey, UserAgent: ua, IP: ip, LastActive: now, Status: model.SessionActive} + return db.CreateSession(s) +} + +// Refresh updates last_active for the session. +func Refresh(userID uint, deviceKey string) { + _ = db.UpdateSessionLastActive(userID, deviceKey, time.Now().Unix()) +} diff --git a/internal/errs/device.go b/internal/errs/device.go new file mode 100644 index 00000000000..3b79298a672 --- /dev/null +++ b/internal/errs/device.go @@ -0,0 +1,8 @@ +package errs + +import "errors" + +var ( + TooManyDevices = errors.New("too many active devices") + SessionInactive = errors.New("session inactive") +) diff --git a/internal/model/session.go b/internal/model/session.go new file mode 100644 index 00000000000..3cb6d0dab90 --- /dev/null +++ b/internal/model/session.go @@ -0,0 +1,16 @@ +package model + +// Session represents a device session of a user. +type Session struct { + UserID uint `json:"user_id" gorm:"index"` + DeviceKey string `json:"device_key" gorm:"primaryKey;size:64"` + UserAgent string `json:"user_agent" gorm:"size:255"` + IP string `json:"ip" gorm:"size:64"` + LastActive int64 `json:"last_active"` + Status int `json:"status"` +} + +const ( + SessionActive = iota + SessionInactive +) diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 00000000000..47d1b70125c --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,8 @@ +package session + +import "github.com/alist-org/alist/v3/internal/db" + +// MarkInactive marks the session with the given ID as inactive. +func MarkInactive(sessionID string) error { + return db.MarkInactive(sessionID) +} diff --git a/pkg/utils/mask.go b/pkg/utils/mask.go new file mode 100644 index 00000000000..1513ad40368 --- /dev/null +++ b/pkg/utils/mask.go @@ -0,0 +1,30 @@ +package utils + +import "strings" + +// MaskIP anonymizes middle segments of an IP address. +func MaskIP(ip string) string { + if ip == "" { + return "" + } + if strings.Contains(ip, ":") { + parts := strings.Split(ip, ":") + if len(parts) > 2 { + for i := 1; i < len(parts)-1; i++ { + if parts[i] != "" { + parts[i] = "*" + } + } + return strings.Join(parts, ":") + } + return ip + } + parts := strings.Split(ip, ".") + if len(parts) == 4 { + for i := 1; i < len(parts)-1; i++ { + parts[i] = "*" + } + return strings.Join(parts, ".") + } + return ip +} diff --git a/server/handles/auth.go b/server/handles/auth.go index 26447ddd594..30714f6544c 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -12,6 +12,7 @@ import ( "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/session" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" @@ -247,6 +248,13 @@ func Verify2FA(c *gin.Context) { } func LogOut(c *gin.Context) { + if keyVal, ok := c.Get("device_key"); ok { + if err := session.MarkInactive(keyVal.(string)); err != nil { + common.ErrorResp(c, err, 500) + return + } + c.Set("session_inactive", true) + } err := common.InvalidateToken(c.GetHeader("Authorization")) if err != nil { common.ErrorResp(c, err, 500) diff --git a/server/handles/session.go b/server/handles/session.go new file mode 100644 index 00000000000..886be66ab01 --- /dev/null +++ b/server/handles/session.go @@ -0,0 +1,92 @@ +package handles + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +type SessionResp struct { + SessionID string `json:"session_id"` + UserID uint `json:"user_id,omitempty"` + LastActive int64 `json:"last_active"` + Status int `json:"status"` + UA string `json:"ua"` + IP string `json:"ip"` +} + +func ListMySessions(c *gin.Context) { + user := c.MustGet("user").(*model.User) + sessions, err := db.ListSessionsByUser(user.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + resp := make([]SessionResp, len(sessions)) + for i, s := range sessions { + resp[i] = SessionResp{ + SessionID: s.DeviceKey, + LastActive: s.LastActive, + Status: s.Status, + UA: s.UserAgent, + IP: s.IP, + } + } + common.SuccessResp(c, resp) +} + +type EvictSessionReq struct { + SessionID string `json:"session_id"` +} + +func EvictMySession(c *gin.Context) { + var req EvictSessionReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if _, err := db.GetSession(user.ID, req.SessionID); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := db.MarkInactive(req.SessionID); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +func ListSessions(c *gin.Context) { + sessions, err := db.ListSessions() + if err != nil { + common.ErrorResp(c, err, 500) + return + } + resp := make([]SessionResp, len(sessions)) + for i, s := range sessions { + resp[i] = SessionResp{ + SessionID: s.DeviceKey, + UserID: s.UserID, + LastActive: s.LastActive, + Status: s.Status, + UA: s.UserAgent, + IP: s.IP, + } + } + common.SuccessResp(c, resp) +} + +func EvictSession(c *gin.Context) { + var req EvictSessionReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := db.MarkInactive(req.SessionID); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index c0743c9ce9d..72eaefe6096 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -5,9 +5,11 @@ import ( "fmt" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -24,7 +26,9 @@ func Auth(c *gin.Context) { c.Abort() return } - c.Set("user", admin) + if !handleSession(c, admin) { + return + } log.Debugf("use admin token: %+v", admin) c.Next() return @@ -50,7 +54,9 @@ func Auth(c *gin.Context) { } guest.RolesDetail = roles } - c.Set("user", guest) + if !handleSession(c, guest) { + return + } log.Debugf("use empty token: %+v", guest) c.Next() return @@ -87,11 +93,29 @@ func Auth(c *gin.Context) { } user.RolesDetail = roles } - c.Set("user", user) + if !handleSession(c, user) { + return + } log.Debugf("use login token: %+v", user) c.Next() } +func handleSession(c *gin.Context, user *model.User) bool { + clientID := c.GetHeader("Client-Id") + if clientID == "" { + clientID = c.Query("client_id") + } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s-%s-%s", user.ID, c.Request.UserAgent(), c.ClientIP(), clientID)) + if err := device.Handle(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + common.ErrorResp(c, err, 403) + c.Abort() + return false + } + c.Set("device_key", key) + c.Set("user", user) + return true +} + func Authn(c *gin.Context) { token := c.GetHeader("Authorization") if subtle.ConstantTimeCompare([]byte(token), []byte(setting.GetStr(conf.Token))) == 1 { diff --git a/server/middlewares/session_refresh.go b/server/middlewares/session_refresh.go new file mode 100644 index 00000000000..2073020ce79 --- /dev/null +++ b/server/middlewares/session_refresh.go @@ -0,0 +1,26 @@ +package middlewares + +import ( + "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/model" + "github.com/gin-gonic/gin" +) + +// SessionRefresh updates session's last_active after successful requests. +func SessionRefresh(c *gin.Context) { + c.Next() + if c.Writer.Status() >= 400 { + return + } + if inactive, ok := c.Get("session_inactive"); ok { + if b, ok := inactive.(bool); ok && b { + return + } + } + userVal, uok := c.Get("user") + keyVal, kok := c.Get("device_key") + if uok && kok { + user := userVal.(*model.User) + device.Refresh(user.ID, keyVal.(string)) + } +} diff --git a/server/router.go b/server/router.go index e8902f7a648..4d79c1fde52 100644 --- a/server/router.go +++ b/server/router.go @@ -22,6 +22,7 @@ func Init(e *gin.Engine) { }) } Cors(e) + e.Use(middlewares.SessionRefresh) g := e.Group(conf.URL.Path) if conf.Conf.Scheme.HttpPort != -1 && conf.Conf.Scheme.HttpsPort != -1 && conf.Conf.Scheme.ForceHttps { e.Use(middlewares.ForceHttps) @@ -70,6 +71,8 @@ func Init(e *gin.Engine) { auth.POST("/auth/2fa/generate", handles.Generate2FA) auth.POST("/auth/2fa/verify", handles.Verify2FA) auth.GET("/auth/logout", handles.LogOut) + auth.GET("/me/sessions", handles.ListMySessions) + auth.POST("/me/sessions/evict", handles.EvictMySession) // auth api.GET("/auth/sso", handles.SSOLoginRedirect) @@ -184,6 +187,10 @@ func admin(g *gin.RouterGroup) { labelFileBinding.POST("/delete", handles.DelLabelByFileName) labelFileBinding.POST("/restore", handles.RestoreLabelFileBinding) + session := g.Group("/session") + session.GET("/list", handles.ListSessions) + session.POST("/evict", handles.EvictSession) + } func _fs(g *gin.RouterGroup) { diff --git a/server/webdav.go b/server/webdav.go index 582c469d73a..e0980139e4f 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -3,6 +3,7 @@ package server import ( "context" "crypto/subtle" + "fmt" "net/http" "net/url" "path" @@ -12,9 +13,11 @@ import ( "github.com/alist-org/alist/v3/server/middlewares" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/webdav" "github.com/gin-gonic/gin" @@ -69,6 +72,13 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", admin.ID, c.ClientIP())) + if err := device.Handle(admin.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + c.Status(http.StatusForbidden) + c.Abort() + return + } + c.Set("device_key", key) c.Set("user", admin) c.Next() return @@ -146,6 +156,13 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", user.ID, c.ClientIP())) + if err := device.Handle(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + c.Status(http.StatusForbidden) + c.Abort() + return + } + c.Set("device_key", key) c.Set("user", user) c.Next() } From de09ba08b6429edd58f3ed0f602e644966fc6ee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Wed, 27 Aug 2025 17:46:34 +0800 Subject: [PATCH 033/133] chore(deps): Update 115driver dependency to v1.1.2 (#9294) - Upgrade `github.com/SheltonZhu/115driver` to v1.1.2 in `go.mod` - Modify `replace` to point to `github.com/okatu-loli/115driver v1.1.2` - Remove old version checksum from `go.sum` and add new version checksum --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 9923c489eef..51f9beec443 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 - github.com/SheltonZhu/115driver v1.1.1 + github.com/SheltonZhu/115driver v1.1.2 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 @@ -265,4 +265,4 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect ) -// replace github.com/xhofe/115-sdk-go => ../../xhofe/115-sdk-go +replace github.com/SheltonZhu/115driver => github.com/okatu-loli/115driver v1.1.2 diff --git a/go.sum b/go.sum index e47d96234ae..1b088a7186e 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,6 @@ github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4 github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.1.1 h1:9EMhe2ZJflGiAaZbYInw2jqxTcqZNF+DtVDsEy70aFU= -github.com/SheltonZhu/115driver v1.1.1/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= @@ -490,6 +488,8 @@ github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/okatu-loli/115driver v1.1.2 h1:XZT3r/51SZRQGzre2IeA+0/k4T1FneqArdhE4Wd600Q= +github.com/okatu-loli/115driver v1.1.2/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= From 3bf0af1e6826ba810df24f05db73cede74f121d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 28 Aug 2025 09:57:13 +0800 Subject: [PATCH 034/133] fix(session): Fixed the session status update logic. (#9296) - Removed the error returned when the session status is `SessionInactive`. - Updated the `LastActive` field of the session to always record the current time. --- internal/device/session.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/device/session.go b/internal/device/session.go index d407c858dfa..49bf74b6e9a 100644 --- a/internal/device/session.go +++ b/internal/device/session.go @@ -25,11 +25,9 @@ func Handle(userID uint, deviceKey, ua, ip string) error { now := time.Now().Unix() sess, err := db.GetSession(userID, deviceKey) if err == nil { - if sess.Status == model.SessionInactive { - return errors.WithStack(errs.SessionInactive) - } - sess.LastActive = now + // reactivate existing session if it was inactive sess.Status = model.SessionActive + sess.LastActive = now sess.UserAgent = ua sess.IP = ip return db.UpsertSession(sess) From 84adba3acc2141dbe61663e4ec199ffc4333f76b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 28 Aug 2025 09:57:34 +0800 Subject: [PATCH 035/133] feat(user): Enhanced role assignment logic (#9297) - Imported the `utils` package - Modified the role assignment logic to prevent assigning administrator or guest roles to users --- server/handles/user.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/handles/user.go b/server/handles/user.go index 01368beec6e..ac3a06e8180 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -1,9 +1,10 @@ package handles import ( - "github.com/alist-org/alist/v3/pkg/utils" "strconv" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/server/common" @@ -97,6 +98,14 @@ func UpdateUser(c *gin.Context) { return } } + + if !utils.SliceEqual(user.Role, req.Role) { + if req.IsAdmin() || req.IsGuest() { + common.ErrorStrResp(c, "cannot assign admin or guest role to user", 400, true) + return + } + } + if err := op.UpdateUser(&req); err != nil { common.ErrorResp(c, err, 500) } else { From 8623da5361e487e3310374a88339d5d98ff8ac77 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 29 Aug 2025 11:53:55 +0800 Subject: [PATCH 036/133] feat(session): Added user session limit and device eviction logic - Renamed `CountSessionsByUser` to `CountActiveSessionsByUser` and added session status filtering - Added user and device session limit, with policy handling when exceeding the limit - Introduced device eviction policy: If the maximum number of devices is exceeded, the oldest session will be evicted using the "evict_oldest" policy - Modified `LastActive` update logic to ensure accurate session activity time --- internal/db/session.go | 6 ++++-- internal/device/session.go | 22 ++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/internal/db/session.go b/internal/db/session.go index 8db9fa69f8b..e8dce441175 100644 --- a/internal/db/session.go +++ b/internal/db/session.go @@ -26,9 +26,11 @@ func DeleteSession(userID uint, deviceKey string) error { return errors.WithStack(db.Where("user_id = ? AND device_key = ?", userID, deviceKey).Delete(&model.Session{}).Error) } -func CountSessionsByUser(userID uint) (int64, error) { +func CountActiveSessionsByUser(userID uint) (int64, error) { var count int64 - err := db.Model(&model.Session{}).Where("user_id = ?", userID).Count(&count).Error + err := db.Model(&model.Session{}). + Where("user_id = ? AND status = ?", userID, model.SessionActive). + Count(&count).Error return count, errors.WithStack(err) } diff --git a/internal/device/session.go b/internal/device/session.go index 49bf74b6e9a..5d5a39969c1 100644 --- a/internal/device/session.go +++ b/internal/device/session.go @@ -25,7 +25,25 @@ func Handle(userID uint, deviceKey, ua, ip string) error { now := time.Now().Unix() sess, err := db.GetSession(userID, deviceKey) if err == nil { - // reactivate existing session if it was inactive + if sess.Status == model.SessionInactive { + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, cerr := db.CountActiveSessionsByUser(userID) + if cerr != nil { + return cerr + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + if oldest, gerr := db.GetOldestSession(userID); gerr == nil { + _ = db.DeleteSession(userID, oldest.DeviceKey) + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + } sess.Status = model.SessionActive sess.LastActive = now sess.UserAgent = ua @@ -38,7 +56,7 @@ func Handle(userID uint, deviceKey, ua, ip string) error { max := setting.GetInt(conf.MaxDevices, 0) if max > 0 { - count, err := db.CountSessionsByUser(userID) + count, err := db.CountActiveSessionsByUser(userID) if err != nil { return err } From 9a7c82a71e3ca00f8cdd61f9b854e87464982ba1 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 29 Aug 2025 13:31:44 +0800 Subject: [PATCH 037/133] feat(auth): Optimized device session handling logic - Introduced middleware to handle device sessions - Changed `handleSession` to `HandleSession` in multiple places in `auth.go` to maintain consistent naming - Updated response structure to return `device_key` and `token` --- server/handles/auth.go | 8 +++++++- server/middlewares/auth.go | 9 +++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/server/handles/auth.go b/server/handles/auth.go index 30714f6544c..8c7d7d9f56f 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -15,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/internal/session" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/server/common" + "github.com/alist-org/alist/v3/server/middlewares" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" ) @@ -82,13 +83,18 @@ func loginHash(c *gin.Context, req *LoginReq) { return } } + // generate device session + if !middlewares.HandleSession(c, user) { + return + } // generate token token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return } - common.SuccessResp(c, gin.H{"token": token}) + key := c.GetString("device_key") + common.SuccessResp(c, gin.H{"token": token, "device_key": key}) loginCache.Del(ip) } diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 72eaefe6096..714c11548c5 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -26,7 +26,7 @@ func Auth(c *gin.Context) { c.Abort() return } - if !handleSession(c, admin) { + if !HandleSession(c, admin) { return } log.Debugf("use admin token: %+v", admin) @@ -54,7 +54,7 @@ func Auth(c *gin.Context) { } guest.RolesDetail = roles } - if !handleSession(c, guest) { + if !HandleSession(c, guest) { return } log.Debugf("use empty token: %+v", guest) @@ -93,14 +93,15 @@ func Auth(c *gin.Context) { } user.RolesDetail = roles } - if !handleSession(c, user) { + if !HandleSession(c, user) { return } log.Debugf("use login token: %+v", user) c.Next() } -func handleSession(c *gin.Context, user *model.User) bool { +// HandleSession verifies device sessions and stores context values. +func HandleSession(c *gin.Context, user *model.User) bool { clientID := c.GetHeader("Client-Id") if clientID == "" { clientID = c.Query("client_id") From 63391a20914b33604ced8f23412feb50e6a51796 Mon Sep 17 00:00:00 2001 From: Sky_slience <95515853+skysliences@users.noreply.github.com> Date: Fri, 29 Aug 2025 14:56:54 +0800 Subject: [PATCH 038/133] fix(readme): remove outdated sponsor links from README files (#9300) Co-authored-by: Sky_slience --- README.md | 2 -- README_cn.md | 2 -- README_ja.md | 2 -- 3 files changed, 6 deletions(-) diff --git a/README.md b/README.md index 5a93997fe40..d352b5b3f50 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,6 @@ https://alistgo.com/guide/sponsor.html ### Special sponsors - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. -- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) -- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## Contributors diff --git a/README_cn.md b/README_cn.md index 79ed864bc84..27f417d1698 100644 --- a/README_cn.md +++ b/README_cn.md @@ -118,8 +118,6 @@ AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我 ### 特别赞助 - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - 苹果生态下优雅的网盘视频播放器，iPhone，iPad，Mac，Apple TV全平台支持。 -- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (国内API服务器赞助) -- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## 贡献者 diff --git a/README_ja.md b/README_ja.md index 9291b2acdc2..cd59086078c 100644 --- a/README_ja.md +++ b/README_ja.md @@ -120,8 +120,6 @@ https://alistgo.com/guide/sponsor.html ### スペシャルスポンサー - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. -- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) -- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## コントリビューター From 4b288a08ef264d4be4b36c638992b1d9f5484526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 29 Aug 2025 21:20:29 +0800 Subject: [PATCH 039/133] fix: session invalid issue (#9301) * feat(auth): Enhanced device login session management - Upon login, obtain and verify `Client-Id` to ensure unique device sessions. - If there are too many device sessions, clean up old ones according to the configured policy or return an error. - If a device session is invalid, deregister the old token and return a 401 error. - Added `EnsureActiveOnLogin` function to handle the creation and refresh of device sessions during login. * feat(session): Modified session deletion logic to mark sessions as inactive. - Changed session deletion logic to mark sessions as inactive using the `MarkInactive` method. - Adjusted error handling to ensure an error is returned if marking fails. * feat(session): Added device limits and eviction policies - Added a device limit, controlling the maximum number of devices using the `MaxDevices` configuration option. - If the number of devices exceeds the limit, the configured eviction policy is used. - If the policy is `evict_oldest`, the oldest device is evicted. - Otherwise, an error message indicating too many devices is returned. * refactor(session): Filter for the user's oldest active session - Renamed `GetOldestSession` to `GetOldestActiveSession` to more accurately reflect its functionality - Updated the SQL query to add the `status = SessionActive` condition to retrieve only active sessions - Replaced all callpoints and unified the new function name to ensure logical consistency --- internal/db/session.go | 8 ++-- internal/device/session.go | 75 +++++++++++++++++++++++++++++++++----- server/handles/auth.go | 24 ++++++++++-- server/middlewares/auth.go | 12 +++++- 4 files changed, 100 insertions(+), 19 deletions(-) diff --git a/internal/db/session.go b/internal/db/session.go index e8dce441175..35c778c3ac8 100644 --- a/internal/db/session.go +++ b/internal/db/session.go @@ -38,10 +38,12 @@ func DeleteSessionsBefore(ts int64) error { return errors.WithStack(db.Where("last_active < ?", ts).Delete(&model.Session{}).Error) } -func GetOldestSession(userID uint) (*model.Session, error) { +// GetOldestActiveSession returns the oldest active session for the specified user. +func GetOldestActiveSession(userID uint) (*model.Session, error) { var s model.Session - if err := db.Where("user_id = ?", userID).Order("last_active ASC").First(&s).Error; err != nil { - return nil, errors.Wrap(err, "failed get oldest session") + if err := db.Where("user_id = ? AND status = ?", userID, model.SessionActive). + Order("last_active ASC").First(&s).Error; err != nil { + return nil, errors.Wrap(err, "failed get oldest active session") } return &s, nil } diff --git a/internal/device/session.go b/internal/device/session.go index 5d5a39969c1..1d9e7ea53cd 100644 --- a/internal/device/session.go +++ b/internal/device/session.go @@ -23,20 +23,68 @@ func Handle(userID uint, deviceKey, ua, ip string) error { ip = utils.MaskIP(ip) now := time.Now().Unix() + sess, err := db.GetSession(userID, deviceKey) + if err == nil { + if sess.Status == model.SessionInactive { + return errors.WithStack(errs.SessionInactive) + } + sess.Status = model.SessionActive + sess.LastActive = now + sess.UserAgent = ua + sess.IP = ip + return db.UpsertSession(sess) + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, err := db.CountActiveSessionsByUser(userID) + if err != nil { + return err + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + if oldest, err := db.GetOldestActiveSession(userID); err == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + + s := &model.Session{UserID: userID, DeviceKey: deviceKey, UserAgent: ua, IP: ip, LastActive: now, Status: model.SessionActive} + return db.CreateSession(s) +} + +// EnsureActiveOnLogin is used only in login flow: +// - If session exists (even Inactive): reactivate and refresh fields. +// - If not exists: apply max-devices policy, then create Active session. +func EnsureActiveOnLogin(userID uint, deviceKey, ua, ip string) error { + ip = utils.MaskIP(ip) + now := time.Now().Unix() + sess, err := db.GetSession(userID, deviceKey) if err == nil { if sess.Status == model.SessionInactive { max := setting.GetInt(conf.MaxDevices, 0) if max > 0 { - count, cerr := db.CountActiveSessionsByUser(userID) - if cerr != nil { - return cerr + count, err := db.CountActiveSessionsByUser(userID) + if err != nil { + return err } if count >= int64(max) { policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") if policy == "evict_oldest" { - if oldest, gerr := db.GetOldestSession(userID); gerr == nil { - _ = db.DeleteSession(userID, oldest.DeviceKey) + if oldest, gerr := db.GetOldestActiveSession(userID); gerr == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } } } else { return errors.WithStack(errs.TooManyDevices) @@ -63,9 +111,10 @@ func Handle(userID uint, deviceKey, ua, ip string) error { if count >= int64(max) { policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") if policy == "evict_oldest" { - oldest, err := db.GetOldestSession(userID) - if err == nil { - _ = db.DeleteSession(userID, oldest.DeviceKey) + if oldest, gerr := db.GetOldestActiveSession(userID); gerr == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } } } else { return errors.WithStack(errs.TooManyDevices) @@ -73,8 +122,14 @@ func Handle(userID uint, deviceKey, ua, ip string) error { } } - s := &model.Session{UserID: userID, DeviceKey: deviceKey, UserAgent: ua, IP: ip, LastActive: now, Status: model.SessionActive} - return db.CreateSession(s) + return db.CreateSession(&model.Session{ + UserID: userID, + DeviceKey: deviceKey, + UserAgent: ua, + IP: ip, + LastActive: now, + Status: model.SessionActive, + }) } // Refresh updates last_active for the session. diff --git a/server/handles/auth.go b/server/handles/auth.go index 8c7d7d9f56f..dd7d202b907 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -3,6 +3,8 @@ package handles import ( "bytes" "encoding/base64" + "errors" + "fmt" "image/png" "path" "strings" @@ -10,12 +12,14 @@ import ( "github.com/Xhofe/go-cache" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/session" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" - "github.com/alist-org/alist/v3/server/middlewares" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" ) @@ -83,17 +87,29 @@ func loginHash(c *gin.Context, req *LoginReq) { return } } - // generate device session - if !middlewares.HandleSession(c, user) { + + clientID := c.GetHeader("Client-Id") + if clientID == "" { + clientID = c.Query("client_id") + } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", + user.ID, clientID)) + + if err := device.EnsureActiveOnLogin(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + if errors.Is(err, errs.TooManyDevices) { + common.ErrorResp(c, err, 403) + } else { + common.ErrorResp(c, err, 400, true) + } return } + // generate token token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return } - key := c.GetString("device_key") common.SuccessResp(c, gin.H{"token": token, "device_key": key}) loginCache.Del(ip) } diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 714c11548c5..204b4b7205e 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -2,10 +2,12 @@ package middlewares import ( "crypto/subtle" + "errors" "fmt" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" @@ -106,9 +108,15 @@ func HandleSession(c *gin.Context, user *model.User) bool { if clientID == "" { clientID = c.Query("client_id") } - key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s-%s-%s", user.ID, c.Request.UserAgent(), c.ClientIP(), clientID)) + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", user.ID, clientID)) if err := device.Handle(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { - common.ErrorResp(c, err, 403) + token := c.GetHeader("Authorization") + if errors.Is(err, errs.SessionInactive) { + _ = common.InvalidateToken(token) + common.ErrorResp(c, err, 401) + } else { + common.ErrorResp(c, err, 403) + } c.Abort() return false } From 23107483a126a53419cbf18f8fbdecf937f0d0a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 4 Sep 2025 22:14:33 +0800 Subject: [PATCH 040/133] Refactor (storage): Comment out the path validation logic (#9308) - Comment out the error return logic for paths with "/" - Remove storage path restrictions to allow for flexible handling of root paths --- internal/op/storage.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/op/storage.go b/internal/op/storage.go index 5ab1da1840a..27221e70e8c 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -47,9 +47,9 @@ func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) - if storage.MountPath == "/" { - return 0, errors.New("Mount path cannot be '/'") - } + //if storage.MountPath == "/" { + // return 0, errors.New("Mount path cannot be '/'") + //} var err error // check driver first @@ -210,9 +210,9 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { } storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) - if storage.MountPath == "/" { - return errors.New("Mount path cannot be '/'") - } + //if storage.MountPath == "/" { + // return errors.New("Mount path cannot be '/'") + //} err = db.UpdateStorage(&storage) if err != nil { return errors.WithMessage(err, "failed update storage in database") From 930f9f6096a17a940573b0378add12f3b08e6aa1 Mon Sep 17 00:00:00 2001 From: Sakkyoi Cheng <22865542+sakkyoi@users.noreply.github.com> Date: Thu, 4 Sep 2025 22:15:39 +0800 Subject: [PATCH 041/133] fix(ssologin): missing role in SSO auto-registration and minor callback issue (#9305) * fix(ssologin): return after error response * fix(ssologin): set default role for SSO user creation --- server/handles/ssologin.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index eb6599e7a4d..779cc13239b 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "errors" "fmt" + "github.com/alist-org/alist/v3/internal/op" "net/http" "net/url" "path" @@ -154,7 +155,7 @@ func autoRegister(username, userID string, err error) (*model.User, error) { Password: random.String(16), Permission: int32(setting.GetInt(conf.SSODefaultPermission, 0)), BasePath: setting.GetStr(conf.SSODefaultDir), - Role: nil, + Role: model.Roles{op.GetDefaultRoleID()}, Disabled: false, SsoID: userID, } @@ -256,6 +257,7 @@ func OIDCLoginCallback(c *gin.Context) { user, err = autoRegister(userID, userID, err) if err != nil { common.ErrorResp(c, err, 400) + return } } token, err := common.GenerateToken(user) From fcbc79cb24971d727c8a0677c7af38c545fc7145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 5 Sep 2025 19:58:27 +0800 Subject: [PATCH 042/133] feat: Support 123pan safebox (#9311) * feat(meta): Added a SafePassword field - Added the SafePassword field to meta.go - Revised the field format to align with the code style - The SafePassword field is used to supplement the extended functionality * feat(driver): Added support for safe unlocking logic - Added safe file unlocking logic in `driver.go`, returning an error if unlocking fails. - Introduced the `safeBoxUnlocked` variable of type `sync.Map` to record the IDs of unlocked files. - Enhanced error handling logic to automatically attempt to unlock safe files and re-retrieve the file list. - Added the `IsLock` field to file types in `types.go` to identify whether they are safe files. - Added a constant definition for the `SafeBoxUnlock` interface address in `util.go`. - Added the `unlockSafeBox` method to unlock a safe with a specified file ID via the API. - Optimized the file retrieval logic to automatically call the unlock method when the safe is locked. * Refactor (driver): Optimize lock field type - Changed the `IsLock` field type from `int` to `bool` for better semantics. - Updated the check logic to use direct Boolean comparisons to improve code readability and accuracy. --- drivers/123/driver.go | 24 ++++++++++++++++++++++-- drivers/123/meta.go | 5 +++-- drivers/123/types.go | 1 + drivers/123/util.go | 26 ++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 32c053e22ab..cf221fee6d8 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -6,6 +6,8 @@ import ( "fmt" "net/http" "net/url" + "strconv" + "strings" "sync" "time" @@ -28,7 +30,8 @@ import ( type Pan123 struct { model.Storage Addition - apiRateLimit sync.Map + apiRateLimit sync.Map + safeBoxUnlocked sync.Map } func (d *Pan123) Config() driver.Config { @@ -52,9 +55,26 @@ func (d *Pan123) Drop(ctx context.Context) error { } func (d *Pan123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if f, ok := dir.(File); ok && f.IsLock { + if err := d.unlockSafeBox(f.FileId); err != nil { + return nil, err + } + } files, err := d.getFiles(ctx, dir.GetID(), dir.GetName()) if err != nil { - return nil, err + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "safe box") || strings.Contains(err.Error(), "保险箱") { + if id, e := strconv.ParseInt(dir.GetID(), 10, 64); e == nil { + if e = d.unlockSafeBox(id); e == nil { + files, err = d.getFiles(ctx, dir.GetID(), dir.GetName()) + } else { + return nil, e + } + } + } + if err != nil { + return nil, err + } } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return src, nil diff --git a/drivers/123/meta.go b/drivers/123/meta.go index cb2cbc15ba0..6c5f013ad4a 100644 --- a/drivers/123/meta.go +++ b/drivers/123/meta.go @@ -6,8 +6,9 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + SafePassword string `json:"safe_password"` driver.RootID //OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"` //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` diff --git a/drivers/123/types.go b/drivers/123/types.go index a8682c52fc9..962e5fbdac0 100644 --- a/drivers/123/types.go +++ b/drivers/123/types.go @@ -20,6 +20,7 @@ type File struct { Etag string `json:"Etag"` S3KeyFlag string `json:"S3KeyFlag"` DownloadUrl string `json:"DownloadUrl"` + IsLock bool `json:"IsLock"` } func (f File) CreateTime() time.Time { diff --git a/drivers/123/util.go b/drivers/123/util.go index b85c9afbac8..bca54b599f4 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -43,6 +43,7 @@ const ( S3Auth = MainApi + "/file/s3_upload_object/auth" UploadCompleteV2 = MainApi + "/file/upload_complete/v2" S3Complete = MainApi + "/file/s3_complete_multipart_upload" + SafeBoxUnlock = MainApi + "/restful/goapi/v1/file/safe_box/auth/unlockbox" //AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) @@ -238,6 +239,22 @@ do: return body, nil } +func (d *Pan123) unlockSafeBox(fileId int64) error { + if _, ok := d.safeBoxUnlocked.Load(fileId); ok { + return nil + } + data := base.Json{"password": d.SafePassword} + url := fmt.Sprintf("%s?fileId=%d", SafeBoxUnlock, fileId) + _, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + if err != nil { + return err + } + d.safeBoxUnlocked.Store(fileId, true) + return nil +} + func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([]File, error) { page := 1 total := 0 @@ -267,6 +284,15 @@ func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([] req.SetQueryParams(query) }, &resp) if err != nil { + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "safe box") || strings.Contains(err.Error(), "保险箱") { + if fid, e := strconv.ParseInt(parentId, 10, 64); e == nil { + if e = d.unlockSafeBox(fid); e == nil { + return d.getFiles(ctx, parentId, name) + } + return nil, e + } + } return nil, err } log.Debug(string(_res)) From d0026030cb36782b92feed55903e85ca14ee6ad0 Mon Sep 17 00:00:00 2001 From: "D@' 3z K!7" <99719341+Da3zKi7@users.noreply.github.com> Date: Wed, 10 Sep 2025 21:46:09 -0600 Subject: [PATCH 043/133] feat(drivers): add MediaFire driver support (#9319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Implement complete MediaFire storage driver - Add authentication via session_token and cookie - Support all core operations: List, Get, Link, Put, Copy, Move, Remove, Rename, MakeDir - Include thumbnail generation for media files - Handle MediaFire's resumable upload API with multi-unit transfers - Add proper error handling and progress reporting Closes 请求支持Mediafire #7869 Co-authored-by: Da3zKi7 --- README.md | 1 + README_cn.md | 1 + README_ja.md | 1 + drivers/all.go | 1 + drivers/mediafire/driver.go | 427 +++++++++++++++++++++++++ drivers/mediafire/meta.go | 54 ++++ drivers/mediafire/types.go | 232 ++++++++++++++ drivers/mediafire/util.go | 620 ++++++++++++++++++++++++++++++++++++ 8 files changed, 1337 insertions(+) create mode 100644 drivers/mediafire/driver.go create mode 100644 drivers/mediafire/meta.go create mode 100644 drivers/mediafire/types.go create mode 100644 drivers/mediafire/util.go diff --git a/README.md b/README.md index d352b5b3f50..b77ed804751 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] WebDav(Support OneDrive/SharePoint without API) - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) + - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) diff --git a/README_cn.md b/README_cn.md index 27f417d1698..757f5f8fb7a 100644 --- a/README_cn.md +++ b/README_cn.md @@ -57,6 +57,7 @@ - [x] [又拍云对象存储](https://www.upyun.com/products/file-storage) - [x] WebDav(支持无API的OneDrive/SharePoint) - [x] Teambition（[中国](https://www.teambition.com/ )，[国际](https://us.teambition.com/ )） + - [x] [MediaFire](https://www.mediafire.com) - [x] [分秒帧](https://www.mediatrack.cn/) - [x] [和彩云](https://yun.139.com/) (个人云, 家庭云，共享群组) - [x] [Yandex.Disk](https://disk.yandex.com/) diff --git a/README_ja.md b/README_ja.md index cd59086078c..e6a624b0929 100644 --- a/README_ja.md +++ b/README_ja.md @@ -57,6 +57,7 @@ - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] WebDav(Support OneDrive/SharePoint without API) - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) + - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) diff --git a/drivers/all.go b/drivers/all.go index a8c8620989e..5c3cc570805 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -41,6 +41,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/lanzou" _ "github.com/alist-org/alist/v3/drivers/lenovonas_share" _ "github.com/alist-org/alist/v3/drivers/local" + _ "github.com/alist-org/alist/v3/drivers/mediafire" _ "github.com/alist-org/alist/v3/drivers/mediatrack" _ "github.com/alist-org/alist/v3/drivers/mega" _ "github.com/alist-org/alist/v3/drivers/misskey" diff --git a/drivers/mediafire/driver.go b/drivers/mediafire/driver.go new file mode 100644 index 00000000000..94d056a74aa --- /dev/null +++ b/drivers/mediafire/driver.go @@ -0,0 +1,427 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "context" + "fmt" + "net/http" + "os" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Mediafire struct { + model.Storage + Addition + + actionToken string + + appBase string + apiBase string + hostBase string + maxRetries int + + secChUa string + secChUaPlatform string + userAgent string +} + +func (d *Mediafire) Config() driver.Config { + return config +} + +func (d *Mediafire) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Mediafire) Init(ctx context.Context) error { + if d.SessionToken == "" { + return fmt.Errorf("Init :: [MediaFire] {critical} missing sessionToken") + } + + if d.Cookie == "" { + return fmt.Errorf("Init :: [MediaFire] {critical} missing Cookie") + } + + if _, err := d.getSessionToken(ctx); err != nil { + + //fmt.Printf("Init :: Obtain Session Token \n\n") + + if err := d.renewToken(ctx); err != nil { + + //fmt.Printf("Init :: Renew Session Token \n\n") + } + } + + return nil +} + +func (d *Mediafire) Drop(ctx context.Context) error { + return nil +} + +func (d *Mediafire) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(ctx, dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return d.fileToObj(src), nil + }) +} + +func (d *Mediafire) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + + downloadUrl, err := d.getDirectDownloadLink(ctx, file.GetID()) + if err != nil { + return nil, err + } + + res, err := base.NoRedirectClient.R().SetDoNotParseResponse(true).SetContext(ctx).Get(downloadUrl) + if err != nil { + return nil, err + } + defer func() { + _ = res.RawBody().Close() + }() + + if res.StatusCode() == 302 { + downloadUrl = res.Header().Get("location") + } + + return &model.Link{ + URL: downloadUrl, + Header: http.Header{ + "Origin": []string{d.appBase}, + "Referer": []string{d.appBase + "/"}, + "sec-ch-ua": []string{d.secChUa}, + "sec-ch-ua-platform": []string{d.secChUaPlatform}, + "User-Agent": []string{d.userAgent}, + //"User-Agent": []string{base.UserAgent}, + }, + }, nil +} + +func (d *Mediafire) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "parent_key": parentDir.GetID(), + "foldername": dirName, + } + + var resp MediafireFolderCreateResponse + _, err := d.postForm("/folder/create.php", data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + created, _ := time.Parse("2006-01-02T15:04:05Z", resp.Response.CreatedUTC) + + return &model.ObjThumb{ + Object: model.Object{ + ID: resp.Response.FolderKey, + Name: resp.Response.Name, + Size: 0, + Modified: created, + Ctime: created, + IsFolder: true, + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/move.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key_src": srcObj.GetID(), + "folder_key_dst": dstDir.GetID(), + } + } else { + + endpoint = "/file/move.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "folder_key": dstDir.GetID(), + } + } + + var resp MediafireMoveResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return srcObj, nil +} + +func (d *Mediafire) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/update.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": srcObj.GetID(), + "foldername": newName, + } + } else { + + endpoint = "/file/update.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "filename": newName, + } + } + + var resp MediafireRenameResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/copy.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key_src": srcObj.GetID(), + "folder_key_dst": dstDir.GetID(), + } + } else { + + endpoint = "/file/copy.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "folder_key": dstDir.GetID(), + } + } + + var resp MediafireCopyResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + var newID string + if srcObj.IsDir() { + if len(resp.Response.NewFolderKeys) > 0 { + newID = resp.Response.NewFolderKeys[0] + } + } else { + if len(resp.Response.NewQuickKeys) > 0 { + newID = resp.Response.NewQuickKeys[0] + } + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: newID, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Remove(ctx context.Context, obj model.Obj) error { + var data map[string]string + var endpoint string + + if obj.IsDir() { + + endpoint = "/folder/delete.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": obj.GetID(), + } + } else { + + endpoint = "/file/delete.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": obj.GetID(), + } + } + + var resp MediafireRemoveResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return err + } + + if resp.Response.Result != "Success" { + return fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return nil +} + +func (d *Mediafire) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + _, err := d.PutResult(ctx, dstDir, file, up) + return err +} + +func (d *Mediafire) PutResult(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return nil, err + } + defer tempFile.Close() + + osFile, ok := tempFile.(*os.File) + if !ok { + return nil, fmt.Errorf("expected *os.File, got %T", tempFile) + } + + fileHash, err := d.calculateSHA256(osFile) + if err != nil { + return nil, err + } + + checkResp, err := d.uploadCheck(ctx, file.GetName(), file.GetSize(), fileHash, dstDir.GetID()) + if err != nil { + return nil, err + } + + if checkResp.Response.ResumableUpload.AllUnitsReady == "yes" { + up(100.0) + } + + if checkResp.Response.HashExists == "yes" && checkResp.Response.InAccount == "yes" { + up(100.0) + existingFile, err := d.getExistingFileInfo(ctx, fileHash, file.GetName(), dstDir.GetID()) + if err == nil { + return existingFile, nil + } + } + + var pollKey string + + if checkResp.Response.ResumableUpload.AllUnitsReady != "yes" { + + var err error + + pollKey, err = d.uploadUnits(ctx, osFile, checkResp, file.GetName(), fileHash, dstDir.GetID(), up) + if err != nil { + return nil, err + } + } else { + + pollKey = checkResp.Response.ResumableUpload.UploadKey + } + + //fmt.Printf("pollKey: %+v\n", pollKey) + + pollResp, err := d.pollUpload(ctx, pollKey) + if err != nil { + return nil, err + } + + quickKey := pollResp.Response.Doupload.QuickKey + + return &model.ObjThumb{ + Object: model.Object{ + ID: quickKey, + Name: file.GetName(), + Size: file.GetSize(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *Mediafire) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Mediafire)(nil) diff --git a/drivers/mediafire/meta.go b/drivers/mediafire/meta.go new file mode 100644 index 00000000000..243d55570af --- /dev/null +++ b/drivers/mediafire/meta.go @@ -0,0 +1,54 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + //driver.RootID + + SessionToken string `json:"session_token" required:"true" type:"string" help:"Required for MediaFire API"` + Cookie string `json:"cookie" required:"true" type:"string" help:"Required for navigation"` + + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"100"` +} + +var config = driver.Config{ + Name: "MediaFire", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Mediafire{ + appBase: "https://app.mediafire.com", + apiBase: "https://www.mediafire.com/api/1.5", + hostBase: "https://www.mediafire.com", + maxRetries: 3, + secChUa: "\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"139\", \"Google Chrome\";v=\"139\"", + secChUaPlatform: "Windows", + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36", + } + }) +} diff --git a/drivers/mediafire/types.go b/drivers/mediafire/types.go new file mode 100644 index 00000000000..0073b58179c --- /dev/null +++ b/drivers/mediafire/types.go @@ -0,0 +1,232 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +type MediafireRenewTokenResponse struct { + Response struct { + Action string `json:"action"` + SessionToken string `json:"session_token"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireResponse struct { + Response struct { + Action string `json:"action"` + FolderContent struct { + ChunkSize string `json:"chunk_size"` + ContentType string `json:"content_type"` + ChunkNumber string `json:"chunk_number"` + FolderKey string `json:"folderkey"` + Folders []MediafireFolder `json:"folders,omitempty"` + Files []MediafireFile `json:"files,omitempty"` + MoreChunks string `json:"more_chunks"` + } `json:"folder_content"` + Result string `json:"result"` + } `json:"response"` +} + +type MediafireFolder struct { + FolderKey string `json:"folderkey"` + Name string `json:"name"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` +} + +type MediafireFile struct { + QuickKey string `json:"quickkey"` + Filename string `json:"filename"` + Size string `json:"size"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + MimeType string `json:"mimetype"` +} + +type File struct { + ID string + Name string + Size int64 + CreatedUTC string + IsFolder bool +} + +type FolderContentResponse struct { + Folders []MediafireFolder + Files []MediafireFile + MoreChunks bool +} + +type MediafireLinksResponse struct { + Response struct { + Action string `json:"action"` + Links []struct { + QuickKey string `json:"quickkey"` + View string `json:"view"` + NormalDownload string `json:"normal_download"` + OneTime struct { + Download string `json:"download"` + View string `json:"view"` + } `json:"one_time"` + } `json:"links"` + OneTimeKeyRequestCount string `json:"one_time_key_request_count"` + OneTimeKeyRequestMaxCount string `json:"one_time_key_request_max_count"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireDirectDownloadResponse struct { + Response struct { + Action string `json:"action"` + Links []struct { + QuickKey string `json:"quickkey"` + DirectDownload string `json:"direct_download"` + } `json:"links"` + DirectDownloadFreeBandwidth string `json:"direct_download_free_bandwidth"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireFolderCreateResponse struct { + Response struct { + Action string `json:"action"` + FolderKey string `json:"folder_key"` + UploadKey string `json:"upload_key"` + ParentFolderKey string `json:"parent_folderkey"` + Name string `json:"name"` + Description string `json:"description"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + Privacy string `json:"privacy"` + FileCount string `json:"file_count"` + FolderCount string `json:"folder_count"` + Revision string `json:"revision"` + DropboxEnabled string `json:"dropbox_enabled"` + Flag string `json:"flag"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireMoveResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + NewNames []string `json:"new_names"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireRenameResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireCopyResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + NewQuickKeys []string `json:"new_quickkeys,omitempty"` + NewFolderKeys []string `json:"new_folderkeys,omitempty"` + SkippedCount string `json:"skipped_count,omitempty"` + OtherCount string `json:"other_count,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireRemoveResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireCheckResponse struct { + Response struct { + Action string `json:"action"` + HashExists string `json:"hash_exists"` + InAccount string `json:"in_account"` + InFolder string `json:"in_folder"` + FileExists string `json:"file_exists"` + ResumableUpload struct { + AllUnitsReady string `json:"all_units_ready"` + NumberOfUnits string `json:"number_of_units"` + UnitSize string `json:"unit_size"` + Bitmap struct { + Count string `json:"count"` + Words []string `json:"words"` + } `json:"bitmap"` + UploadKey string `json:"upload_key"` + } `json:"resumable_upload"` + AvailableSpace string `json:"available_space"` + UsedStorageSize string `json:"used_storage_size"` + StorageLimit string `json:"storage_limit"` + StorageLimitExceeded string `json:"storage_limit_exceeded"` + UploadURL struct { + Simple string `json:"simple"` + SimpleFallback string `json:"simple_fallback"` + Resumable string `json:"resumable"` + ResumableFallback string `json:"resumable_fallback"` + } `json:"upload_url"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} +type MediafireActionTokenResponse struct { + Response struct { + Action string `json:"action"` + ActionToken string `json:"action_token"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafirePollResponse struct { + Response struct { + Action string `json:"action"` + Doupload struct { + Result string `json:"result"` + Status string `json:"status"` + Description string `json:"description"` + QuickKey string `json:"quickkey"` + Hash string `json:"hash"` + Filename string `json:"filename"` + Size string `json:"size"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + Revision string `json:"revision"` + } `json:"doupload"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireFileSearchResponse struct { + Response struct { + Action string `json:"action"` + FileInfo []File `json:"file_info"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} diff --git a/drivers/mediafire/util.go b/drivers/mediafire/util.go new file mode 100644 index 00000000000..42febf0bb7a --- /dev/null +++ b/drivers/mediafire/util.go @@ -0,0 +1,620 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { + tokenURL := d.hostBase + "/application/get_session_token.php" + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, tokenURL, nil) + if err != nil { + return "", err + } + + req.Header.Set("Accept", "*/*") + req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Content-Length", "0") + req.Header.Set("Cookie", d.Cookie) + req.Header.Set("DNT", "1") + req.Header.Set("Origin", d.hostBase) + req.Header.Set("Priority", "u=1, i") + req.Header.Set("Referer", (d.hostBase + "/")) + req.Header.Set("Sec-Ch-Ua", d.secChUa) + req.Header.Set("Sec-Ch-Ua-Mobile", "?0") + req.Header.Set("Sec-Ch-Ua-Platform", d.secChUaPlatform) + req.Header.Set("Sec-Fetch-Dest", "empty") + req.Header.Set("Sec-Fetch-Mode", "cors") + req.Header.Set("Sec-Fetch-Site", "same-site") + req.Header.Set("User-Agent", d.userAgent) + //req.Header.Set("Connection", "keep-alive") + + resp, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + //fmt.Printf("getSessionToken :: Raw response: %s\n", string(body)) + //fmt.Printf("getSessionToken :: Parsed response: %+v\n", resp) + + var tokenResp struct { + Response struct { + SessionToken string `json:"session_token"` + } `json:"response"` + } + + if resp.StatusCode == 200 { + if err := json.Unmarshal(body, &tokenResp); err != nil { + return "", err + } + + if tokenResp.Response.SessionToken == "" { + return "", fmt.Errorf("empty session token received") + } + + cookieMap := make(map[string]string) + for _, cookie := range resp.Cookies() { + cookieMap[cookie.Name] = cookie.Value + } + + if len(cookieMap) > 0 { + + var cookies []string + for name, value := range cookieMap { + cookies = append(cookies, fmt.Sprintf("%s=%s", name, value)) + } + d.Cookie = strings.Join(cookies, "; ") + op.MustSaveDriverStorage(d) + + //fmt.Printf("getSessionToken :: Captured cookies: %s\n", d.Cookie) + } + + } else { + return "", fmt.Errorf("getSessionToken :: failed to get session token, status code: %d", resp.StatusCode) + } + + d.SessionToken = tokenResp.Response.SessionToken + op.MustSaveDriverStorage(d) + + return d.SessionToken, nil +} + +func (d *Mediafire) renewToken(_ context.Context) error { + query := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + } + + var resp MediafireRenewTokenResponse + _, err := d.postForm("/user/renew_session_token.php", query, &resp) + if err != nil { + return fmt.Errorf("failed to renew token: %w", err) + } + + //fmt.Printf("getInfo :: Raw response: %s\n", string(body)) + //fmt.Printf("getInfo :: Parsed response: %+v\n", resp) + + if resp.Response.Result != "Success" { + return fmt.Errorf("MediaFire token renewal failed: %s", resp.Response.Result) + } + + d.SessionToken = resp.Response.SessionToken + op.MustSaveDriverStorage(d) + + return nil +} + +func (d *Mediafire) getFiles(ctx context.Context, folderKey string) ([]File, error) { + files := make([]File, 0) + hasMore := true + chunkNumber := 1 + + for hasMore { + resp, err := d.getFolderContent(ctx, folderKey, chunkNumber) + if err != nil { + return nil, err + } + + for _, folder := range resp.Folders { + files = append(files, File{ + ID: folder.FolderKey, + Name: folder.Name, + Size: 0, + CreatedUTC: folder.CreatedUTC, + IsFolder: true, + }) + } + + for _, file := range resp.Files { + size, _ := strconv.ParseInt(file.Size, 10, 64) + files = append(files, File{ + ID: file.QuickKey, + Name: file.Filename, + Size: size, + CreatedUTC: file.CreatedUTC, + IsFolder: false, + }) + } + + hasMore = resp.MoreChunks + chunkNumber++ + } + + return files, nil +} + +func (d *Mediafire) getFolderContent(ctx context.Context, folderKey string, chunkNumber int) (*FolderContentResponse, error) { + + foldersResp, err := d.getFolderContentByType(ctx, folderKey, "folders", chunkNumber) + if err != nil { + return nil, err + } + + filesResp, err := d.getFolderContentByType(ctx, folderKey, "files", chunkNumber) + if err != nil { + return nil, err + } + + return &FolderContentResponse{ + Folders: foldersResp.Response.FolderContent.Folders, + Files: filesResp.Response.FolderContent.Files, + MoreChunks: foldersResp.Response.FolderContent.MoreChunks == "yes" || filesResp.Response.FolderContent.MoreChunks == "yes", + }, nil +} + +func (d *Mediafire) getFolderContentByType(_ context.Context, folderKey, contentType string, chunkNumber int) (*MediafireResponse, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": folderKey, + "content_type": contentType, + "chunk": strconv.Itoa(chunkNumber), + "chunk_size": strconv.FormatInt(d.ChunkSize, 10), + "details": "yes", + "order_direction": d.OrderDirection, + "order_by": d.OrderBy, + "filter": "", + } + + var resp MediafireResponse + _, err := d.postForm("/folder/get_content.php", data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) fileToObj(f File) *model.ObjThumb { + created, _ := time.Parse("2006-01-02T15:04:05Z", f.CreatedUTC) + + var thumbnailURL string + if !f.IsFolder && f.ID != "" { + thumbnailURL = d.hostBase + "/convkey/acaa/" + f.ID + "3g.jpg" + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: f.ID, + //Path: "", + Name: f.Name, + Size: f.Size, + Modified: created, + Ctime: created, + IsFolder: f.IsFolder, + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumbnailURL, + }, + } +} + +func (d *Mediafire) getForm(endpoint string, query map[string]string, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + + req.SetQueryParams(query) + + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + //"User-Agent": base.UserAgent, + "User-Agent": d.userAgent, + "Origin": d.appBase, + "Referer": d.appBase + "/", + }) + + // If response OK + if resp != nil { + req.SetResult(resp) + } + + // Targets MediaFire API + res, err := req.Get(d.apiBase + endpoint) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (d *Mediafire) postForm(endpoint string, data map[string]string, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + + req.SetFormData(data) + + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + "Content-Type": "application/x-www-form-urlencoded", + //"User-Agent": base.UserAgent, + "User-Agent": d.userAgent, + "Origin": d.appBase, + "Referer": d.appBase + "/", + }) + + // If response OK + if resp != nil { + req.SetResult(resp) + } + + // Targets MediaFire API + res, err := req.Post(d.apiBase + endpoint) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (d *Mediafire) getDirectDownloadLink(_ context.Context, fileID string) (string, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "quick_key": fileID, + "link_type": "direct_download", + "response_format": "json", + } + + var resp MediafireDirectDownloadResponse + _, err := d.getForm("/file/get_links.php", data, &resp) + if err != nil { + return "", err + } + + if resp.Response.Result != "Success" { + return "", fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + if len(resp.Response.Links) == 0 { + return "", fmt.Errorf("no download links found") + } + + return resp.Response.Links[0].DirectDownload, nil +} + +func (d *Mediafire) calculateSHA256(file *os.File) (string, error) { + hasher := sha256.New() + if _, err := file.Seek(0, 0); err != nil { + return "", err + } + if _, err := io.Copy(hasher, file); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func (d *Mediafire) uploadCheck(ctx context.Context, filename string, filesize int64, filehash, folderKey string) (*MediafireCheckResponse, error) { + + actionToken, err := d.getActionToken(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get action token: %w", err) + } + + query := map[string]string{ + "session_token": actionToken, /* d.SessionToken */ + "filename": filename, + "size": strconv.FormatInt(filesize, 10), + "hash": filehash, + "folder_key": folderKey, + "resumable": "yes", + "response_format": "json", + } + + var resp MediafireCheckResponse + _, err = d.postForm("/upload/check.php", query, &resp) + if err != nil { + return nil, err + } + + //fmt.Printf("uploadCheck :: Raw response: %s\n", string(body)) + //fmt.Printf("uploadCheck :: Parsed response: %+v\n", resp) + + //fmt.Printf("uploadCheck :: ResumableUpload section: %+v\n", resp.Response.ResumableUpload) + //fmt.Printf("uploadCheck :: Upload key specifically: '%s'\n", resp.Response.ResumableUpload.UploadKey) + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire upload check failed: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) resumableUpload(ctx context.Context, folderKey, uploadKey string, unitData []byte, unitID int, fileHash, filename string, totalFileSize int64) (string, error) { + actionToken, err := d.getActionToken(ctx) + if err != nil { + return "", err + } + + url := d.apiBase + "/upload/resumable.php" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(unitData)) + if err != nil { + return "", err + } + + q := req.URL.Query() + q.Add("folder_key", folderKey) + q.Add("response_format", "json") + q.Add("session_token", actionToken) + q.Add("key", uploadKey) + req.URL.RawQuery = q.Encode() + + req.Header.Set("x-filehash", fileHash) + req.Header.Set("x-filesize", strconv.FormatInt(totalFileSize, 10)) + req.Header.Set("x-unit-id", strconv.Itoa(unitID)) + req.Header.Set("x-unit-size", strconv.FormatInt(int64(len(unitData)), 10)) + req.Header.Set("x-unit-hash", d.sha256Hex(bytes.NewReader(unitData))) + req.Header.Set("x-filename", filename) + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = int64(len(unitData)) + + /* fmt.Printf("Debug resumable upload request:\n") + fmt.Printf(" URL: %s\n", req.URL.String()) + fmt.Printf(" Headers: %+v\n", req.Header) + fmt.Printf(" Unit ID: %d\n", unitID) + fmt.Printf(" Unit Size: %d\n", len(unitData)) + fmt.Printf(" Upload Key: %s\n", uploadKey) + fmt.Printf(" Action Token: %s\n", actionToken) */ + + res, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %v", err) + } + + //fmt.Printf("MediaFire resumable upload response (status %d): %s\n", res.StatusCode, string(body)) + + var uploadResp struct { + Response struct { + Doupload struct { + Key string `json:"key"` + } `json:"doupload"` + Result string `json:"result"` + } `json:"response"` + } + + if err := json.Unmarshal(body, &uploadResp); err != nil { + return "", fmt.Errorf("failed to parse response: %v", err) + } + + if res.StatusCode != 200 { + return "", fmt.Errorf("resumable upload failed with status %d", res.StatusCode) + } + + return uploadResp.Response.Doupload.Key, nil +} + +func (d *Mediafire) uploadUnits(ctx context.Context, file *os.File, checkResp *MediafireCheckResponse, filename, fileHash, folderKey string, up driver.UpdateProgress) (string, error) { + unitSize, _ := strconv.ParseInt(checkResp.Response.ResumableUpload.UnitSize, 10, 64) + numUnits, _ := strconv.Atoi(checkResp.Response.ResumableUpload.NumberOfUnits) + uploadKey := checkResp.Response.ResumableUpload.UploadKey + + stringWords := checkResp.Response.ResumableUpload.Bitmap.Words + intWords := make([]int, len(stringWords)) + for i, word := range stringWords { + intWords[i], _ = strconv.Atoi(word) + } + + var finalUploadKey string + + for unitID := 0; unitID < numUnits; unitID++ { + + if utils.IsCanceled(ctx) { + return "", ctx.Err() + } + + if d.isUnitUploaded(intWords, unitID) { + up(float64(unitID+1) * 100 / float64(numUnits)) + continue + } + + uploadKey, err := d.uploadSingleUnit(ctx, file, unitID, unitSize, fileHash, filename, uploadKey, folderKey) + if err != nil { + return "", err + } + + finalUploadKey = uploadKey + + up(float64(unitID+1) * 100 / float64(numUnits)) + } + + return finalUploadKey, nil +} + +func (d *Mediafire) uploadSingleUnit(ctx context.Context, file *os.File, unitID int, unitSize int64, fileHash, filename, uploadKey, folderKey string) (string, error) { + start := int64(unitID) * unitSize + size := unitSize + + stat, err := file.Stat() + if err != nil { + return "", err + } + fileSize := stat.Size() + + if start+size > fileSize { + size = fileSize - start + } + + unitData := make([]byte, size) + if _, err := file.ReadAt(unitData, start); err != nil { + return "", err + } + + return d.resumableUpload(ctx, folderKey, uploadKey, unitData, unitID, fileHash, filename, fileSize) +} + +func (d *Mediafire) getActionToken(_ context.Context) (string, error) { + + if d.actionToken != "" { + return d.actionToken, nil + } + + data := map[string]string{ + "type": "upload", + "lifespan": "1440", + "response_format": "json", + "session_token": d.SessionToken, + } + + var resp MediafireActionTokenResponse + _, err := d.postForm("/user/get_action_token.php", data, &resp) + if err != nil { + return "", err + } + + if resp.Response.Result != "Success" { + return "", fmt.Errorf("MediaFire action token failed: %s", resp.Response.Result) + } + + return resp.Response.ActionToken, nil +} + +func (d *Mediafire) pollUpload(ctx context.Context, key string) (*MediafirePollResponse, error) { + + actionToken, err := d.getActionToken(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get action token: %w", err) + } + + //fmt.Printf("Debug Key: %+v\n", key) + + query := map[string]string{ + "key": key, + "response_format": "json", + "session_token": actionToken, /* d.SessionToken */ + } + + var resp MediafirePollResponse + _, err = d.postForm("/upload/poll_upload.php", query, &resp) + if err != nil { + return nil, err + } + + //fmt.Printf("pollUpload :: Raw response: %s\n", string(body)) + //fmt.Printf("pollUpload :: Parsed response: %+v\n", resp) + + //fmt.Printf("pollUpload :: Debug Result: %+v\n", resp.Response.Result) + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire poll upload failed: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) sha256Hex(r io.Reader) string { + h := sha256.New() + io.Copy(h, r) + return hex.EncodeToString(h.Sum(nil)) +} + +func (d *Mediafire) isUnitUploaded(words []int, unitID int) bool { + wordIndex := unitID / 16 + bitIndex := unitID % 16 + if wordIndex >= len(words) { + return false + } + return (words[wordIndex]>>bitIndex)&1 == 1 +} + +func (d *Mediafire) getExistingFileInfo(ctx context.Context, fileHash, filename, folderKey string) (*model.ObjThumb, error) { + + if fileInfo, err := d.getFileByHash(ctx, fileHash); err == nil && fileInfo != nil { + return fileInfo, nil + } + + files, err := d.getFiles(ctx, folderKey) + if err != nil { + return nil, err + } + + for _, file := range files { + if file.Name == filename && !file.IsFolder { + return d.fileToObj(file), nil + } + } + + return nil, fmt.Errorf("existing file not found") +} + +func (d *Mediafire) getFileByHash(_ context.Context, hash string) (*model.ObjThumb, error) { + query := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "hash": hash, + } + + var resp MediafireFileSearchResponse + _, err := d.postForm("/file/get_info.php", query, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire file search failed: %s", resp.Response.Result) + } + + if len(resp.Response.FileInfo) == 0 { + return nil, fmt.Errorf("file not found by hash") + } + + file := resp.Response.FileInfo[0] + return d.fileToObj(file), nil +} From 28a8428559fe8a1a4b7dd3bc6f2a7549e1ea52de Mon Sep 17 00:00:00 2001 From: Chesyre <56254560+Chesyre@users.noreply.github.com> Date: Thu, 11 Sep 2025 05:46:31 +0200 Subject: [PATCH 044/133] feat(driver): add Gofile storage driver (#9318) Add support for Gofile.io cloud storage service with full CRUD operations. Features: - File and folder listing - Upload and download functionality - Create, move, rename, copy, and delete operations - Direct link generation for file access - API token authentication The driver implements all required driver interfaces and follows the existing driver patterns in the codebase. --- drivers/all.go | 1 + drivers/gofile/driver.go | 261 +++++++++++++++++++++++++++++++++++++++ drivers/gofile/meta.go | 26 ++++ drivers/gofile/types.go | 124 +++++++++++++++++++ drivers/gofile/util.go | 257 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 669 insertions(+) create mode 100644 drivers/gofile/driver.go create mode 100644 drivers/gofile/meta.go create mode 100644 drivers/gofile/types.go create mode 100644 drivers/gofile/util.go diff --git a/drivers/all.go b/drivers/all.go index 5c3cc570805..140908a8945 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -32,6 +32,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/ftp" _ "github.com/alist-org/alist/v3/drivers/github" _ "github.com/alist-org/alist/v3/drivers/github_releases" + _ "github.com/alist-org/alist/v3/drivers/gofile" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" _ "github.com/alist-org/alist/v3/drivers/halalcloud" diff --git a/drivers/gofile/driver.go b/drivers/gofile/driver.go new file mode 100644 index 00000000000..301eaef3405 --- /dev/null +++ b/drivers/gofile/driver.go @@ -0,0 +1,261 @@ +package gofile + +import ( + "context" + "fmt" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" +) + +type Gofile struct { + model.Storage + Addition + + accountId string +} + +func (d *Gofile) Config() driver.Config { + return config +} + +func (d *Gofile) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Gofile) Init(ctx context.Context) error { + if d.APIToken == "" { + return fmt.Errorf("API token is required") + } + + // Get account ID + accountId, err := d.getAccountId(ctx) + if err != nil { + return fmt.Errorf("failed to get account ID: %w", err) + } + d.accountId = accountId + + // Get account info to set root folder if not specified + if d.RootFolderID == "" { + accountInfo, err := d.getAccountInfo(ctx, accountId) + if err != nil { + return fmt.Errorf("failed to get account info: %w", err) + } + d.RootFolderID = accountInfo.Data.RootFolder + } + + // Save driver storage + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Gofile) Drop(ctx context.Context) error { + return nil +} + +func (d *Gofile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var folderId string + if dir.GetID() == "" { + folderId = d.GetRootId() + } else { + folderId = dir.GetID() + } + + endpoint := fmt.Sprintf("/contents/%s", folderId) + + var response ContentsResponse + err := d.getJSON(ctx, endpoint, &response) + if err != nil { + return nil, err + } + + var objects []model.Obj + + // Process children or contents + contents := response.Data.Children + if contents == nil { + contents = response.Data.Contents + } + + for _, content := range contents { + objects = append(objects, d.convertContentToObj(content)) + } + + return objects, nil +} + +func (d *Gofile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + + // Create a direct link for the file + directLink, err := d.createDirectLink(ctx, file.GetID()) + if err != nil { + return nil, fmt.Errorf("failed to create direct link: %w", err) + } + + return &model.Link{ + URL: directLink, + }, nil +} + +func (d *Gofile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var parentId string + if parentDir.GetID() == "" { + parentId = d.GetRootId() + } else { + parentId = parentDir.GetID() + } + + data := map[string]interface{}{ + "parentFolderId": parentId, + "folderName": dirName, + } + + var response CreateFolderResponse + err := d.postJSON(ctx, "/contents/createFolder", data, &response) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: response.Data.ID, + Name: response.Data.Name, + IsFolder: true, + }, nil +} + +func (d *Gofile) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var dstId string + if dstDir.GetID() == "" { + dstId = d.GetRootId() + } else { + dstId = dstDir.GetID() + } + + data := map[string]interface{}{ + "contentsId": srcObj.GetID(), + "folderId": dstId, + } + + err := d.putJSON(ctx, "/contents/move", data, nil) + if err != nil { + return nil, err + } + + // Return updated object + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + data := map[string]interface{}{ + "attribute": "name", + "attributeValue": newName, + } + + var response UpdateResponse + err := d.putJSON(ctx, fmt.Sprintf("/contents/%s/update", srcObj.GetID()), data, &response) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var dstId string + if dstDir.GetID() == "" { + dstId = d.GetRootId() + } else { + dstId = dstDir.GetID() + } + + data := map[string]interface{}{ + "contentsId": srcObj.GetID(), + "folderId": dstId, + } + + var response CopyResponse + err := d.postJSON(ctx, "/contents/copy", data, &response) + if err != nil { + return nil, err + } + + // Get the new ID from the response + newId := srcObj.GetID() + if response.Data.CopiedContents != nil { + if id, ok := response.Data.CopiedContents[srcObj.GetID()]; ok { + newId = id + } + } + + return &model.Object{ + ID: newId, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Remove(ctx context.Context, obj model.Obj) error { + data := map[string]interface{}{ + "contentsId": obj.GetID(), + } + + return d.deleteJSON(ctx, "/contents", data) +} + +func (d *Gofile) Put(ctx context.Context, dstDir model.Obj, fileStreamer model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var folderId string + if dstDir.GetID() == "" { + folderId = d.GetRootId() + } else { + folderId = dstDir.GetID() + } + + response, err := d.uploadFile(ctx, folderId, fileStreamer, up) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: response.Data.FileId, + Name: response.Data.FileName, + Size: fileStreamer.GetSize(), + IsFolder: false, + }, nil +} + +func (d *Gofile) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*Gofile)(nil) \ No newline at end of file diff --git a/drivers/gofile/meta.go b/drivers/gofile/meta.go new file mode 100644 index 00000000000..b8126e337b4 --- /dev/null +++ b/drivers/gofile/meta.go @@ -0,0 +1,26 @@ +package gofile + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APIToken string `json:"api_token" required:"true" help:"Get your API token from your Gofile profile page"` +} + +var config = driver.Config{ + Name: "Gofile", + DefaultRoot: "", + LocalSort: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Gofile{} + }) +} \ No newline at end of file diff --git a/drivers/gofile/types.go b/drivers/gofile/types.go new file mode 100644 index 00000000000..93c9f5d2e6a --- /dev/null +++ b/drivers/gofile/types.go @@ -0,0 +1,124 @@ +package gofile + +import "time" + +type APIResponse struct { + Status string `json:"status"` + Data interface{} `json:"data"` +} + +type AccountResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + } `json:"data"` +} + +type AccountInfoResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Email string `json:"email"` + RootFolder string `json:"rootFolder"` + } `json:"data"` +} + +type Content struct { + ID string `json:"id"` + Type string `json:"type"` // "file" or "folder" + Name string `json:"name"` + Size int64 `json:"size,omitempty"` + CreateTime int64 `json:"createTime"` + ModTime int64 `json:"modTime,omitempty"` + DirectLink string `json:"directLink,omitempty"` + Children map[string]Content `json:"children,omitempty"` + ParentFolder string `json:"parentFolder,omitempty"` + MD5 string `json:"md5,omitempty"` + MimeType string `json:"mimeType,omitempty"` + Link string `json:"link,omitempty"` +} + +type ContentsResponse struct { + Status string `json:"status"` + Data struct { + IsOwner bool `json:"isOwner"` + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + ParentFolder string `json:"parentFolder"` + CreateTime int64 `json:"createTime"` + ChildrenList []string `json:"childrenList,omitempty"` + Children map[string]Content `json:"children,omitempty"` + Contents map[string]Content `json:"contents,omitempty"` + Public bool `json:"public,omitempty"` + Description string `json:"description,omitempty"` + Tags string `json:"tags,omitempty"` + Expiry int64 `json:"expiry,omitempty"` + } `json:"data"` +} + +type UploadResponse struct { + Status string `json:"status"` + Data struct { + DownloadPage string `json:"downloadPage"` + Code string `json:"code"` + ParentFolder string `json:"parentFolder"` + FileId string `json:"fileId"` + FileName string `json:"fileName"` + GuestToken string `json:"guestToken,omitempty"` + } `json:"data"` +} + +type DirectLinkResponse struct { + Status string `json:"status"` + Data struct { + DirectLink string `json:"directLink"` + ID string `json:"id"` + } `json:"data"` +} + +type CreateFolderResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + ParentFolder string `json:"parentFolder"` + CreateTime int64 `json:"createTime"` + } `json:"data"` +} + +type CopyResponse struct { + Status string `json:"status"` + Data struct { + CopiedContents map[string]string `json:"copiedContents"` // oldId -> newId mapping + } `json:"data"` +} + +type UpdateResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"data"` +} + +type ErrorResponse struct { + Status string `json:"status"` + Error struct { + Message string `json:"message"` + Code string `json:"code"` + } `json:"error"` +} + +func (c *Content) ModifiedTime() time.Time { + if c.ModTime > 0 { + return time.Unix(c.ModTime, 0) + } + return time.Unix(c.CreateTime, 0) +} + +func (c *Content) IsDir() bool { + return c.Type == "folder" +} \ No newline at end of file diff --git a/drivers/gofile/util.go b/drivers/gofile/util.go new file mode 100644 index 00000000000..5f39dae5a81 --- /dev/null +++ b/drivers/gofile/util.go @@ -0,0 +1,257 @@ +package gofile + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" +) + +const ( + baseAPI = "https://api.gofile.io" + uploadAPI = "https://upload.gofile.io" +) + +func (d *Gofile) request(ctx context.Context, method, endpoint string, body io.Reader, headers map[string]string) (*http.Response, error) { + var url string + if strings.HasPrefix(endpoint, "http") { + url = endpoint + } else { + url = baseAPI + endpoint + } + + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return nil, err + } + + req.Header.Set("Authorization", "Bearer "+d.APIToken) + req.Header.Set("User-Agent", "AList/3.0") + + for k, v := range headers { + req.Header.Set(k, v) + } + + return base.HttpClient.Do(req) +} + +func (d *Gofile) getJSON(ctx context.Context, endpoint string, result interface{}) error { + resp, err := d.request(ctx, "GET", endpoint, nil, nil) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + return json.NewDecoder(resp.Body).Decode(result) +} + +func (d *Gofile) postJSON(ctx context.Context, endpoint string, data interface{}, result interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "POST", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + if result != nil { + return json.NewDecoder(resp.Body).Decode(result) + } + + return nil +} + +func (d *Gofile) putJSON(ctx context.Context, endpoint string, data interface{}, result interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "PUT", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + if result != nil { + return json.NewDecoder(resp.Body).Decode(result) + } + + return nil +} + +func (d *Gofile) deleteJSON(ctx context.Context, endpoint string, data interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "DELETE", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + return nil +} + +func (d *Gofile) handleError(resp *http.Response) error { + body, _ := io.ReadAll(resp.Body) + + var errorResp ErrorResponse + if err := json.Unmarshal(body, &errorResp); err == nil { + return fmt.Errorf("gofile API error: %s (code: %s)", errorResp.Error.Message, errorResp.Error.Code) + } + + return fmt.Errorf("gofile API error: HTTP %d - %s", resp.StatusCode, string(body)) +} + +func (d *Gofile) uploadFile(ctx context.Context, folderId string, file model.FileStreamer, up driver.UpdateProgress) (*UploadResponse, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + if folderId != "" { + writer.WriteField("folderId", folderId) + } + + part, err := writer.CreateFormFile("file", filepath.Base(file.GetName())) + if err != nil { + return nil, err + } + + // Copy with progress tracking if available + if up != nil { + reader := &progressReader{ + reader: file, + total: file.GetSize(), + up: up, + } + _, err = io.Copy(part, reader) + } else { + _, err = io.Copy(part, file) + } + + if err != nil { + return nil, err + } + + writer.Close() + + headers := map[string]string{ + "Content-Type": writer.FormDataContentType(), + } + + resp, err := d.request(ctx, "POST", uploadAPI+"/uploadfile", &body, headers) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, d.handleError(resp) + } + + var result UploadResponse + err = json.NewDecoder(resp.Body).Decode(&result) + return &result, err +} + +func (d *Gofile) createDirectLink(ctx context.Context, contentId string) (string, error) { + data := map[string]interface{}{} + + var result DirectLinkResponse + err := d.postJSON(ctx, fmt.Sprintf("/contents/%s/directlinks", contentId), data, &result) + if err != nil { + return "", err + } + + return result.Data.DirectLink, nil +} + +func (d *Gofile) convertContentToObj(content Content) model.Obj { + return &model.ObjThumb{ + Object: model.Object{ + ID: content.ID, + Name: content.Name, + Size: content.Size, + Modified: content.ModifiedTime(), + IsFolder: content.IsDir(), + }, + } +} + +func (d *Gofile) getAccountId(ctx context.Context) (string, error) { + var result AccountResponse + err := d.getJSON(ctx, "/accounts/getid", &result) + if err != nil { + return "", err + } + return result.Data.ID, nil +} + +func (d *Gofile) getAccountInfo(ctx context.Context, accountId string) (*AccountInfoResponse, error) { + var result AccountInfoResponse + err := d.getJSON(ctx, fmt.Sprintf("/accounts/%s", accountId), &result) + if err != nil { + return nil, err + } + return &result, nil +} + +// progressReader wraps an io.Reader to track upload progress +type progressReader struct { + reader io.Reader + total int64 + read int64 + up driver.UpdateProgress +} + +func (pr *progressReader) Read(p []byte) (n int, err error) { + n, err = pr.reader.Read(p) + pr.read += int64(n) + if pr.up != nil && pr.total > 0 { + progress := float64(pr.read) * 100 / float64(pr.total) + pr.up(progress) + } + return n, err +} From 6e7c7d1dd02305c29a04f0492f6154cb14e77378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 11 Sep 2025 21:16:33 +0800 Subject: [PATCH 045/133] refactor (auth): Optimize permission path processing logic (#9320) - Changed permission path collection from map to slice to improve code readability - Removed redundant path checks to improve path addition efficiency - Restructured the loop logic for path processing to simplify the path permission assignment process --- server/handles/auth.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/handles/auth.go b/server/handles/auth.go index dd7d202b907..3520a459c3d 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -165,25 +165,25 @@ func CurrentUser(c *gin.Context) { var roleNames []string permMap := map[string]int32{} - addedPaths := map[string]bool{} + paths := make([]string, 0) for _, role := range user.RolesDetail { roleNames = append(roleNames, role.Name) for _, entry := range role.PermissionScopes { cleanPath := path.Clean("/" + strings.TrimPrefix(entry.Path, "/")) + if _, ok := permMap[cleanPath]; !ok { + paths = append(paths, cleanPath) + } permMap[cleanPath] |= entry.Permission } } userResp.RoleNames = roleNames - for fullPath, perm := range permMap { - if !addedPaths[fullPath] { - userResp.Permissions = append(userResp.Permissions, model.PermissionEntry{ - Path: fullPath, - Permission: perm, - }) - addedPaths[fullPath] = true - } + for _, fullPath := range paths { + userResp.Permissions = append(userResp.Permissions, model.PermissionEntry{ + Path: fullPath, + Permission: permMap[fullPath], + }) } common.SuccessResp(c, userResp) From 16cce37947cab989863a2d444a020ef1c3c46f26 Mon Sep 17 00:00:00 2001 From: "D@' 3z K!7" <99719341+Da3zKi7@users.noreply.github.com> Date: Fri, 12 Sep 2025 03:53:47 -0600 Subject: [PATCH 046/133] fix(drivers): add session renewal cron for MediaFire driver (#9321) - Implement automatic session token renewal every 6-9 minutes - Add validation for required SessionToken and Cookie fields in Init - Handle session expiration by calling renewToken on validation failure - Prevent storage failures due to MediaFire session timeouts Fixes session closure issues that occur after server restarts or extended periods. Co-authored-by: Da3zKi7 --- drivers/mediafire/driver.go | 14 ++++++++++---- drivers/mediafire/util.go | 6 ++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/mediafire/driver.go b/drivers/mediafire/driver.go index 94d056a74aa..e77510eabc0 100644 --- a/drivers/mediafire/driver.go +++ b/drivers/mediafire/driver.go @@ -11,6 +11,7 @@ D@' 3z K!7 - The King Of Cracking import ( "context" "fmt" + "math/rand" "net/http" "os" "time" @@ -19,12 +20,14 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/pkg/utils" ) type Mediafire struct { model.Storage Addition + cron *cron.Cron actionToken string @@ -57,12 +60,15 @@ func (d *Mediafire) Init(ctx context.Context) error { if _, err := d.getSessionToken(ctx); err != nil { - //fmt.Printf("Init :: Obtain Session Token \n\n") + d.renewToken(ctx) - if err := d.renewToken(ctx); err != nil { + num := rand.Intn(4) + 6 + + d.cron = cron.NewCron(time.Minute * time.Duration(num)) + d.cron.Do(func() { + d.renewToken(ctx) + }) - //fmt.Printf("Init :: Renew Session Token \n\n") - } } return nil diff --git a/drivers/mediafire/util.go b/drivers/mediafire/util.go index 42febf0bb7a..091abd0cd64 100644 --- a/drivers/mediafire/util.go +++ b/drivers/mediafire/util.go @@ -106,6 +106,9 @@ func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { } d.SessionToken = tokenResp.Response.SessionToken + + //fmt.Printf("Init :: Obtain Session Token %v", d.SessionToken) + op.MustSaveDriverStorage(d) return d.SessionToken, nil @@ -131,6 +134,9 @@ func (d *Mediafire) renewToken(_ context.Context) error { } d.SessionToken = resp.Response.SessionToken + + //fmt.Printf("Init :: Renew Session Token: %s", resp.Response.Result) + op.MustSaveDriverStorage(d) return nil From e1800f18e4e3746d7792a9064fa2fc0a7fef2b8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 12 Sep 2025 17:56:23 +0800 Subject: [PATCH 047/133] feat: Check usage before deleting storage (#9322) * feat(storage): Added role and user path checking functionality - Added `GetAllRoles` function to retrieve all roles - Added `GetAllUsers` function to retrieve all users - Added `firstPathSegment` function to extract the first segment of a path - Checks whether a storage object is used by a role or user, and returns relevant information for unusing it * fix(storage): Fixed a potential null value issue with not checking firstMount. - Added a check to see if `firstMount` is null to prevent logic errors. - Adjusted the loading logic of `GetAllRoles` and `GetAllUsers` to only execute when `firstMount` is non-null. - Fixed the `usedBy` check logic to ensure that an error message is returned under the correct conditions. - Optimized code structure to reduce unnecessary execution paths. --- internal/db/role.go | 8 ++++++++ internal/db/user.go | 8 ++++++++ internal/op/storage.go | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/internal/db/role.go b/internal/db/role.go index ae62a8ed898..808a6f5f06b 100644 --- a/internal/db/role.go +++ b/internal/db/role.go @@ -34,6 +34,14 @@ func GetRoles(pageIndex, pageSize int) (roles []model.Role, count int64, err err return roles, count, nil } +func GetAllRoles() ([]model.Role, error) { + var roles []model.Role + if err := db.Find(&roles).Error; err != nil { + return nil, errors.WithStack(err) + } + return roles, nil +} + func CreateRole(r *model.Role) error { if err := db.Create(r).Error; err != nil { return errors.WithStack(err) diff --git a/internal/db/user.go b/internal/db/user.go index 8f1c28b92f2..4e5d67ad28e 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -83,6 +83,14 @@ func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err err return users, count, nil } +func GetAllUsers() ([]model.User, error) { + var users []model.User + if err := db.Find(&users).Error; err != nil { + return nil, errors.WithStack(err) + } + return users, nil +} + func DeleteUserById(id uint) error { return errors.WithStack(db.Delete(&model.User{}, id).Error) } diff --git a/internal/op/storage.go b/internal/op/storage.go index 27221e70e8c..dfb305aaa43 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -41,6 +41,18 @@ func GetStorageByMountPath(mountPath string) (driver.Driver, error) { return storageDriver, nil } +func firstPathSegment(p string) string { + p = utils.FixAndCleanPath(p) + p = strings.TrimPrefix(p, "/") + if p == "" { + return "" + } + if i := strings.Index(p, "/"); i >= 0 { + return p[:i] + } + return p +} + // CreateStorage Save the storage to database so storage can get an id // then instantiate corresponding driver and save it in memory func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { @@ -267,6 +279,34 @@ func DeleteStorageById(ctx context.Context, id uint) error { if err != nil { return errors.WithMessage(err, "failed get storage") } + firstMount := firstPathSegment(storage.MountPath) + if firstMount != "" { + roles, err := db.GetAllRoles() + if err != nil { + return errors.WithMessage(err, "failed to load roles") + } + users, err := db.GetAllUsers() + if err != nil { + return errors.WithMessage(err, "failed to load users") + } + var usedBy []string + for _, r := range roles { + for _, entry := range r.PermissionScopes { + if firstPathSegment(entry.Path) == firstMount { + usedBy = append(usedBy, "role:"+r.Name) + break + } + } + } + for _, u := range users { + if firstPathSegment(u.BasePath) == firstMount { + usedBy = append(usedBy, "user:"+u.Username) + } + } + if len(usedBy) > 0 { + return errors.Errorf("storage is used by %s, please cancel usage first", strings.Join(usedBy, ", ")) + } + } if !storage.Disabled { storageDriver, err := GetStorageByMountPath(storage.MountPath) if err != nil { From 4f8bc478d51c6885f168571b718e7341ec8a2653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sun, 14 Sep 2025 21:03:58 +0800 Subject: [PATCH 048/133] refactor(driver): Refactored directory link check logic (#9324) - Use `filePath` variable to simplify path handling - Replace `isSymlinkDir` with `isLinkedDir` in `isFolder` check - Use simplified path variables in `times.Stat` function calls refactor(util): Optimized directory link check functions - Renamed `isSymlinkDir` to `isLinkedDir` to expand Windows platform support - Corrected path resolution logic to ensure link paths are absolute - Added error handling to prevent path resolution failures --- drivers/local/driver.go | 9 +++++---- drivers/local/util.go | 13 +++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index faa2b3bd157..7ff72d11cdf 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -146,13 +146,14 @@ func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(reqPath, f.Name())) } } - isFolder := f.IsDir() || isSymlinkDir(f, fullPath) + filePath := filepath.Join(fullPath, f.Name()) + isFolder := f.IsDir() || isLinkedDir(f, filePath) var size int64 if !isFolder { size = f.Size() } var ctime time.Time - t, err := times.Stat(stdpath.Join(fullPath, f.Name())) + t, err := times.Stat(filePath) if err == nil { if t.HasBirthTime() { ctime = t.BirthTime() @@ -161,7 +162,7 @@ func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string file := model.ObjThumb{ Object: model.Object{ - Path: filepath.Join(fullPath, f.Name()), + Path: filePath, Name: f.Name(), Modified: f.ModTime(), Size: size, @@ -197,7 +198,7 @@ func (d *Local) Get(ctx context.Context, path string) (model.Obj, error) { } return nil, err } - isFolder := f.IsDir() || isSymlinkDir(f, path) + isFolder := f.IsDir() || isLinkedDir(f, path) size := f.Size() if isFolder { size = 0 diff --git a/drivers/local/util.go b/drivers/local/util.go index 802f60cf627..b9df717fb40 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -7,6 +7,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -18,14 +19,18 @@ import ( ffmpeg "github.com/u2takey/ffmpeg-go" ) -func isSymlinkDir(f fs.FileInfo, path string) bool { - if f.Mode()&os.ModeSymlink == os.ModeSymlink { - dst, err := os.Readlink(filepath.Join(path, f.Name())) +func isLinkedDir(f fs.FileInfo, path string) bool { + if f.Mode()&os.ModeSymlink == os.ModeSymlink || (runtime.GOOS == "windows" && f.Mode()&os.ModeIrregular != 0) { + dst, err := os.Readlink(path) if err != nil { return false } if !filepath.IsAbs(dst) { - dst = filepath.Join(path, dst) + dst = filepath.Join(filepath.Dir(path), dst) + } + dst, err = filepath.Abs(dst) + if err != nil { + return false } stat, err := os.Stat(dst) if err != nil { From d17889bf8e592192eb321aa2bd640c1bf97dfadf Mon Sep 17 00:00:00 2001 From: Chesyre <56254560+Chesyre@users.noreply.github.com> Date: Tue, 30 Sep 2025 08:16:28 +0200 Subject: [PATCH 049/133] feat(gofile): add configurable link expiration handling (#9329) * feat(driver): add Gofile storage driver Add support for Gofile.io cloud storage service with full CRUD operations. Features: - File and folder listing - Upload and download functionality - Create, move, rename, copy, and delete operations - Direct link generation for file access - API token authentication The driver implements all required driver interfaces and follows the existing driver patterns in the codebase. * feat(gofile): add configurable link expiration handling - Adjusts driver addition metadata to accept LinkExpiry and DirectLinkExpiry options for caching and API expiry control (drivers/gofile/meta.go:10). - Applies the new options when building file links, setting optional local cache expiration (drivers/gofile/driver.go:101) and sending an expireTime to the direct-link API (drivers/gofile/util.go:202). - Logs Gofile API error payloads and validates the structured error response before returning it (drivers/gofile/util.go:141). - Adds the required imports and returns the configured model.Link instance (drivers/gofile/driver.go:6). --- drivers/gofile/driver.go | 18 ++++++++++++++---- drivers/gofile/meta.go | 24 +++++++++++++----------- drivers/gofile/types.go | 2 +- drivers/gofile/util.go | 10 +++++++++- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/drivers/gofile/driver.go b/drivers/gofile/driver.go index 301eaef3405..8046bd163fb 100644 --- a/drivers/gofile/driver.go +++ b/drivers/gofile/driver.go @@ -3,6 +3,7 @@ package gofile import ( "context" "fmt" + "time" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -72,7 +73,7 @@ func (d *Gofile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } var objects []model.Obj - + // Process children or contents contents := response.Data.Children if contents == nil { @@ -97,9 +98,18 @@ func (d *Gofile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, fmt.Errorf("failed to create direct link: %w", err) } - return &model.Link{ + // Configure cache expiration based on user setting + link := &model.Link{ URL: directLink, - }, nil + } + + // Only set expiration if LinkExpiry > 0 (0 means no caching) + if d.LinkExpiry > 0 { + expiration := time.Duration(d.LinkExpiry) * 24 * time.Hour + link.Expiration = &expiration + } + + return link, nil } func (d *Gofile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { @@ -258,4 +268,4 @@ func (d *Gofile) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj return nil, errs.NotImplement } -var _ driver.Driver = (*Gofile)(nil) \ No newline at end of file +var _ driver.Driver = (*Gofile)(nil) diff --git a/drivers/gofile/meta.go b/drivers/gofile/meta.go index b8126e337b4..00656025770 100644 --- a/drivers/gofile/meta.go +++ b/drivers/gofile/meta.go @@ -1,26 +1,28 @@ package gofile import ( - "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" ) type Addition struct { - driver.RootID - APIToken string `json:"api_token" required:"true" help:"Get your API token from your Gofile profile page"` + driver.RootID + APIToken string `json:"api_token" required:"true" help:"Get your API token from your Gofile profile page"` + LinkExpiry int `json:"link_expiry" type:"number" default:"30" help:"Direct link cache duration in days. Set to 0 to disable caching"` + DirectLinkExpiry int `json:"direct_link_expiry" type:"number" default:"0" help:"Direct link expiration time in hours on Gofile server. Set to 0 for no expiration"` } var config = driver.Config{ - Name: "Gofile", - DefaultRoot: "", - LocalSort: false, - OnlyProxy: false, - NoCache: false, - NoUpload: false, + Name: "Gofile", + DefaultRoot: "", + LocalSort: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, } func init() { op.RegisterDriver(func() driver.Driver { return &Gofile{} }) -} \ No newline at end of file +} diff --git a/drivers/gofile/types.go b/drivers/gofile/types.go index 93c9f5d2e6a..be307347081 100644 --- a/drivers/gofile/types.go +++ b/drivers/gofile/types.go @@ -121,4 +121,4 @@ func (c *Content) ModifiedTime() time.Time { func (c *Content) IsDir() bool { return c.Type == "folder" -} \ No newline at end of file +} diff --git a/drivers/gofile/util.go b/drivers/gofile/util.go index 5f39dae5a81..1dd6229a773 100644 --- a/drivers/gofile/util.go +++ b/drivers/gofile/util.go @@ -10,10 +10,12 @@ import ( "net/http" "path/filepath" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + log "github.com/sirupsen/logrus" ) const ( @@ -137,9 +139,10 @@ func (d *Gofile) deleteJSON(ctx context.Context, endpoint string, data interface func (d *Gofile) handleError(resp *http.Response) error { body, _ := io.ReadAll(resp.Body) + log.Debugf("Gofile API error (HTTP %d): %s", resp.StatusCode, string(body)) var errorResp ErrorResponse - if err := json.Unmarshal(body, &errorResp); err == nil { + if err := json.Unmarshal(body, &errorResp); err == nil && errorResp.Status == "error" { return fmt.Errorf("gofile API error: %s (code: %s)", errorResp.Error.Message, errorResp.Error.Code) } @@ -199,6 +202,11 @@ func (d *Gofile) uploadFile(ctx context.Context, folderId string, file model.Fil func (d *Gofile) createDirectLink(ctx context.Context, contentId string) (string, error) { data := map[string]interface{}{} + if d.DirectLinkExpiry > 0 { + expireTime := time.Now().Add(time.Duration(d.DirectLinkExpiry) * time.Hour).Unix() + data["expireTime"] = expireTime + } + var result DirectLinkResponse err := d.postJSON(ctx, fmt.Sprintf("/contents/%s/directlinks", contentId), data, &result) if err != nil { From fe564c42dac0015e5c98baba67934209b2485be8 Mon Sep 17 00:00:00 2001 From: textrix Date: Tue, 30 Sep 2025 15:17:54 +0900 Subject: [PATCH 050/133] feat: add pCloud driver support (#9339) - Implement OAuth2 authentication with US/EU region support - Add file operations (list, upload, download, delete, rename, move, copy) - Add folder operations (create, rename, move, delete) - Enhance error handling with pCloud-specific retry logic - Use correct API methods: GET for reads, POST for writes - Implement direct upload approach for better performance - Add exponential backoff for failed requests with 4xxx/5xxx classification --- drivers/all.go | 1 + drivers/pcloud/driver.go | 189 +++++++++++++++++++++++++ drivers/pcloud/meta.go | 30 ++++ drivers/pcloud/types.go | 91 ++++++++++++ drivers/pcloud/util.go | 297 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 608 insertions(+) create mode 100644 drivers/pcloud/driver.go create mode 100644 drivers/pcloud/meta.go create mode 100644 drivers/pcloud/types.go create mode 100644 drivers/pcloud/util.go diff --git a/drivers/all.go b/drivers/all.go index 140908a8945..2ce0c2c6846 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -51,6 +51,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/onedrive" _ "github.com/alist-org/alist/v3/drivers/onedrive_app" _ "github.com/alist-org/alist/v3/drivers/onedrive_sharelink" + _ "github.com/alist-org/alist/v3/drivers/pcloud" _ "github.com/alist-org/alist/v3/drivers/pikpak" _ "github.com/alist-org/alist/v3/drivers/pikpak_share" _ "github.com/alist-org/alist/v3/drivers/quark_uc" diff --git a/drivers/pcloud/driver.go b/drivers/pcloud/driver.go new file mode 100644 index 00000000000..036dcc40e6c --- /dev/null +++ b/drivers/pcloud/driver.go @@ -0,0 +1,189 @@ +package pcloud + +import ( + "context" + "fmt" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type PCloud struct { + model.Storage + Addition + AccessToken string // Actual access token obtained from refresh token +} + +func (d *PCloud) Config() driver.Config { + return config +} + +func (d *PCloud) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *PCloud) Init(ctx context.Context) error { + // Map hostname selection to actual API endpoints + if d.Hostname == "us" { + d.Hostname = "api.pcloud.com" + } else if d.Hostname == "eu" { + d.Hostname = "eapi.pcloud.com" + } + + // Set default root folder ID if not provided + if d.RootFolderID == "" { + d.RootFolderID = "d0" + } + + // Use the access token directly (like rclone) + d.AccessToken = d.RefreshToken // RefreshToken field actually contains the access_token + return nil +} + +func (d *PCloud) Drop(ctx context.Context) error { + return nil +} + +func (d *PCloud) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := d.RootFolderID + if dir.GetID() != "" { + folderID = dir.GetID() + } + + files, err := d.getFiles(folderID) + if err != nil { + return nil, err + } + + return utils.SliceConvert(files, func(src FileObject) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *PCloud) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + downloadURL, err := d.getDownloadLink(file.GetID()) + if err != nil { + return nil, err + } + + return &model.Link{ + URL: downloadURL, + }, nil +} + +// Mkdir implements driver.Mkdir +func (d *PCloud) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + parentID := d.RootFolderID + if parentDir.GetID() != "" { + parentID = parentDir.GetID() + } + + return d.createFolder(parentID, dirName) +} + +// Move implements driver.Move +func (d *PCloud) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + // pCloud uses renamefile/renamefolder for both rename and move + endpoint := "/renamefile" + paramName := "fileid" + + if srcObj.IsDir() { + endpoint = "/renamefolder" + paramName = "folderid" + } + + var resp ItemResult + _, err := d.requestWithRetry(endpoint, "POST", func(req *resty.Request) { + req.SetFormData(map[string]string{ + paramName: extractID(srcObj.GetID()), + "tofolderid": extractID(dstDir.GetID()), + "toname": srcObj.GetName(), + }) + }, &resp) + + if err != nil { + return err + } + + if resp.Result != 0 { + return fmt.Errorf("pCloud error: result code %d", resp.Result) + } + + return nil +} + +// Rename implements driver.Rename +func (d *PCloud) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + endpoint := "/renamefile" + paramName := "fileid" + + if srcObj.IsDir() { + endpoint = "/renamefolder" + paramName = "folderid" + } + + var resp ItemResult + _, err := d.requestWithRetry(endpoint, "POST", func(req *resty.Request) { + req.SetFormData(map[string]string{ + paramName: extractID(srcObj.GetID()), + "toname": newName, + }) + }, &resp) + + if err != nil { + return err + } + + if resp.Result != 0 { + return fmt.Errorf("pCloud error: result code %d", resp.Result) + } + + return nil +} + +// Copy implements driver.Copy +func (d *PCloud) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + endpoint := "/copyfile" + paramName := "fileid" + + if srcObj.IsDir() { + endpoint = "/copyfolder" + paramName = "folderid" + } + + var resp ItemResult + _, err := d.requestWithRetry(endpoint, "POST", func(req *resty.Request) { + req.SetFormData(map[string]string{ + paramName: extractID(srcObj.GetID()), + "tofolderid": extractID(dstDir.GetID()), + "toname": srcObj.GetName(), + }) + }, &resp) + + if err != nil { + return err + } + + if resp.Result != 0 { + return fmt.Errorf("pCloud error: result code %d", resp.Result) + } + + return nil +} + +// Remove implements driver.Remove +func (d *PCloud) Remove(ctx context.Context, obj model.Obj) error { + return d.delete(obj.GetID(), obj.IsDir()) +} + +// Put implements driver.Put +func (d *PCloud) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + parentID := d.RootFolderID + if dstDir.GetID() != "" { + parentID = dstDir.GetID() + } + + return d.uploadFile(ctx, stream, parentID, stream.GetName(), stream.GetSize()) +} \ No newline at end of file diff --git a/drivers/pcloud/meta.go b/drivers/pcloud/meta.go new file mode 100644 index 00000000000..84e3dfe474f --- /dev/null +++ b/drivers/pcloud/meta.go @@ -0,0 +1,30 @@ +package pcloud + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Using json tag "access_token" for UI display, but internally it's a refresh token + RefreshToken string `json:"access_token" required:"true" help:"OAuth token from pCloud authorization"` + Hostname string `json:"hostname" type:"select" options:"us,eu" default:"us" help:"Select pCloud server region"` + RootFolderID string `json:"root_folder_id" help:"Get folder ID from URL like https://my.pcloud.com/#/filemanager?folder=12345678901 (leave empty for root folder)"` + ClientID string `json:"client_id" help:"Custom OAuth client ID (optional)"` + ClientSecret string `json:"client_secret" help:"Custom OAuth client secret (optional)"` +} + +// Implement IRootId interface +func (a Addition) GetRootId() string { + return a.RootFolderID +} + +var config = driver.Config{ + Name: "pCloud", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &PCloud{} + }) +} \ No newline at end of file diff --git a/drivers/pcloud/types.go b/drivers/pcloud/types.go new file mode 100644 index 00000000000..d0a6943c347 --- /dev/null +++ b/drivers/pcloud/types.go @@ -0,0 +1,91 @@ +package pcloud + +import ( + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +// ErrorResult represents a pCloud API error response +type ErrorResult struct { + Result int `json:"result"` + Error string `json:"error"` +} + +// TokenResponse represents OAuth token response +type TokenResponse struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` +} + +// ItemResult represents a common pCloud API response +type ItemResult struct { + Result int `json:"result"` + Metadata *FolderMeta `json:"metadata,omitempty"` +} + +// FolderMeta contains folder metadata including contents +type FolderMeta struct { + Contents []FileObject `json:"contents,omitempty"` +} + +// DownloadLinkResult represents download link response +type DownloadLinkResult struct { + Result int `json:"result"` + Hosts []string `json:"hosts"` + Path string `json:"path"` +} + +// FileObject represents a file or folder object in pCloud +type FileObject struct { + Name string `json:"name"` + Created string `json:"created"` // pCloud returns RFC1123 format string + Modified string `json:"modified"` // pCloud returns RFC1123 format string + IsFolder bool `json:"isfolder"` + FolderID uint64 `json:"folderid,omitempty"` + FileID uint64 `json:"fileid,omitempty"` + Size uint64 `json:"size"` + ParentID uint64 `json:"parentfolderid"` + Icon string `json:"icon,omitempty"` + Hash uint64 `json:"hash,omitempty"` + Category int `json:"category,omitempty"` + ID string `json:"id,omitempty"` +} + +// Convert FileObject to model.Obj +func fileToObj(f FileObject) model.Obj { + // Parse RFC1123 format time from pCloud + modTime, _ := time.Parse(time.RFC1123, f.Modified) + + obj := model.Object{ + Name: f.Name, + Size: int64(f.Size), + Modified: modTime, + IsFolder: f.IsFolder, + } + + if f.IsFolder { + obj.ID = "d" + strconv.FormatUint(f.FolderID, 10) + } else { + obj.ID = "f" + strconv.FormatUint(f.FileID, 10) + } + + return &obj +} + +// Extract numeric ID from string ID (remove 'd' or 'f' prefix) +func extractID(id string) string { + if len(id) > 1 && (id[0] == 'd' || id[0] == 'f') { + return id[1:] + } + return id +} + +// Get folder ID from path, return "0" for root +func getFolderID(path string) string { + if path == "/" || path == "" { + return "0" + } + return extractID(path) +} \ No newline at end of file diff --git a/drivers/pcloud/util.go b/drivers/pcloud/util.go new file mode 100644 index 00000000000..f2c1875e133 --- /dev/null +++ b/drivers/pcloud/util.go @@ -0,0 +1,297 @@ +package pcloud + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + defaultClientID = "DnONSzyJXpm" + defaultClientSecret = "VKEnd3ze4jsKFGg8TJiznwFG8" +) + +// Get API base URL +func (d *PCloud) getAPIURL() string { + return "https://" + d.Hostname +} + +// Get OAuth client credentials +func (d *PCloud) getClientCredentials() (string, string) { + clientID := d.ClientID + clientSecret := d.ClientSecret + + if clientID == "" { + clientID = defaultClientID + } + if clientSecret == "" { + clientSecret = defaultClientSecret + } + + return clientID, clientSecret +} + +// Refresh OAuth access token +func (d *PCloud) refreshToken() error { + clientID, clientSecret := d.getClientCredentials() + + var resp TokenResponse + _, err := base.RestyClient.R(). + SetFormData(map[string]string{ + "client_id": clientID, + "client_secret": clientSecret, + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + }). + SetResult(&resp). + Post(d.getAPIURL() + "/oauth2_token") + + if err != nil { + return err + } + + d.AccessToken = resp.AccessToken + return nil +} + +// shouldRetry determines if an error should be retried based on pCloud-specific logic +func (d *PCloud) shouldRetry(statusCode int, apiError *ErrorResult) bool { + // HTTP-level retry conditions + if statusCode == 429 || statusCode >= 500 { + return true + } + + // pCloud API-specific retry conditions (like rclone) + if apiError != nil && apiError.Result != 0 { + // 4xxx: rate limiting + if apiError.Result/1000 == 4 { + return true + } + // 5xxx: internal errors + if apiError.Result/1000 == 5 { + return true + } + } + + return false +} + +// requestWithRetry makes authenticated API request with retry logic +func (d *PCloud) requestWithRetry(endpoint string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + maxRetries := 3 + baseDelay := 500 * time.Millisecond + + for attempt := 0; attempt <= maxRetries; attempt++ { + body, err := d.request(endpoint, method, callback, resp) + if err == nil { + return body, nil + } + + // If this is the last attempt, return the error + if attempt == maxRetries { + return nil, err + } + + // Check if we should retry based on error type + if !d.shouldRetryError(err) { + return nil, err + } + + // Exponential backoff + delay := baseDelay * time.Duration(1< Date: Tue, 30 Sep 2025 00:18:58 -0600 Subject: [PATCH 051/133] feat(drivers): add ProtonDrive driver (#9331) - Implement complete ProtonDrive storage driver with end-to-end encryption support - Add authentication via username/password with credential caching and reusable login - Support all core operations: List, Link, Put, Copy, Move, Remove, Rename, MakeDir - Include encrypted file operations with PGP key management and node passphrase handling - Add temporary HTTP server for secure file downloads with range request support - Support media streaming using temp server range requests - Implement progress tracking for uploads and downloads - Support directory operations with circular move detection - Add proper error handling and panic recovery for external library integration Closes #9312 --- README.md | 1 + README_cn.md | 1 + README_ja.md | 1 + drivers/all.go | 1 + drivers/proton_drive/driver.go | 418 +++++++++++++++ drivers/proton_drive/meta.go | 69 +++ drivers/proton_drive/types.go | 124 +++++ drivers/proton_drive/util.go | 918 +++++++++++++++++++++++++++++++++ go.mod | 23 +- go.sum | 45 ++ 10 files changed, 1600 insertions(+), 1 deletion(-) create mode 100644 drivers/proton_drive/driver.go create mode 100644 drivers/proton_drive/meta.go create mode 100644 drivers/proton_drive/types.go create mode 100644 drivers/proton_drive/util.go diff --git a/README.md b/README.md index b77ed804751..032c2d17362 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) - [x] [BaiduNetdisk](http://pan.baidu.com/) diff --git a/README_cn.md b/README_cn.md index 757f5f8fb7a..cf1b1e1c29a 100644 --- a/README_cn.md +++ b/README_cn.md @@ -59,6 +59,7 @@ - [x] Teambition（[中国](https://www.teambition.com/ )，[国际](https://us.teambition.com/ )） - [x] [MediaFire](https://www.mediafire.com) - [x] [分秒帧](https://www.mediatrack.cn/) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [和彩云](https://yun.139.com/) (个人云, 家庭云，共享群组) - [x] [Yandex.Disk](https://disk.yandex.com/) - [x] [百度网盘](http://pan.baidu.com/) diff --git a/README_ja.md b/README_ja.md index e6a624b0929..a1a21253068 100644 --- a/README_ja.md +++ b/README_ja.md @@ -59,6 +59,7 @@ - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) - [x] [BaiduNetdisk](http://pan.baidu.com/) diff --git a/drivers/all.go b/drivers/all.go index 2ce0c2c6846..5c0f1ca04f4 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -54,6 +54,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/pcloud" _ "github.com/alist-org/alist/v3/drivers/pikpak" _ "github.com/alist-org/alist/v3/drivers/pikpak_share" + _ "github.com/alist-org/alist/v3/drivers/proton_drive" _ "github.com/alist-org/alist/v3/drivers/quark_uc" _ "github.com/alist-org/alist/v3/drivers/quark_uc_tv" _ "github.com/alist-org/alist/v3/drivers/quqi" diff --git a/drivers/proton_drive/driver.go b/drivers/proton_drive/driver.go new file mode 100644 index 00000000000..6ff8b069cb3 --- /dev/null +++ b/drivers/proton_drive/driver.go @@ -0,0 +1,418 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "sync" + "time" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + proton_api_bridge "github.com/henrybear327/Proton-API-Bridge" + "github.com/henrybear327/Proton-API-Bridge/common" + "github.com/henrybear327/go-proton-api" +) + +type ProtonDrive struct { + model.Storage + Addition + + protonDrive *proton_api_bridge.ProtonDrive + credentials *common.ProtonDriveCredential + + apiBase string + appVersion string + protonJson string + userAgent string + sdkVersion string + webDriveAV string + + tempServer *http.Server + tempServerPort int + downloadTokens map[string]*downloadInfo + tokenMutex sync.RWMutex + + c *proton.Client + //m *proton.Manager + + credentialCacheFile string + + //userKR *crypto.KeyRing + addrKRs map[string]*crypto.KeyRing + addrData map[string]proton.Address + + MainShare *proton.Share + RootLink *proton.Link + + DefaultAddrKR *crypto.KeyRing + MainShareKR *crypto.KeyRing +} + +func (d *ProtonDrive) Config() driver.Config { + return config +} + +func (d *ProtonDrive) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ProtonDrive) Init(ctx context.Context) error { + + defer func() { + if r := recover(); r != nil { + fmt.Printf("ProtonDrive initialization panic: %v", r) + } + }() + + if d.Username == "" { + return fmt.Errorf("username is required") + } + if d.Password == "" { + return fmt.Errorf("password is required") + } + + //fmt.Printf("ProtonDrive Init: Username=%s, TwoFACode=%s", d.Username, d.TwoFACode) + + if ctx == nil { + return fmt.Errorf("context cannot be nil") + } + + cachedCredentials, err := d.loadCachedCredentials() + useReusableLogin := false + var reusableCredential *common.ReusableCredentialData + + if err == nil && cachedCredentials != nil && + cachedCredentials.UID != "" && cachedCredentials.AccessToken != "" && + cachedCredentials.RefreshToken != "" && cachedCredentials.SaltedKeyPass != "" { + useReusableLogin = true + reusableCredential = cachedCredentials + } else { + useReusableLogin = false + reusableCredential = &common.ReusableCredentialData{} + } + + config := &common.Config{ + AppVersion: d.appVersion, + UserAgent: d.userAgent, + FirstLoginCredential: &common.FirstLoginCredentialData{ + Username: d.Username, + Password: d.Password, + TwoFA: d.TwoFACode, + }, + EnableCaching: true, + ConcurrentBlockUploadCount: 5, + ConcurrentFileCryptoCount: 2, + UseReusableLogin: false, + ReplaceExistingDraft: true, + ReusableCredential: reusableCredential, + CredentialCacheFile: d.credentialCacheFile, + } + + if config.FirstLoginCredential == nil { + return fmt.Errorf("failed to create login credentials, FirstLoginCredential cannot be nil") + } + + //fmt.Printf("Calling NewProtonDrive...") + + protonDrive, credentials, err := proton_api_bridge.NewProtonDrive( + ctx, + config, + func(auth proton.Auth) {}, + func() {}, + ) + + if credentials == nil && !useReusableLogin { + return fmt.Errorf("failed to get credentials from NewProtonDrive") + } + + if err != nil { + return fmt.Errorf("failed to initialize ProtonDrive: %w", err) + } + + d.protonDrive = protonDrive + + var finalCredentials *common.ProtonDriveCredential + + if useReusableLogin { + + // For reusable login, create credentials from cached data + finalCredentials = &common.ProtonDriveCredential{ + UID: reusableCredential.UID, + AccessToken: reusableCredential.AccessToken, + RefreshToken: reusableCredential.RefreshToken, + SaltedKeyPass: reusableCredential.SaltedKeyPass, + } + + d.credentials = finalCredentials + } else { + d.credentials = credentials + } + + clientOptions := []proton.Option{ + proton.WithAppVersion(d.appVersion), + proton.WithUserAgent(d.userAgent), + } + manager := proton.New(clientOptions...) + d.c = manager.NewClient(d.credentials.UID, d.credentials.AccessToken, d.credentials.RefreshToken) + + saltedKeyPassBytes, err := base64.StdEncoding.DecodeString(d.credentials.SaltedKeyPass) + if err != nil { + return fmt.Errorf("failed to decode salted key pass: %w", err) + } + + _, addrKRs, addrs, _, err := getAccountKRs(ctx, d.c, nil, saltedKeyPassBytes) + if err != nil { + return fmt.Errorf("failed to get account keyrings: %w", err) + } + + d.MainShare = protonDrive.MainShare + d.RootLink = protonDrive.RootLink + d.MainShareKR = protonDrive.MainShareKR + d.DefaultAddrKR = protonDrive.DefaultAddrKR + d.addrKRs = addrKRs + d.addrData = addrs + + return nil +} + +func (d *ProtonDrive) Drop(ctx context.Context) error { + if d.tempServer != nil { + d.tempServer.Shutdown(ctx) + } + return nil +} + +func (d *ProtonDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var linkID string + + if dir.GetPath() == "/" { + linkID = d.protonDrive.RootLink.LinkID + } else { + + link, err := d.searchByPath(ctx, dir.GetPath(), true) + if err != nil { + return nil, err + } + linkID = link.LinkID + } + + entries, err := d.protonDrive.ListDirectory(ctx, linkID) + if err != nil { + return nil, fmt.Errorf("failed to list directory: %w", err) + } + + //fmt.Printf("Found %d entries for path %s\n", len(entries), dir.GetPath()) + //fmt.Printf("Found %d entries\n", len(entries)) + + if len(entries) == 0 { + emptySlice := []model.Obj{} + + //fmt.Printf("Returning empty slice (entries): %+v\n", emptySlice) + + return emptySlice, nil + } + + var objects []model.Obj + for _, entry := range entries { + obj := &model.Object{ + Name: entry.Name, + Size: entry.Link.Size, + Modified: time.Unix(entry.Link.ModifyTime, 0), + IsFolder: entry.IsFolder, + } + objects = append(objects, obj) + } + + return objects, nil +} + +func (d *ProtonDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + link, err := d.searchByPath(ctx, file.GetPath(), false) + if err != nil { + return nil, err + } + + if err := d.ensureTempServer(); err != nil { + return nil, fmt.Errorf("failed to start temp server: %w", err) + } + + token := d.generateDownloadToken(link.LinkID, file.GetName()) + + /* return &model.Link{ + URL: fmt.Sprintf("protondrive://download/%s", link.LinkID), + }, nil */ + + return &model.Link{ + URL: fmt.Sprintf("http://localhost:%d/temp/%s", d.tempServerPort, token), + }, nil +} + +func (d *ProtonDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var parentLinkID string + + if parentDir.GetPath() == "/" { + parentLinkID = d.protonDrive.RootLink.LinkID + } else { + link, err := d.searchByPath(ctx, parentDir.GetPath(), true) + if err != nil { + return nil, err + } + parentLinkID = link.LinkID + } + + _, err := d.protonDrive.CreateNewFolderByID(ctx, parentLinkID, dirName) + if err != nil { + return nil, fmt.Errorf("failed to create directory: %w", err) + } + + newDir := &model.Object{ + Name: dirName, + IsFolder: true, + Modified: time.Now(), + } + return newDir, nil +} + +func (d *ProtonDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.DirectMove(ctx, srcObj, dstDir) +} + +func (d *ProtonDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + return d.DirectRename(ctx, srcObj, newName) +} + +func (d *ProtonDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, fmt.Errorf("directory copy not supported") + } + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), false) + if err != nil { + return nil, err + } + + reader, linkSize, fileSystemAttrs, err := d.protonDrive.DownloadFile(ctx, srcLink, 0) + if err != nil { + return nil, fmt.Errorf("failed to download source file: %w", err) + } + defer reader.Close() + + actualSize := linkSize + if fileSystemAttrs != nil && fileSystemAttrs.Size > 0 { + actualSize = fileSystemAttrs.Size + } + + tempFile, err := utils.CreateTempFile(reader, actualSize) + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + defer tempFile.Close() + + updatedObj := &model.Object{ + Name: srcObj.GetName(), + // Use the accurate and real size + Size: actualSize, + Modified: srcObj.ModTime(), + IsFolder: false, + } + + return d.Put(ctx, dstDir, &fileStreamer{ + ReadCloser: tempFile, + obj: updatedObj, + }, nil) +} + +func (d *ProtonDrive) Remove(ctx context.Context, obj model.Obj) error { + link, err := d.searchByPath(ctx, obj.GetPath(), obj.IsDir()) + if err != nil { + return err + } + + if obj.IsDir() { + return d.protonDrive.MoveFolderToTrashByID(ctx, link.LinkID, false) + } else { + return d.protonDrive.MoveFileToTrashByID(ctx, link.LinkID) + } +} + +func (d *ProtonDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var parentLinkID string + + if dstDir.GetPath() == "/" { + parentLinkID = d.protonDrive.RootLink.LinkID + } else { + link, err := d.searchByPath(ctx, dstDir.GetPath(), true) + if err != nil { + return nil, err + } + parentLinkID = link.LinkID + } + + tempFile, err := utils.CreateTempFile(file, file.GetSize()) + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + defer tempFile.Close() + + err = d.uploadFile(ctx, parentLinkID, file.GetName(), tempFile, file.GetSize(), up) + if err != nil { + return nil, err + } + + uploadedObj := &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + Modified: file.ModTime(), + IsFolder: false, + } + return uploadedObj, nil +} + +func (d *ProtonDrive) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +var _ driver.Driver = (*ProtonDrive)(nil) diff --git a/drivers/proton_drive/meta.go b/drivers/proton_drive/meta.go new file mode 100644 index 00000000000..ed33a41a2f8 --- /dev/null +++ b/drivers/proton_drive/meta.go @@ -0,0 +1,69 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + //driver.RootID + + Username string `json:"username" required:"true" type:"string"` + Password string `json:"password" required:"true" type:"string"` + TwoFACode string `json:"two_fa_code,omitempty" type:"string"` +} + +type Config struct { + Name string `json:"name"` + LocalSort bool `json:"local_sort"` + OnlyLocal bool `json:"only_local"` + OnlyProxy bool `json:"only_proxy"` + NoCache bool `json:"no_cache"` + NoUpload bool `json:"no_upload"` + NeedMs bool `json:"need_ms"` + DefaultRoot string `json:"default_root"` +} + +var config = driver.Config{ + Name: "ProtonDrive", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ProtonDrive{ + apiBase: "https://drive.proton.me/api", + appVersion: "windows-drive@1.11.3+rclone+proton", + credentialCacheFile: ".prtcrd", + protonJson: "application/vnd.protonmail.v1+json", + sdkVersion: "js@0.3.0", + userAgent: "ProtonDrive/v1.70.0 (Windows NT 10.0.22000; Win64; x64)", + webDriveAV: "web-drive@5.2.0+0f69f7a8", + } + }) +} diff --git a/drivers/proton_drive/types.go b/drivers/proton_drive/types.go new file mode 100644 index 00000000000..37a9edc6c46 --- /dev/null +++ b/drivers/proton_drive/types.go @@ -0,0 +1,124 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "errors" + "io" + "os" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/henrybear327/go-proton-api" +) + +type ProtonFile struct { + *proton.Link + Name string + IsFolder bool +} + +func (p *ProtonFile) GetName() string { + return p.Name +} + +func (p *ProtonFile) GetSize() int64 { + return p.Link.Size +} + +func (p *ProtonFile) GetPath() string { + return p.Name +} + +func (p *ProtonFile) IsDir() bool { + return p.IsFolder +} + +func (p *ProtonFile) ModTime() time.Time { + return time.Unix(p.Link.ModifyTime, 0) +} + +func (p *ProtonFile) CreateTime() time.Time { + return time.Unix(p.Link.CreateTime, 0) +} + +type downloadInfo struct { + LinkID string + FileName string +} + +type fileStreamer struct { + io.ReadCloser + obj model.Obj +} + +func (fs *fileStreamer) GetMimetype() string { return "" } +func (fs *fileStreamer) NeedStore() bool { return false } +func (fs *fileStreamer) IsForceStreamUpload() bool { return false } +func (fs *fileStreamer) GetExist() model.Obj { return nil } +func (fs *fileStreamer) SetExist(model.Obj) {} +func (fs *fileStreamer) RangeRead(http_range.Range) (io.Reader, error) { + return nil, errors.New("not supported") +} +func (fs *fileStreamer) CacheFullInTempFile() (model.File, error) { + return nil, errors.New("not supported") +} +func (fs *fileStreamer) SetTmpFile(r *os.File) {} +func (fs *fileStreamer) GetFile() model.File { return nil } +func (fs *fileStreamer) GetName() string { return fs.obj.GetName() } +func (fs *fileStreamer) GetSize() int64 { return fs.obj.GetSize() } +func (fs *fileStreamer) GetPath() string { return fs.obj.GetPath() } +func (fs *fileStreamer) IsDir() bool { return fs.obj.IsDir() } +func (fs *fileStreamer) ModTime() time.Time { return fs.obj.ModTime() } +func (fs *fileStreamer) CreateTime() time.Time { return fs.obj.ModTime() } +func (fs *fileStreamer) GetHash() utils.HashInfo { return fs.obj.GetHash() } +func (fs *fileStreamer) GetID() string { return fs.obj.GetID() } + +type httpRange struct { + start, end int64 +} + +type MoveRequest struct { + ParentLinkID string `json:"ParentLinkID"` + NodePassphrase string `json:"NodePassphrase"` + NodePassphraseSignature *string `json:"NodePassphraseSignature"` + Name string `json:"Name"` + NameSignatureEmail string `json:"NameSignatureEmail"` + Hash string `json:"Hash"` + OriginalHash string `json:"OriginalHash"` + ContentHash *string `json:"ContentHash"` // Maybe null +} + +type progressReader struct { + reader io.Reader + total int64 + current int64 + callback driver.UpdateProgress +} + +type RenameRequest struct { + Name string `json:"Name"` // PGP encrypted name + NameSignatureEmail string `json:"NameSignatureEmail"` // User's signature email + Hash string `json:"Hash"` // New name hash + OriginalHash string `json:"OriginalHash"` // Current name hash +} + +type RenameResponse struct { + Code int `json:"Code"` +} diff --git a/drivers/proton_drive/util.go b/drivers/proton_drive/util.go new file mode 100644 index 00000000000..7bd52fcaa33 --- /dev/null +++ b/drivers/proton_drive/util.go @@ -0,0 +1,918 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/henrybear327/Proton-API-Bridge/common" + "github.com/henrybear327/go-proton-api" +) + +func (d *ProtonDrive) loadCachedCredentials() (*common.ReusableCredentialData, error) { + if d.credentialCacheFile == "" { + return nil, nil + } + + if _, err := os.Stat(d.credentialCacheFile); os.IsNotExist(err) { + return nil, nil + } + + data, err := os.ReadFile(d.credentialCacheFile) + if err != nil { + return nil, fmt.Errorf("failed to read credential cache file: %w", err) + } + + var credentials common.ReusableCredentialData + if err := json.Unmarshal(data, &credentials); err != nil { + return nil, fmt.Errorf("failed to parse cached credentials: %w", err) + } + + if credentials.UID == "" || credentials.AccessToken == "" || + credentials.RefreshToken == "" || credentials.SaltedKeyPass == "" { + return nil, fmt.Errorf("cached credentials are incomplete") + } + + return &credentials, nil +} + +func (d *ProtonDrive) searchByPath(ctx context.Context, fullPath string, isFolder bool) (*proton.Link, error) { + if fullPath == "/" { + return d.protonDrive.RootLink, nil + } + + cleanPath := strings.Trim(fullPath, "/") + pathParts := strings.Split(cleanPath, "/") + + currentLink := d.protonDrive.RootLink + + for i, part := range pathParts { + isLastPart := i == len(pathParts)-1 + searchForFolder := !isLastPart || isFolder + + entries, err := d.protonDrive.ListDirectory(ctx, currentLink.LinkID) + if err != nil { + return nil, fmt.Errorf("failed to list directory: %w", err) + + } + + found := false + for _, entry := range entries { + // entry.Name is already decrypted! + if entry.Name == part && entry.IsFolder == searchForFolder { + currentLink = entry.Link + found = true + break + } + } + + if !found { + return nil, fmt.Errorf("path not found: %s (looking for part: %s)", fullPath, part) + } + } + + return currentLink, nil +} + +func (pr *progressReader) Read(p []byte) (int, error) { + n, err := pr.reader.Read(p) + pr.current += int64(n) + + if pr.callback != nil { + percentage := float64(pr.current) / float64(pr.total) * 100 + pr.callback(percentage) + } + + return n, err +} + +func (d *ProtonDrive) uploadFile(ctx context.Context, parentLinkID, fileName string, file *os.File, size int64, up driver.UpdateProgress) error { + + fileInfo, err := file.Stat() + if err != nil { + return fmt.Errorf("failed to get file info: %w", err) + } + + _, err = d.protonDrive.GetLink(ctx, parentLinkID) + if err != nil { + return fmt.Errorf("failed to get parent link: %w", err) + } + + reader := &progressReader{ + reader: bufio.NewReader(file), + total: size, + current: 0, + callback: up, + } + + _, _, err = d.protonDrive.UploadFileByReader(ctx, parentLinkID, fileName, fileInfo.ModTime(), reader, 0) + if err != nil { + return fmt.Errorf("failed to upload file: %w", err) + } + + return nil +} + +func (d *ProtonDrive) ensureTempServer() error { + if d.tempServer != nil { + + // Already running + return nil + } + + listener, err := net.Listen("tcp", ":0") + if err != nil { + return err + } + d.tempServerPort = listener.Addr().(*net.TCPAddr).Port + + mux := http.NewServeMux() + mux.HandleFunc("/temp/", d.handleTempDownload) + + d.tempServer = &http.Server{ + Handler: mux, + } + + go func() { + d.tempServer.Serve(listener) + }() + + return nil +} + +func (d *ProtonDrive) handleTempDownload(w http.ResponseWriter, r *http.Request) { + token := strings.TrimPrefix(r.URL.Path, "/temp/") + + d.tokenMutex.RLock() + info, exists := d.downloadTokens[token] + d.tokenMutex.RUnlock() + + if !exists { + http.Error(w, "Invalid or expired token", http.StatusNotFound) + return + } + + link, err := d.protonDrive.GetLink(r.Context(), info.LinkID) + if err != nil { + http.Error(w, "Failed to get file link", http.StatusInternalServerError) + return + } + + // Get file size for range calculations + _, _, attrs, err := d.protonDrive.DownloadFile(r.Context(), link, 0) + if err != nil { + http.Error(w, "Failed to get file info", http.StatusInternalServerError) + return + } + + fileSize := attrs.Size + + rangeHeader := r.Header.Get("Range") + if rangeHeader != "" { + + // Parse range header like "bytes=0-1023" or "bytes=1024-" + ranges, err := parseRange(rangeHeader, fileSize) + if err != nil { + http.Error(w, "Invalid range", http.StatusRequestedRangeNotSatisfiable) + return + } + + if len(ranges) == 1 { + + // Single range request, small + start, end := ranges[0].start, ranges[0].end + contentLength := end - start + 1 + + // Start download from offset + reader, _, _, err := d.protonDrive.DownloadFile(r.Context(), link, start) + if err != nil { + http.Error(w, "Failed to start download", http.StatusInternalServerError) + return + } + defer reader.Close() + + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, fileSize)) + w.Header().Set("Content-Length", fmt.Sprintf("%d", contentLength)) + w.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(link.Name))) + + // Partial content... + // Setting fileName is more cosmetical here + //.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", link.Name)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", info.FileName)) + w.Header().Set("Accept-Ranges", "bytes") + + w.WriteHeader(http.StatusPartialContent) + + io.CopyN(w, reader, contentLength) + return + } + } + + // Full file download (non-range request) + reader, _, _, err := d.protonDrive.DownloadFile(r.Context(), link, 0) + if err != nil { + http.Error(w, "Failed to start download", http.StatusInternalServerError) + return + } + defer reader.Close() + + // Set headers for full content + w.Header().Set("Content-Length", fmt.Sprintf("%d", fileSize)) + w.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(link.Name))) + + // Setting fileName is needed since ProtonDrive fileName is more like a random string + //w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", link.Name)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", info.FileName)) + + w.Header().Set("Accept-Ranges", "bytes") + + // Stream the full file + io.Copy(w, reader) +} + +func (d *ProtonDrive) generateDownloadToken(linkID, fileName string) string { + token := fmt.Sprintf("%d_%s", time.Now().UnixNano(), linkID[:8]) + + d.tokenMutex.Lock() + if d.downloadTokens == nil { + d.downloadTokens = make(map[string]*downloadInfo) + } + + d.downloadTokens[token] = &downloadInfo{ + LinkID: linkID, + FileName: fileName, + } + + d.tokenMutex.Unlock() + + go func() { + + // Token expires in 1 hour + time.Sleep(1 * time.Hour) + d.tokenMutex.Lock() + + delete(d.downloadTokens, token) + d.tokenMutex.Unlock() + }() + + return token +} + +func parseRange(rangeHeader string, size int64) ([]httpRange, error) { + if !strings.HasPrefix(rangeHeader, "bytes=") { + return nil, fmt.Errorf("invalid range header") + } + + rangeSpec := strings.TrimPrefix(rangeHeader, "bytes=") + ranges := strings.Split(rangeSpec, ",") + + var result []httpRange + for _, r := range ranges { + r = strings.TrimSpace(r) + if strings.Contains(r, "-") { + parts := strings.Split(r, "-") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid range format") + } + + var start, end int64 + var err error + + if parts[0] == "" { + + // Suffix range (e.g., "-500") + if parts[1] == "" { + return nil, fmt.Errorf("invalid range format") + } + end = size - 1 + start, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, err + } + start = size - start + if start < 0 { + start = 0 + } + } else if parts[1] == "" { + + // Prefix range (e.g., "500-") + start, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, err + } + end = size - 1 + } else { + // Full range (e.g., "0-1023") + start, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, err + } + end, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, err + } + } + + if start >= size || end >= size || start > end { + return nil, fmt.Errorf("range out of bounds") + } + + result = append(result, httpRange{start: start, end: end}) + } + } + + return result, nil +} + +func (d *ProtonDrive) encryptFileName(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Temporary file (request) + tempReq := proton.CreateFileReq{ + SignatureAddress: d.MainShare.Creator, + } + + // Encrypt the filename + err = tempReq.SetName(name, d.DefaultAddrKR, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to encrypt filename: %w", err) + } + + return tempReq.Name, nil +} + +func (d *ProtonDrive) generateFileNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func (d *ProtonDrive) getOriginalNameHash(link *proton.Link) (string, error) { + if link == nil { + return "", fmt.Errorf("link cannot be nil") + } + + if link.Hash == "" { + return "", fmt.Errorf("link hash is empty") + } + + return link.Hash, nil +} + +func (d *ProtonDrive) getLink(ctx context.Context, linkID string) (*proton.Link, error) { + if linkID == "" { + return nil, fmt.Errorf("linkID cannot be empty") + } + + link, err := d.c.GetLink(ctx, d.MainShare.ShareID, linkID) + if err != nil { + return nil, err + } + + return &link, nil +} + +func (d *ProtonDrive) getLinkKR(ctx context.Context, link *proton.Link) (*crypto.KeyRing, error) { + if link == nil { + return nil, fmt.Errorf("link cannot be nil") + } + + // Root Link or Root Dir + if link.ParentLinkID == "" { + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + return link.GetKeyRing(d.MainShareKR, signatureVerificationKR) + } + + // Get parent keyring recursively + parentLink, err := d.getLink(ctx, link.ParentLinkID) + if err != nil { + return nil, err + } + + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return nil, err + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + + return link.GetKeyRing(parentNodeKR, signatureVerificationKR) +} + +var ( + ErrKeyPassOrSaltedKeyPassMustBeNotNil = errors.New("either keyPass or saltedKeyPass must be not nil") + ErrFailedToUnlockUserKeys = errors.New("failed to unlock user keys") +) + +func getAccountKRs(ctx context.Context, c *proton.Client, keyPass, saltedKeyPass []byte) (*crypto.KeyRing, map[string]*crypto.KeyRing, map[string]proton.Address, []byte, error) { + + user, err := c.GetUser(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("user %#v", user) + + addrsArr, err := c.GetAddresses(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("addr %#v", addr) + + if saltedKeyPass == nil { + if keyPass == nil { + return nil, nil, nil, nil, ErrKeyPassOrSaltedKeyPassMustBeNotNil + } + + // Due to limitations, salts are stored using cacheCredentialToFile + salts, err := c.GetSalts(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("salts %#v", salts) + + saltedKeyPass, err = salts.SaltForKey(keyPass, user.Keys.Primary().ID) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("saltedKeyPass ok") + } + + userKR, addrKRs, err := proton.Unlock(user, addrsArr, saltedKeyPass, nil) + if err != nil { + return nil, nil, nil, nil, err + + } else if userKR.CountDecryptionEntities() == 0 { + return nil, nil, nil, nil, ErrFailedToUnlockUserKeys + } + + addrs := make(map[string]proton.Address) + for _, addr := range addrsArr { + addrs[addr.Email] = addr + } + + return userKR, addrKRs, addrs, saltedKeyPass, nil +} + +func (d *ProtonDrive) getSignatureVerificationKeyring(emailAddresses []string, verificationAddrKRs ...*crypto.KeyRing) (*crypto.KeyRing, error) { + ret, err := crypto.NewKeyRing(nil) + if err != nil { + return nil, err + } + + for _, emailAddress := range emailAddresses { + if addr, ok := d.addrData[emailAddress]; ok { + if addrKR, exists := d.addrKRs[addr.ID]; exists { + err = d.addKeysFromKR(ret, addrKR) + if err != nil { + return nil, err + } + } + } + } + + for _, kr := range verificationAddrKRs { + err = d.addKeysFromKR(ret, kr) + if err != nil { + return nil, err + } + } + + if ret.CountEntities() == 0 { + return nil, fmt.Errorf("no keyring for signature verification") + } + + return ret, nil +} + +func (d *ProtonDrive) addKeysFromKR(kr *crypto.KeyRing, newKRs ...*crypto.KeyRing) error { + for i := range newKRs { + for _, key := range newKRs[i].GetKeys() { + err := kr.AddKey(key) + if err != nil { + return err + } + } + } + return nil +} + +func (d *ProtonDrive) DirectRename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + //fmt.Printf("DEBUG DirectRename: path=%s, newName=%s", srcObj.GetPath(), newName) + + if d.MainShare == nil || d.DefaultAddrKR == nil { + return nil, fmt.Errorf("missing required fields: MainShare=%v, DefaultAddrKR=%v", + d.MainShare != nil, d.DefaultAddrKR != nil) + } + + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), srcObj.IsDir()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + parentLinkID := srcLink.ParentLinkID + if parentLinkID == "" { + return nil, fmt.Errorf("cannot rename root folder") + } + + encryptedName, err := d.encryptFileName(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateFileNameHash(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + renameReq := RenameRequest{ + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + } + + err = d.executeRenameAPI(ctx, srcLink.LinkID, renameReq) + if err != nil { + return nil, fmt.Errorf("rename API call failed: %w", err) + } + + return &model.Object{ + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) executeRenameAPI(ctx context.Context, linkID string, req RenameRequest) error { + + renameURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/rename", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal rename request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", renameURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.credentials.UID) + httpReq.Header.Set("Authorization", "Bearer "+d.credentials.AccessToken) + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute rename request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("rename failed with status %d", resp.StatusCode) + } + + var renameResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&renameResp); err != nil { + return fmt.Errorf("failed to decode rename response: %w", err) + } + + if renameResp.Code != 1000 { + return fmt.Errorf("rename failed with code %d", renameResp.Code) + } + + return nil +} + +func (d *ProtonDrive) executeMoveAPI(ctx context.Context, linkID string, req MoveRequest) error { + //fmt.Printf("DEBUG Move Request - Name: %s\n", req.Name) + //fmt.Printf("DEBUG Move Request - Hash: %s\n", req.Hash) + //fmt.Printf("DEBUG Move Request - OriginalHash: %s\n", req.OriginalHash) + //fmt.Printf("DEBUG Move Request - ParentLinkID: %s\n", req.ParentLinkID) + + //fmt.Printf("DEBUG Move Request - Name length: %d\n", len(req.Name)) + //fmt.Printf("DEBUG Move Request - NameSignatureEmail: %s\n", req.NameSignatureEmail) + //fmt.Printf("DEBUG Move Request - ContentHash: %v\n", req.ContentHash) + //fmt.Printf("DEBUG Move Request - NodePassphrase length: %d\n", len(req.NodePassphrase)) + //fmt.Printf("DEBUG Move Request - NodePassphraseSignature length: %d\n", len(req.NodePassphraseSignature)) + + //fmt.Printf("DEBUG Move Request - SrcLinkID: %s\n", linkID) + //fmt.Printf("DEBUG Move Request - DstParentLinkID: %s\n", req.ParentLinkID) + //fmt.Printf("DEBUG Move Request - ShareID: %s\n", d.MainShare.ShareID) + + srcLink, _ := d.getLink(ctx, linkID) + if srcLink != nil && srcLink.ParentLinkID == req.ParentLinkID { + return fmt.Errorf("cannot move to same parent directory") + } + + moveURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/move", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal move request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", moveURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Authorization", "Bearer "+d.credentials.AccessToken) + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.credentials.UID) + httpReq.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute move request: %w", err) + } + defer resp.Body.Close() + + var moveResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&moveResp); err != nil { + return fmt.Errorf("failed to decode move response: %w", err) + } + + if moveResp.Code != 1000 { + return fmt.Errorf("move operation failed with code: %d", moveResp.Code) + } + + return nil +} + +func (d *ProtonDrive) DirectMove(ctx context.Context, srcObj model.Obj, dstDir model.Obj) (model.Obj, error) { + //fmt.Printf("DEBUG DirectMove: srcPath=%s, dstPath=%s", srcObj.GetPath(), dstDir.GetPath()) + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), srcObj.IsDir()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + var dstParentLinkID string + if dstDir.GetPath() == "/" { + dstParentLinkID = d.RootLink.LinkID + } else { + dstLink, err := d.searchByPath(ctx, dstDir.GetPath(), true) + if err != nil { + return nil, fmt.Errorf("failed to find destination: %w", err) + } + dstParentLinkID = dstLink.LinkID + } + + if srcObj.IsDir() { + + // Check if destination is a descendant of source + if err := d.checkCircularMove(ctx, srcLink.LinkID, dstParentLinkID); err != nil { + return nil, err + } + } + + // Encrypt the filename for the new location + encryptedName, err := d.encryptFileName(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateNameHash(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + // Re-encrypt node passphrase for new parent context + reencryptedPassphrase, err := d.reencryptNodePassphrase(ctx, srcLink, dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to re-encrypt node passphrase: %w", err) + } + + moveReq := MoveRequest{ + ParentLinkID: dstParentLinkID, + NodePassphrase: reencryptedPassphrase, + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + ContentHash: nil, + + // *** Causes rejection *** + /* NodePassphraseSignature: srcLink.NodePassphraseSignature, */ + } + + //fmt.Printf("DEBUG MoveRequest validation:\n") + //fmt.Printf(" Name length: %d\n", len(moveReq.Name)) + //fmt.Printf(" Hash: %s\n", moveReq.Hash) + //fmt.Printf(" OriginalHash: %s\n", moveReq.OriginalHash) + //fmt.Printf(" NodePassphrase length: %d\n", len(moveReq.NodePassphrase)) + /* fmt.Printf(" NodePassphraseSignature length: %d\n", len(moveReq.NodePassphraseSignature)) */ + //fmt.Printf(" NameSignatureEmail: %s\n", moveReq.NameSignatureEmail) + + err = d.executeMoveAPI(ctx, srcLink.LinkID, moveReq) + if err != nil { + return nil, fmt.Errorf("move API call failed: %w", err) + } + + return &model.Object{ + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) reencryptNodePassphrase(ctx context.Context, srcLink *proton.Link, dstParentLinkID string) (string, error) { + // Get source parent link with metadata + srcParentLink, err := d.getLink(ctx, srcLink.ParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get source parent link: %w", err) + } + + // Get source parent keyring using link object + srcParentKR, err := d.getLinkKR(ctx, srcParentLink) + if err != nil { + return "", fmt.Errorf("failed to get source parent keyring: %w", err) + } + + // Get destination parent link with metadata + dstParentLink, err := d.getLink(ctx, dstParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get destination parent link: %w", err) + } + + // Get destination parent keyring using link object + dstParentKR, err := d.getLinkKR(ctx, dstParentLink) + if err != nil { + return "", fmt.Errorf("failed to get destination parent keyring: %w", err) + } + + // Re-encrypt the node passphrase from source parent context to destination parent context + reencryptedPassphrase, err := reencryptKeyPacket(srcParentKR, dstParentKR, d.DefaultAddrKR, srcLink.NodePassphrase) + if err != nil { + return "", fmt.Errorf("failed to re-encrypt key packet: %w", err) + } + + return reencryptedPassphrase, nil +} + +func (d *ProtonDrive) generateNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Get signature verification keyring + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func reencryptKeyPacket(srcKR, dstKR, _ *crypto.KeyRing, passphrase string) (string, error) { // addrKR (3) + oldSplitMessage, err := crypto.NewPGPSplitMessageFromArmored(passphrase) + if err != nil { + return "", err + } + + sessionKey, err := srcKR.DecryptSessionKey(oldSplitMessage.KeyPacket) + if err != nil { + return "", err + } + + newKeyPacket, err := dstKR.EncryptSessionKey(sessionKey) + if err != nil { + return "", err + } + + newSplitMessage := crypto.NewPGPSplitMessage(newKeyPacket, oldSplitMessage.DataPacket) + + return newSplitMessage.GetArmored() +} + +func (d *ProtonDrive) checkCircularMove(ctx context.Context, srcLinkID, dstParentLinkID string) error { + currentLinkID := dstParentLinkID + + for currentLinkID != "" && currentLinkID != d.RootLink.LinkID { + if currentLinkID == srcLinkID { + return fmt.Errorf("cannot move folder into itself or its subfolder") + } + + currentLink, err := d.getLink(ctx, currentLinkID) + if err != nil { + return err + } + currentLinkID = currentLink.ParentLinkID + } + + return nil +} diff --git a/go.mod b/go.mod index 51f9beec443..5772806a7b4 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 + github.com/ProtonMail/gopenpgp/v2 v2.7.4 github.com/SheltonZhu/115driver v1.1.2 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 @@ -38,6 +39,8 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/hekmon/transmissionrpc/v3 v3.0.0 + github.com/henrybear327/Proton-API-Bridge v1.0.0 + github.com/henrybear327/go-proton-api v1.0.0 github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 @@ -81,7 +84,21 @@ require ( gorm.io/gorm v1.25.11 ) -require github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect +require ( + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect + github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect + github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect + github.com/ProtonMail/go-srp v0.0.7 // indirect + github.com/PuerkitoBio/goquery v1.8.1 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/bradenaw/juniper v0.15.2 // indirect + github.com/cronokirby/saferith v0.33.0 // indirect + github.com/emersion/go-message v0.18.0 // indirect + github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect + github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect + github.com/relvacode/iso8601 v1.3.0 // indirect +) require ( github.com/STARRY-S/zip v0.2.1 // indirect @@ -265,4 +282,8 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect ) +replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton-api v1.0.0 + +replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed + replace github.com/SheltonZhu/115driver => github.com/okatu-loli/115driver v1.1.2 diff --git a/go.sum b/go.sum index 1b088a7186e..4b21f881331 100644 --- a/go.sum +++ b/go.sum @@ -34,14 +34,33 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= +github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 h1:ikwCzeqoqN6wvBHOB9OI6dde/jbV7EoTMpUcxtYl5Po= github.com/KirCute/ftpserverlib-pasvportmap v1.25.0/go.mod h1:v0NgMtKDDi/6CM6r4P+daCljCW3eO9yS+Z+pZDTKo1E= github.com/KirCute/sftpd-alist v0.0.12 h1:GNVM5QLbQLAfXP4wGUlXFA2IO6fVek0n0IsGnOuISdg= github.com/KirCute/sftpd-alist v0.0.12/go.mod h1:2wNK7yyW2XfjyJq10OY6xB4COLac64hOwfV6clDJn6s= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= +github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo= +github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= +github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= +github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= +github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk= +github.com/ProtonMail/gopenpgp/v2 v2.7.4 h1:Vz/8+HViFFnf2A6XX8JOvZMrA6F5puwNvvF21O1mRlo= +github.com/ProtonMail/gopenpgp/v2 v2.7.4/go.mod h1:IhkNEDaxec6NyzSI0PlxapinnwPVIESk8/76da3Ct3g= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= @@ -67,6 +86,9 @@ github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAP github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -132,6 +154,9 @@ github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bradenaw/juniper v0.15.2 h1:0JdjBGEF2jP1pOxmlNIrPhAoQN7Ng5IMAY5D0PHMW4U= +github.com/bradenaw/juniper v0.15.2/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= @@ -162,6 +187,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= @@ -197,6 +223,12 @@ github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj6 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= +github.com/emersion/go-message v0.18.0 h1:7LxAXHRpSeoO/Wom3ZApVZYG7c3d17yCScYce8WiXA8= +github.com/emersion/go-message v0.18.0/go.mod h1:Zi69ACvzaoV/MBnrxfVBPV3xWEuCmC2nEN39oJF4B8A= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= @@ -337,6 +369,10 @@ github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= github.com/hekmon/transmissionrpc/v3 v3.0.0/go.mod h1:38SlNhFzinVUuY87wGj3acOmRxeYZAZfrj6Re7UgCDg= +github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0LTKWQciUm8PMZb0= +github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= +github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= +github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -529,6 +565,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rclone/rclone v1.67.0 h1:yLRNgHEG2vQ60HCuzFqd0hYwKCRuWuvPUhvhMJ2jI5E= github.com/rclone/rclone v1.67.0/go.mod h1:Cb3Ar47M/SvwfhAjZTbVXdtrP/JLtPFCq2tkdtBVC6w= +github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= +github.com/relvacode/iso8601 v1.3.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -658,6 +696,8 @@ go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGX go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -735,6 +775,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -743,6 +784,7 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= @@ -791,6 +833,7 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -803,6 +846,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -819,6 +863,7 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= From 35d322443b1ba8b3fa9c2ed7421111808d86b6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sat, 11 Oct 2025 04:14:13 -0700 Subject: [PATCH 052/133] feat(driver): Add URL signing support (#9347) Introduces the ability to sign generated URLs for enhanced security and access control. This feature is activated by configuring a `PrivateKey`, `UID`, and `ValidDuration` in the driver settings. If a private key is provided, the driver will sign the output URLs, making them time-limited based on the `ValidDuration`. The `ValidDuration` defaults to 30 minutes if not specified. The core signing logic is encapsulated in the new `sign.go` file. The `driver.go` file integrates this signing process before returning the final URL. --- drivers/123_open/driver.go | 19 ++++++++++++++++++- drivers/123_open/meta.go | 7 +++++-- drivers/123_open/sign.go | 27 +++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 drivers/123_open/sign.go diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 39ed146eb03..ace86bb981a 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -11,6 +11,7 @@ import ( "github.com/go-resty/resty/v2" "net/http" "strconv" + "time" ) type Open123 struct { @@ -89,8 +90,24 @@ func (d *Open123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, fmt.Errorf("get link failed: %s", result.Message) } + linkURL := result.Data.URL + if d.PrivateKey != "" { + if d.UID == 0 { + return nil, fmt.Errorf("uid is required when private key is set") + } + duration := time.Duration(d.ValidDuration) + if duration <= 0 { + duration = 30 + } + signedURL, err := SignURL(linkURL, d.PrivateKey, d.UID, duration*time.Minute) + if err != nil { + return nil, err + } + linkURL = signedURL + } + return &model.Link{ - URL: result.Data.URL, + URL: linkURL, }, nil } diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go index d99bb75ba2c..d0b117aa7c0 100644 --- a/drivers/123_open/meta.go +++ b/drivers/123_open/meta.go @@ -8,8 +8,11 @@ import ( type Addition struct { driver.RootID - ClientID string `json:"client_id" required:"true" label:"Client ID"` - ClientSecret string `json:"client_secret" required:"true" label:"Client Secret"` + ClientID string `json:"client_id" required:"true" label:"Client ID"` + ClientSecret string `json:"client_secret" required:"true" label:"Client Secret"` + PrivateKey string `json:"private_key"` + UID uint64 `json:"uid" type:"number"` + ValidDuration int64 `json:"valid_duration" type:"number" default:"30" help:"minutes"` } var config = driver.Config{ diff --git a/drivers/123_open/sign.go b/drivers/123_open/sign.go new file mode 100644 index 00000000000..549d7ad8fdf --- /dev/null +++ b/drivers/123_open/sign.go @@ -0,0 +1,27 @@ +package _123Open + +import ( + "crypto/md5" + "fmt" + "math/rand" + "net/url" + "time" +) + +func SignURL(originURL, privateKey string, uid uint64, validDuration time.Duration) (string, error) { + if privateKey == "" { + return originURL, nil + } + parsed, err := url.Parse(originURL) + if err != nil { + return "", err + } + ts := time.Now().Add(validDuration).Unix() + randInt := rand.Int() + signature := fmt.Sprintf("%d-%d-%d-%x", ts, randInt, uid, md5.Sum([]byte(fmt.Sprintf("%s-%d-%d-%d-%s", + parsed.Path, ts, randInt, uid, privateKey)))) + query := parsed.Query() + query.Add("auth_key", signature) + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} From a6bd90a9b2c3b595990400ab061e4ab3d6720378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 16 Oct 2025 02:22:54 -0700 Subject: [PATCH 053/133] feat(driver/s3): Add OSS Archive Support (#9350) * feat(s3): Add support for S3 object storage classes Introduces a new 'storage_class' configuration option for S3 providers. Users can now specify the desired storage class (e.g., Standard, GLACIER, DEEP_ARCHIVE) for objects uploaded to S3-compatible services like AWS S3 and Tencent COS. The input storage class string is normalized to match AWS SDK constants, supporting various common aliases. If an unknown storage class is provided, it will be used as a raw value with a warning. This enhancement provides greater control over storage costs and data access patterns. * feat(storage): Support for displaying file storage classes Adds storage class information to file metadata and API responses. This change introduces the ability to store file storage classes in file metadata and display them in API responses. This allows users to view a file's storage tier (e.g., S3 Standard, Glacier), enhancing data management capabilities. Implementation details include: - Introducing the StorageClassProvider interface and the ObjWrapStorageClass structure to uniformly handle and communicate object storage class information. - Updated file metadata structures (e.g., ArchiveObj, FileInfo, RespFile) to include a StorageClass field. - Modified relevant API response functions (e.g., GetFileInfo, GetFileList) to populate and return storage classes. - Integrated functionality for retrieving object storage classes from underlying storage systems (e.g., S3) and wrapping them in lists. * feat(driver/s3): Added the "Other" interface and implemented it by the S3 driver. A new `driver.Other` interface has been added and defined in the `other.go` file. The S3 driver has been updated to implement this new interface, extending its functionality. * feat(s3): Add S3 object archive and thaw task management This commit introduces comprehensive support for S3 object archive and thaw operations, managed asynchronously through a new task system. - **S3 Transition Task System**: - Adds a new `S3Transition` task configuration, including workers, max retries, and persistence options. - Initializes `S3TransitionTaskManager` to handle asynchronous S3 archive/thaw requests. - Registers dedicated API routes for monitoring S3 transition tasks. - **Integrate S3 Archive/Thaw with Other API**: - Modifies the `Other` API handler to intercept `archive` and `thaw` methods for S3 storage drivers. - Dispatches these operations as `S3TransitionTask` instances to the task manager for background processing. - Returns a task ID to the client for tracking the status of the dispatched operation. - **Refactor `other` package for improved API consistency**: - Exports previously internal structs such as `archiveRequest`, `thawRequest`, `objectDescriptor`, `archiveResponse`, `thawResponse`, and `restoreStatus` by making their names public. - Makes helper functions like `decodeOtherArgs`, `normalizeStorageClass`, and `normalizeRestoreTier` public. - Introduces new constants for various S3 `Other` API methods. --- drivers/s3/driver.go | 36 +++- drivers/s3/meta.go | 1 + drivers/s3/other.go | 286 ++++++++++++++++++++++++++++++++ drivers/s3/util.go | 11 +- internal/bootstrap/task.go | 12 ++ internal/conf/config.go | 6 + internal/fs/other.go | 37 +++++ internal/fs/s3_transition.go | 310 +++++++++++++++++++++++++++++++++++ internal/model/obj.go | 25 +++ internal/model/object.go | 19 +++ server/handles/archive.go | 22 +-- server/handles/fsread.go | 106 ++++++------ server/handles/task.go | 1 + 13 files changed, 807 insertions(+), 65 deletions(-) create mode 100644 drivers/s3/other.go create mode 100644 internal/fs/s3_transition.go diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index b741148983e..7825ca6f935 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -15,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/server/common" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -32,6 +33,33 @@ type S3 struct { cron *cron.Cron } +var storageClassLookup = map[string]string{ + "standard": s3.ObjectStorageClassStandard, + "reduced_redundancy": s3.ObjectStorageClassReducedRedundancy, + "glacier": s3.ObjectStorageClassGlacier, + "standard_ia": s3.ObjectStorageClassStandardIa, + "onezone_ia": s3.ObjectStorageClassOnezoneIa, + "intelligent_tiering": s3.ObjectStorageClassIntelligentTiering, + "deep_archive": s3.ObjectStorageClassDeepArchive, + "outposts": s3.ObjectStorageClassOutposts, + "glacier_ir": s3.ObjectStorageClassGlacierIr, + "snow": s3.ObjectStorageClassSnow, + "express_onezone": s3.ObjectStorageClassExpressOnezone, +} + +func (d *S3) resolveStorageClass() *string { + value := strings.TrimSpace(d.StorageClass) + if value == "" { + return nil + } + normalized := strings.ToLower(strings.ReplaceAll(value, "-", "_")) + if v, ok := storageClassLookup[normalized]; ok { + return aws.String(v) + } + log.Warnf("s3: unknown storage class %q, using raw value", d.StorageClass) + return aws.String(value) +} + func (d *S3) Config() driver.Config { return d.config } @@ -179,8 +207,14 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up }), ContentType: &contentType, } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } _, err := uploader.UploadWithContext(ctx, input) return err } -var _ driver.Driver = (*S3)(nil) +var ( + _ driver.Driver = (*S3)(nil) + _ driver.Other = (*S3)(nil) +) diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 4de4b60a690..89d723b60b5 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -21,6 +21,7 @@ type Addition struct { ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` + StorageClass string `json:"storage_class" type:"select" options:",standard,standard_ia,onezone_ia,intelligent_tiering,glacier,glacier_ir,deep_archive,archive" help:"Storage class for new objects. AWS and Tencent COS support different subsets (COS uses ARCHIVE/DEEP_ARCHIVE)."` } func init() { diff --git a/drivers/s3/other.go b/drivers/s3/other.go new file mode 100644 index 00000000000..e299dae05d1 --- /dev/null +++ b/drivers/s3/other.go @@ -0,0 +1,286 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +const ( + OtherMethodArchive = "archive" + OtherMethodArchiveStatus = "archive_status" + OtherMethodThaw = "thaw" + OtherMethodThawStatus = "thaw_status" +) + +type ArchiveRequest struct { + StorageClass string `json:"storage_class"` +} + +type ThawRequest struct { + Days int64 `json:"days"` + Tier string `json:"tier"` +} + +type ObjectDescriptor struct { + Path string `json:"path"` + Bucket string `json:"bucket"` + Key string `json:"key"` +} + +type ArchiveResponse struct { + Action string `json:"action"` + Object ObjectDescriptor `json:"object"` + StorageClass string `json:"storage_class"` + RequestID string `json:"request_id,omitempty"` + VersionID string `json:"version_id,omitempty"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` +} + +type ThawResponse struct { + Action string `json:"action"` + Object ObjectDescriptor `json:"object"` + RequestID string `json:"request_id,omitempty"` + Status *RestoreStatus `json:"status,omitempty"` +} + +type RestoreStatus struct { + Ongoing bool `json:"ongoing"` + Expiry string `json:"expiry,omitempty"` + Raw string `json:"raw"` +} + +func (d *S3) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + if args.Obj == nil { + return nil, fmt.Errorf("missing object reference") + } + if args.Obj.IsDir() { + return nil, errs.NotSupport + } + + switch strings.ToLower(strings.TrimSpace(args.Method)) { + case "archive": + return d.archive(ctx, args) + case "archive_status": + return d.archiveStatus(ctx, args) + case "thaw": + return d.thaw(ctx, args) + case "thaw_status": + return d.thawStatus(ctx, args) + default: + return nil, errs.NotSupport + } +} + +func (d *S3) archive(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + payload := ArchiveRequest{} + if err := DecodeOtherArgs(args.Data, &payload); err != nil { + return nil, fmt.Errorf("parse archive request: %w", err) + } + if payload.StorageClass == "" { + return nil, fmt.Errorf("storage_class is required") + } + storageClass := NormalizeStorageClass(payload.StorageClass) + input := &s3.CopyObjectInput{ + Bucket: &d.Bucket, + Key: &key, + CopySource: aws.String(url.PathEscape(d.Bucket + "/" + key)), + MetadataDirective: aws.String(s3.MetadataDirectiveCopy), + StorageClass: aws.String(storageClass), + } + copyReq, output := d.client.CopyObjectRequest(input) + copyReq.SetContext(ctx) + if err := copyReq.Send(); err != nil { + return nil, err + } + + resp := ArchiveResponse{ + Action: "archive", + Object: d.describeObject(args.Obj, key), + StorageClass: storageClass, + RequestID: copyReq.RequestID, + } + if output.VersionId != nil { + resp.VersionID = aws.StringValue(output.VersionId) + } + if result := output.CopyObjectResult; result != nil { + resp.ETag = aws.StringValue(result.ETag) + if result.LastModified != nil { + resp.LastModified = result.LastModified.UTC().Format(time.RFC3339) + } + } + if status, err := d.describeObjectStatus(ctx, key); err == nil { + if status.StorageClass != "" { + resp.StorageClass = status.StorageClass + } + } + return resp, nil +} + +func (d *S3) archiveStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + status, err := d.describeObjectStatus(ctx, key) + if err != nil { + return nil, err + } + return ArchiveResponse{ + Action: "archive_status", + Object: d.describeObject(args.Obj, key), + StorageClass: status.StorageClass, + }, nil +} + +func (d *S3) thaw(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + payload := ThawRequest{Days: 1} + if err := DecodeOtherArgs(args.Data, &payload); err != nil { + return nil, fmt.Errorf("parse thaw request: %w", err) + } + if payload.Days <= 0 { + payload.Days = 1 + } + restoreRequest := &s3.RestoreRequest{ + Days: aws.Int64(payload.Days), + } + if tier := NormalizeRestoreTier(payload.Tier); tier != "" { + restoreRequest.GlacierJobParameters = &s3.GlacierJobParameters{Tier: aws.String(tier)} + } + input := &s3.RestoreObjectInput{ + Bucket: &d.Bucket, + Key: &key, + RestoreRequest: restoreRequest, + } + restoreReq, _ := d.client.RestoreObjectRequest(input) + restoreReq.SetContext(ctx) + if err := restoreReq.Send(); err != nil { + return nil, err + } + status, _ := d.describeObjectStatus(ctx, key) + resp := ThawResponse{ + Action: "thaw", + Object: d.describeObject(args.Obj, key), + RequestID: restoreReq.RequestID, + } + if status != nil { + resp.Status = status.Restore + } + return resp, nil +} + +func (d *S3) thawStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + status, err := d.describeObjectStatus(ctx, key) + if err != nil { + return nil, err + } + return ThawResponse{ + Action: "thaw_status", + Object: d.describeObject(args.Obj, key), + Status: status.Restore, + }, nil +} + +func (d *S3) describeObject(obj model.Obj, key string) ObjectDescriptor { + return ObjectDescriptor{ + Path: obj.GetPath(), + Bucket: d.Bucket, + Key: key, + } +} + +type objectStatus struct { + StorageClass string + Restore *RestoreStatus +} + +func (d *S3) describeObjectStatus(ctx context.Context, key string) (*objectStatus, error) { + head, err := d.client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{Bucket: &d.Bucket, Key: &key}) + if err != nil { + return nil, err + } + status := &objectStatus{ + StorageClass: aws.StringValue(head.StorageClass), + Restore: parseRestoreHeader(head.Restore), + } + return status, nil +} + +func parseRestoreHeader(header *string) *RestoreStatus { + if header == nil { + return nil + } + value := strings.TrimSpace(*header) + if value == "" { + return nil + } + status := &RestoreStatus{Raw: value} + parts := strings.Split(value, ",") + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if strings.HasPrefix(part, "ongoing-request=") { + status.Ongoing = strings.Contains(part, "\"true\"") + } + if strings.HasPrefix(part, "expiry-date=") { + expiry := strings.Trim(part[len("expiry-date="):], "\"") + if expiry != "" { + if t, err := time.Parse(time.RFC1123, expiry); err == nil { + status.Expiry = t.UTC().Format(time.RFC3339) + } else { + status.Expiry = expiry + } + } + } + } + return status +} + +func DecodeOtherArgs(data interface{}, target interface{}) error { + if data == nil { + return nil + } + raw, err := json.Marshal(data) + if err != nil { + return err + } + return json.Unmarshal(raw, target) +} + +func NormalizeStorageClass(value string) string { + normalized := strings.ToLower(strings.TrimSpace(strings.ReplaceAll(value, "-", "_"))) + if normalized == "" { + return value + } + if v, ok := storageClassLookup[normalized]; ok { + return v + } + return value +} + +func NormalizeRestoreTier(value string) string { + normalized := strings.ToLower(strings.TrimSpace(value)) + switch normalized { + case "", "default": + return "" + case "bulk": + return s3.TierBulk + case "standard": + return s3.TierStandard + case "expedited": + return s3.TierExpedited + default: + return value + } +} diff --git a/drivers/s3/util.go b/drivers/s3/util.go index e02945a07d2..9d2b285ce1d 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -109,13 +109,13 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue } - file := model.Object{ + file := &model.Object{ //Id: *object.Key, Name: name, Size: *object.Size, Modified: *object.LastModified, } - files = append(files, &file) + files = append(files, model.WrapObjStorageClass(file, aws.StringValue(object.StorageClass))) } if listObjectsResult.IsTruncated == nil { return nil, errors.New("IsTruncated nil") @@ -164,13 +164,13 @@ func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue } - file := model.Object{ + file := &model.Object{ //Id: *object.Key, Name: name, Size: *object.Size, Modified: *object.LastModified, } - files = append(files, &file) + files = append(files, model.WrapObjStorageClass(file, aws.StringValue(object.StorageClass))) } if !aws.BoolValue(listObjectsResult.IsTruncated) { break @@ -202,6 +202,9 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error { CopySource: aws.String(url.PathEscape(d.Bucket + "/" + srcKey)), Key: &dstKey, } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } _, err := d.client.CopyObject(input) return err } diff --git a/internal/bootstrap/task.go b/internal/bootstrap/task.go index c67e3029b61..8f05b84a347 100644 --- a/internal/bootstrap/task.go +++ b/internal/bootstrap/task.go @@ -37,6 +37,18 @@ func InitTaskManager() { if len(tool.TransferTaskManager.GetAll()) == 0 { //prevent offline downloaded files from being deleted CleanTempDir() } + workers := conf.Conf.Tasks.S3Transition.Workers + if workers < 0 { + workers = 0 + } + fs.S3TransitionTaskManager = tache.NewManager[*fs.S3TransitionTask]( + tache.WithWorks(workers), + tache.WithPersistFunction( + db.GetTaskDataFunc("s3_transition", conf.Conf.Tasks.S3Transition.TaskPersistant), + db.UpdateTaskDataFunc("s3_transition", conf.Conf.Tasks.S3Transition.TaskPersistant), + ), + tache.WithMaxRetry(conf.Conf.Tasks.S3Transition.MaxRetry), + ) fs.ArchiveDownloadTaskManager = tache.NewManager[*fs.ArchiveDownloadTask](tache.WithWorks(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant), db.UpdateTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Decompress.MaxRetry)) op.RegisterSettingChangingCallback(func() { fs.ArchiveDownloadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers))) diff --git a/internal/conf/config.go b/internal/conf/config.go index cdb86fee3ad..cf7cde0bcf0 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -60,6 +60,7 @@ type TasksConfig struct { Copy TaskConfig `json:"copy" envPrefix:"COPY_"` Decompress TaskConfig `json:"decompress" envPrefix:"DECOMPRESS_"` DecompressUpload TaskConfig `json:"decompress_upload" envPrefix:"DECOMPRESS_UPLOAD_"` + S3Transition TaskConfig `json:"s3_transition" envPrefix:"S3_TRANSITION_"` AllowRetryCanceled bool `json:"allow_retry_canceled" env:"ALLOW_RETRY_CANCELED"` } @@ -184,6 +185,11 @@ func DefaultConfig() *Config { Workers: 5, MaxRetry: 2, }, + S3Transition: TaskConfig{ + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, + }, AllowRetryCanceled: false, }, Cors: Cors{ diff --git a/internal/fs/other.go b/internal/fs/other.go index 85b7b1d17bf..14f8f63d3ad 100644 --- a/internal/fs/other.go +++ b/internal/fs/other.go @@ -2,10 +2,15 @@ package fs import ( "context" + "encoding/json" + stdpath "path" + "strings" + "github.com/alist-org/alist/v3/drivers/s3" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" "github.com/pkg/errors" ) @@ -53,6 +58,38 @@ func other(ctx context.Context, args model.FsOtherArgs) (interface{}, error) { if err != nil { return nil, errors.WithMessage(err, "failed get storage") } + originalPath := args.Path + + if _, ok := storage.(*s3.S3); ok { + method := strings.ToLower(strings.TrimSpace(args.Method)) + if method == s3.OtherMethodArchive || method == s3.OtherMethodThaw { + if S3TransitionTaskManager == nil { + return nil, errors.New("s3 transition task manager is not initialized") + } + var payload json.RawMessage + if args.Data != nil { + raw, err := json.Marshal(args.Data) + if err != nil { + return nil, errors.WithMessage(err, "failed to encode request payload") + } + payload = raw + } + taskCreator, _ := ctx.Value("user").(*model.User) + tsk := &S3TransitionTask{ + TaskExtension: task.TaskExtension{Creator: taskCreator}, + status: "queued", + StorageMountPath: storage.GetStorage().MountPath, + ObjectPath: actualPath, + DisplayPath: originalPath, + ObjectName: stdpath.Base(actualPath), + Transition: method, + Payload: payload, + } + S3TransitionTaskManager.Add(tsk) + return map[string]string{"task_id": tsk.GetID()}, nil + } + } + args.Path = actualPath return op.Other(ctx, storage, args) } diff --git a/internal/fs/s3_transition.go b/internal/fs/s3_transition.go new file mode 100644 index 00000000000..395f3c5be8c --- /dev/null +++ b/internal/fs/s3_transition.go @@ -0,0 +1,310 @@ +package fs + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/s3" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" + "github.com/pkg/errors" + "github.com/xhofe/tache" +) + +const s3TransitionPollInterval = 15 * time.Second + +// S3TransitionTask represents an asynchronous S3 archive/thaw request that is +// tracked via the task manager so that clients can monitor the progress of the +// operation. +type S3TransitionTask struct { + task.TaskExtension + status string + + StorageMountPath string `json:"storage_mount_path"` + ObjectPath string `json:"object_path"` + DisplayPath string `json:"display_path"` + ObjectName string `json:"object_name"` + Transition string `json:"transition"` + Payload json.RawMessage `json:"payload,omitempty"` + + TargetStorageClass string `json:"target_storage_class,omitempty"` + RequestID string `json:"request_id,omitempty"` + VersionID string `json:"version_id,omitempty"` + + storage driver.Driver `json:"-"` +} + +// S3TransitionTaskManager holds asynchronous S3 archive/thaw tasks. +var S3TransitionTaskManager *tache.Manager[*S3TransitionTask] + +var _ task.TaskExtensionInfo = (*S3TransitionTask)(nil) + +func (t *S3TransitionTask) GetName() string { + action := strings.ToLower(t.Transition) + if action == "" { + action = "transition" + } + display := t.DisplayPath + if display == "" { + display = t.ObjectPath + } + if display == "" { + display = t.ObjectName + } + return fmt.Sprintf("s3 %s %s", action, display) +} + +func (t *S3TransitionTask) GetStatus() string { + return t.status +} + +func (t *S3TransitionTask) Run() error { + t.ReinitCtx() + t.ClearEndTime() + start := time.Now() + t.SetStartTime(start) + defer func() { t.SetEndTime(time.Now()) }() + + if err := t.ensureStorage(); err != nil { + t.status = fmt.Sprintf("locate storage failed: %v", err) + return err + } + + payload, err := t.decodePayload() + if err != nil { + t.status = fmt.Sprintf("decode payload failed: %v", err) + return err + } + + method := strings.ToLower(strings.TrimSpace(t.Transition)) + switch method { + case s3.OtherMethodArchive: + t.status = "submitting archive request" + t.SetProgress(0) + resp, err := op.Other(t.Ctx(), t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodArchive, + Data: payload, + }) + if err != nil { + t.status = fmt.Sprintf("archive request failed: %v", err) + return err + } + archiveResp, ok := toArchiveResponse(resp) + if ok { + if t.TargetStorageClass == "" { + t.TargetStorageClass = archiveResp.StorageClass + } + t.RequestID = archiveResp.RequestID + t.VersionID = archiveResp.VersionID + if archiveResp.StorageClass != "" { + t.status = fmt.Sprintf("archive requested, waiting for %s", archiveResp.StorageClass) + } else { + t.status = "archive requested" + } + } else if sc := t.extractTargetStorageClass(); sc != "" { + t.TargetStorageClass = sc + t.status = fmt.Sprintf("archive requested, waiting for %s", sc) + } else { + t.status = "archive requested" + } + if t.TargetStorageClass != "" { + t.TargetStorageClass = s3.NormalizeStorageClass(t.TargetStorageClass) + } + t.SetProgress(25) + return t.waitForArchive() + case s3.OtherMethodThaw: + t.status = "submitting thaw request" + t.SetProgress(0) + resp, err := op.Other(t.Ctx(), t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodThaw, + Data: payload, + }) + if err != nil { + t.status = fmt.Sprintf("thaw request failed: %v", err) + return err + } + thawResp, ok := toThawResponse(resp) + if ok { + t.RequestID = thawResp.RequestID + if thawResp.Status != nil && !thawResp.Status.Ongoing { + t.SetProgress(100) + t.status = thawCompletionMessage(thawResp.Status) + return nil + } + } + t.status = "thaw requested" + t.SetProgress(25) + return t.waitForThaw() + default: + return errors.Errorf("unsupported transition method: %s", t.Transition) + } +} + +func (t *S3TransitionTask) ensureStorage() error { + if t.storage != nil { + return nil + } + storage, err := op.GetStorageByMountPath(t.StorageMountPath) + if err != nil { + return err + } + t.storage = storage + return nil +} + +func (t *S3TransitionTask) decodePayload() (interface{}, error) { + if len(t.Payload) == 0 { + return nil, nil + } + var payload interface{} + if err := json.Unmarshal(t.Payload, &payload); err != nil { + return nil, err + } + return payload, nil +} + +func (t *S3TransitionTask) extractTargetStorageClass() string { + if len(t.Payload) == 0 { + return "" + } + var req s3.ArchiveRequest + if err := json.Unmarshal(t.Payload, &req); err != nil { + return "" + } + return s3.NormalizeStorageClass(req.StorageClass) +} + +func (t *S3TransitionTask) waitForArchive() error { + ticker := time.NewTicker(s3TransitionPollInterval) + defer ticker.Stop() + + ctx := t.Ctx() + for { + select { + case <-ctx.Done(): + t.status = "archive canceled" + return ctx.Err() + case <-ticker.C: + resp, err := op.Other(ctx, t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodArchiveStatus, + }) + if err != nil { + t.status = fmt.Sprintf("archive status error: %v", err) + return err + } + archiveResp, ok := toArchiveResponse(resp) + if !ok { + t.status = fmt.Sprintf("unexpected archive status response: %T", resp) + return errors.Errorf("unexpected archive status response: %T", resp) + } + currentClass := strings.TrimSpace(archiveResp.StorageClass) + target := strings.TrimSpace(t.TargetStorageClass) + if target == "" { + target = currentClass + t.TargetStorageClass = currentClass + } + if currentClass == "" { + t.status = "waiting for storage class update" + t.SetProgress(50) + continue + } + if strings.EqualFold(currentClass, target) { + t.SetProgress(100) + t.status = fmt.Sprintf("archive complete (%s)", currentClass) + return nil + } + t.status = fmt.Sprintf("storage class %s (target %s)", currentClass, target) + t.SetProgress(75) + } + } +} + +func (t *S3TransitionTask) waitForThaw() error { + ticker := time.NewTicker(s3TransitionPollInterval) + defer ticker.Stop() + + ctx := t.Ctx() + for { + select { + case <-ctx.Done(): + t.status = "thaw canceled" + return ctx.Err() + case <-ticker.C: + resp, err := op.Other(ctx, t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodThawStatus, + }) + if err != nil { + t.status = fmt.Sprintf("thaw status error: %v", err) + return err + } + thawResp, ok := toThawResponse(resp) + if !ok { + t.status = fmt.Sprintf("unexpected thaw status response: %T", resp) + return errors.Errorf("unexpected thaw status response: %T", resp) + } + status := thawResp.Status + if status == nil { + t.status = "waiting for thaw status" + t.SetProgress(50) + continue + } + if status.Ongoing { + t.status = fmt.Sprintf("thaw in progress (%s)", status.Raw) + t.SetProgress(75) + continue + } + t.SetProgress(100) + t.status = thawCompletionMessage(status) + return nil + } + } +} + +func thawCompletionMessage(status *s3.RestoreStatus) string { + if status == nil { + return "thaw complete" + } + if status.Expiry != "" { + return fmt.Sprintf("thaw complete, expires %s", status.Expiry) + } + return "thaw complete" +} + +func toArchiveResponse(v interface{}) (s3.ArchiveResponse, bool) { + switch resp := v.(type) { + case s3.ArchiveResponse: + return resp, true + case *s3.ArchiveResponse: + if resp != nil { + return *resp, true + } + } + return s3.ArchiveResponse{}, false +} + +func toThawResponse(v interface{}) (s3.ThawResponse, bool) { + switch resp := v.(type) { + case s3.ThawResponse: + return resp, true + case *s3.ThawResponse: + if resp != nil { + return *resp, true + } + } + return s3.ThawResponse{}, false +} + +// Ensure compatibility with persistence when tasks are restored. +func (t *S3TransitionTask) OnRestore() { + // The storage handle is not persisted intentionally; it will be lazily + // re-fetched on the next Run invocation. + t.storage = nil +} diff --git a/internal/model/obj.go b/internal/model/obj.go index 93fa7a96475..ed4e0451ec7 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -20,6 +20,10 @@ type ObjUnwrap interface { Unwrap() Obj } +type StorageClassProvider interface { + StorageClass() string +} + type Obj interface { GetSize() int64 GetName() string @@ -141,6 +145,13 @@ func WrapObjsName(objs []Obj) { } } +func WrapObjStorageClass(obj Obj, storageClass string) Obj { + if storageClass == "" { + return obj + } + return &ObjWrapStorageClass{Obj: obj, storageClass: storageClass} +} + func UnwrapObj(obj Obj) Obj { if unwrap, ok := obj.(ObjUnwrap); ok { obj = unwrap.Unwrap() @@ -168,6 +179,20 @@ func GetUrl(obj Obj) (url string, ok bool) { return url, false } +func GetStorageClass(obj Obj) (string, bool) { + if provider, ok := obj.(StorageClassProvider); ok { + value := provider.StorageClass() + if value == "" { + return "", false + } + return value, true + } + if unwrap, ok := obj.(ObjUnwrap); ok { + return GetStorageClass(unwrap.Unwrap()) + } + return "", false +} + func GetRawObject(obj Obj) *Object { switch v := obj.(type) { case *ObjThumbURL: diff --git a/internal/model/object.go b/internal/model/object.go index c8c10bb9d92..1617662cf9b 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -11,6 +11,11 @@ type ObjWrapName struct { Obj } +type ObjWrapStorageClass struct { + storageClass string + Obj +} + func (o *ObjWrapName) Unwrap() Obj { return o.Obj } @@ -19,6 +24,20 @@ func (o *ObjWrapName) GetName() string { return o.Name } +func (o *ObjWrapStorageClass) Unwrap() Obj { + return o.Obj +} + +func (o *ObjWrapStorageClass) StorageClass() string { + return o.storageClass +} + +func (o *ObjWrapStorageClass) SetPath(path string) { + if setter, ok := o.Obj.(SetPath); ok { + setter.SetPath(path) + } +} + type Object struct { ID string Path string diff --git a/server/handles/archive.go b/server/handles/archive.go index 0bb8d94a728..5787897cc90 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -44,17 +44,19 @@ type ArchiveContentResp struct { } func toObjsRespWithoutSignAndThumb(obj model.Obj) ObjResp { + storageClass, _ := model.GetStorageClass(obj) return ObjResp{ - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: "", - Thumb: "", - Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: "", + Thumb: "", + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + StorageClass: storageClass, } } diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 8cf3c9b0290..676d64b166e 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -33,18 +33,19 @@ type DirReq struct { } type ObjResp struct { - Id string `json:"id"` - Path string `json:"path"` - Name string `json:"name"` - Size int64 `json:"size"` - IsDir bool `json:"is_dir"` - Modified time.Time `json:"modified"` - Created time.Time `json:"created"` - Sign string `json:"sign"` - Thumb string `json:"thumb"` - Type int `json:"type"` - HashInfoStr string `json:"hashinfo"` - HashInfo map[*utils.HashType]string `json:"hash_info"` + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + StorageClass string `json:"storage_class,omitempty"` } type FsListResp struct { @@ -57,19 +58,20 @@ type FsListResp struct { } type ObjLabelResp struct { - Id string `json:"id"` - Path string `json:"path"` - Name string `json:"name"` - Size int64 `json:"size"` - IsDir bool `json:"is_dir"` - Modified time.Time `json:"modified"` - Created time.Time `json:"created"` - Sign string `json:"sign"` - Thumb string `json:"thumb"` - Type int `json:"type"` - HashInfoStr string `json:"hashinfo"` - HashInfo map[*utils.HashType]string `json:"hash_info"` - LabelList []model.Label `json:"label_list"` + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + LabelList []model.Label `json:"label_list"` + StorageClass string `json:"storage_class,omitempty"` } func FsList(c *gin.Context) { @@ -256,20 +258,22 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjLabelResp { labels = labelsByName[obj.GetName()] } thumb, _ := model.GetThumb(obj) + storageClass, _ := model.GetStorageClass(obj) resp = append(resp, ObjLabelResp{ - Id: obj.GetID(), - Path: obj.GetPath(), - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: common.Sign(obj, parent, encrypt), - Thumb: thumb, - Type: utils.GetObjType(obj.GetName(), obj.IsDir()), - LabelList: labels, + Id: obj.GetID(), + Path: obj.GetPath(), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: common.Sign(obj, parent, encrypt), + Thumb: thumb, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + LabelList: labels, + StorageClass: storageClass, }) } return resp @@ -374,20 +378,22 @@ func FsGet(c *gin.Context) { } parentMeta, _ := op.GetNearestMeta(parentPath) thumb, _ := model.GetThumb(obj) + storageClass, _ := model.GetStorageClass(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ - Id: obj.GetID(), - Path: obj.GetPath(), - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), - Type: utils.GetFileType(obj.GetName()), - Thumb: thumb, + Id: obj.GetID(), + Path: obj.GetPath(), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), + Type: utils.GetFileType(obj.GetName()), + Thumb: thumb, + StorageClass: storageClass, }, RawURL: rawURL, Readme: getReadme(meta, reqPath), diff --git a/server/handles/task.go b/server/handles/task.go index 6d49f9e5027..a4dbce0f1fa 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -220,6 +220,7 @@ func SetupTaskRoute(g *gin.RouterGroup) { taskRoute(g.Group("/copy"), fs.CopyTaskManager) taskRoute(g.Group("/offline_download"), tool.DownloadTaskManager) taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager) + taskRoute(g.Group("/s3_transition"), fs.S3TransitionTaskManager) taskRoute(g.Group("/decompress"), fs.ArchiveDownloadTaskManager) taskRoute(g.Group("/decompress_upload"), fs.ArchiveContentUploadTaskManager) } From e2016dd03174366442727f30cb62029fb330b9a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 16 Oct 2025 02:23:11 -0700 Subject: [PATCH 054/133] refactor(webdav): Use ResolvePath instead of JoinPath (#9344) - Changed the path concatenation method between `reqPath` and `src` and `dst` to use `ResolvePath` - Updated the implementation of path handling in multiple functions - Improved the consistency and reliability of path resolution --- server/webdav.go | 2 +- server/webdav/path.go | 22 ++++++++++++++++++++++ server/webdav/webdav.go | 20 ++++++++++---------- 3 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 server/webdav/path.go diff --git a/server/webdav.go b/server/webdav.go index e0980139e4f..ac520070686 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -113,7 +113,7 @@ func WebDAVAuth(c *gin.Context) { reqPath = "/" } reqPath, _ = url.PathUnescape(reqPath) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = webdav.ResolvePath(user, reqPath) if err != nil { c.Status(http.StatusForbidden) c.Abort() diff --git a/server/webdav/path.go b/server/webdav/path.go new file mode 100644 index 00000000000..9a18da9551f --- /dev/null +++ b/server/webdav/path.go @@ -0,0 +1,22 @@ +package webdav + +import ( + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// ResolvePath normalizes the provided raw path and resolves it against the user's base path +// before delegating to the user-aware JoinPath permission checks. +func ResolvePath(user *model.User, raw string) (string, error) { + cleaned := utils.FixAndCleanPath(raw) + basePath := utils.FixAndCleanPath(user.BasePath) + + if cleaned != "/" && basePath != "/" && !utils.IsSubPath(basePath, cleaned) { + cleaned = path.Join(basePath, strings.TrimPrefix(cleaned, "/")) + } + + return user.JoinPath(cleaned) +} diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 93211e8a77e..00c0471f743 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -194,7 +194,7 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status } ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -222,7 +222,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta // TODO: check locks for read-only access?? ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return http.StatusForbidden, err } @@ -282,7 +282,7 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -321,7 +321,7 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, // comments in http.checkEtag. ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return http.StatusForbidden, err } @@ -375,7 +375,7 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -439,11 +439,11 @@ func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request) (status ctx := r.Context() user := ctx.Value("user").(*model.User) - src, err = user.JoinPath(src) + src, err = ResolvePath(user, src) if err != nil { return 403, err } - dst, err = user.JoinPath(dst) + dst, err = ResolvePath(user, dst) if err != nil { return 403, err } @@ -540,7 +540,7 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus if err != nil { return status, err } - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -623,7 +623,7 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status userAgent := r.Header.Get("User-Agent") ctx = context.WithValue(ctx, "userAgent", userAgent) user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -801,7 +801,7 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } From 4c8401855c777e2a4f869b11a8fffbe400320323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 23 Oct 2025 09:29:33 -0700 Subject: [PATCH 055/133] feat: Add new driver bitqiu support (#9355) * feat(bitqiu): Add Bitqiu cloud drive support - Implement the new Bitqiu cloud drive. - Add core driver logic, metadata handling, and utility functions. - Register the Bitqiu driver for use. * feat(driver): Implement GetLink, CreateDir, and Move operations - Implement `GetLink` method to retrieve download links for files. - Implement `CreateDir` method to create new directories. - Implement `Move` method to relocate files and directories. - Add new API endpoints and data structures for download and directory creation responses. - Integrate retry logic with re-authentication for API calls in implemented methods. - Update HTTP request headers to include `x-requested-with`. * feat(bitqiu): Add rename, copy, and delete operations - Implement `Rename` operation with retry logic and API calls. - Implement `Copy` operation, including asynchronous handling, polling for completion, and status checks. - Implement `Remove` operation with retry logic and API calls. - Add new API endpoint URLs for rename, copy, and delete, and a new copy success code. - Introduce `AsyncManagerData`, `AsyncTask`, and `AsyncTaskInfo` types to support async copy status monitoring. - Add utility functions `updateObjectName` and `parentPathOf` for object manipulation. - Integrate login retry mechanism for all file operations. * feat(bitqiu-upload): Implement chunked file upload support - Implement multi-part chunked upload logic for the BitQiu service. - Introduce `UploadInitData` and `ChunkUploadResponse` structs for structured API communication. - Refactor the `Save` method to orchestrate initial upload, chunked data transfer, and finalization. - Add `uploadFileInChunks` function to handle sequential uploading of file parts. - Add `completeChunkUpload` function to finalize the chunked upload process on the server. - Ensure proper temporary file cleanup using `defer tmpFile.Close()`. * feat(driver): Implement automatic root folder ID retrieval - Add `userInfoURL` constant for fetching user information. - Implement `ensureRootFolderID` function to retrieve and set the driver's root folder ID if not already present. - Integrate `ensureRootFolderID` into the driver's `Init` process. - Define `UserInfoData` struct to parse the `rootDirId` from user information responses. * feat(client): Implement configurable user agent * Introduce a configurable `UserAgent` field in the client's settings. * Add a `userAgent()` method to retrieve the user agent, prioritizing the custom setting or using a predefined default. * Apply the determined user agent to all outbound HTTP requests made by the `BitQiu` client. --- drivers/all.go | 1 + drivers/bitqiu/driver.go | 767 +++++++++++++++++++++++++++++++++++++++ drivers/bitqiu/meta.go | 28 ++ drivers/bitqiu/types.go | 107 ++++++ drivers/bitqiu/util.go | 102 ++++++ 5 files changed, 1005 insertions(+) create mode 100644 drivers/bitqiu/driver.go create mode 100644 drivers/bitqiu/meta.go create mode 100644 drivers/bitqiu/types.go create mode 100644 drivers/bitqiu/util.go diff --git a/drivers/all.go b/drivers/all.go index 5c0f1ca04f4..efeb6f7716a 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -21,6 +21,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_netdisk" _ "github.com/alist-org/alist/v3/drivers/baidu_photo" _ "github.com/alist-org/alist/v3/drivers/baidu_share" + _ "github.com/alist-org/alist/v3/drivers/bitqiu" _ "github.com/alist-org/alist/v3/drivers/chaoxing" _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" diff --git a/drivers/bitqiu/driver.go b/drivers/bitqiu/driver.go new file mode 100644 index 00000000000..048377fee93 --- /dev/null +++ b/drivers/bitqiu/driver.go @@ -0,0 +1,767 @@ +package bitqiu + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http/cookiejar" + "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + streamPkg "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" +) + +const ( + baseURL = "https://pan.bitqiu.com" + loginURL = baseURL + "/loginServer/login" + userInfoURL = baseURL + "/user/getInfo" + listURL = baseURL + "/apiToken/cfi/fs/resources/pages" + uploadInitializeURL = baseURL + "/apiToken/cfi/fs/upload/v2/initialize" + uploadCompleteURL = baseURL + "/apiToken/cfi/fs/upload/v2/complete" + downloadURL = baseURL + "/download/getUrl" + createDirURL = baseURL + "/resource/create" + moveResourceURL = baseURL + "/resource/remove" + renameResourceURL = baseURL + "/resource/rename" + copyResourceURL = baseURL + "/apiToken/cfi/fs/async/copy" + copyManagerURL = baseURL + "/apiToken/cfi/fs/async/manager" + deleteResourceURL = baseURL + "/resource/delete" + + successCode = "10200" + uploadSuccessCode = "30010" + copySubmittedCode = "10300" + orgChannel = "default|default|default" +) + +const ( + copyPollInterval = time.Second + copyPollMaxAttempts = 60 + chunkSize = int64(1 << 20) +) + +const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36" + +type BitQiu struct { + model.Storage + Addition + + client *resty.Client + userID string +} + +func (d *BitQiu) Config() driver.Config { + return config +} + +func (d *BitQiu) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *BitQiu) Init(ctx context.Context) error { + if d.Addition.UserPlatform == "" { + d.Addition.UserPlatform = uuid.NewString() + op.MustSaveDriverStorage(d) + } + + if d.client == nil { + jar, err := cookiejar.New(nil) + if err != nil { + return err + } + d.client = base.NewRestyClient() + d.client.SetBaseURL(baseURL) + d.client.SetCookieJar(jar) + } + d.client.SetHeader("user-agent", d.userAgent()) + + return d.login(ctx) +} + +func (d *BitQiu) Drop(ctx context.Context) error { + d.client = nil + d.userID = "" + return nil +} + +func (d *BitQiu) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + parentID := d.resolveParentID(dir) + dirPath := "" + if dir != nil { + dirPath = dir.GetPath() + } + pageSize := d.pageSize() + orderType := d.orderType() + desc := d.orderDesc() + + var results []model.Obj + page := 1 + for { + form := map[string]string{ + "parentId": parentID, + "limit": strconv.Itoa(pageSize), + "orderType": orderType, + "desc": desc, + "model": "1", + "userId": d.userID, + "currentPage": strconv.Itoa(page), + "page": strconv.Itoa(page), + "org_channel": orgChannel, + } + var resp Response[ResourcePage] + if err := d.postForm(ctx, listURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != successCode { + if resp.Code == "10401" || resp.Code == "10404" { + if err := d.login(ctx); err != nil { + return nil, err + } + continue + } + return nil, fmt.Errorf("list failed: %s", resp.Message) + } + + objs, err := utils.SliceConvert(resp.Data.Data, func(item Resource) (model.Obj, error) { + return item.toObject(parentID, dirPath) + }) + if err != nil { + return nil, err + } + results = append(results, objs...) + + if !resp.Data.HasNext || len(resp.Data.Data) == 0 { + break + } + page++ + } + + return results, nil +} + +func (d *BitQiu) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + form := map[string]string{ + "fileIds": file.GetID(), + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[DownloadData] + if err := d.postForm(ctx, downloadURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + if resp.Data.URL == "" { + return nil, fmt.Errorf("empty download url returned") + } + return &model.Link{URL: resp.Data.URL}, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("get link failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("get link failed: retry limit reached") +} + +func (d *BitQiu) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + parentID := d.resolveParentID(parentDir) + parentPath := "" + if parentDir != nil { + parentPath = parentDir.GetPath() + } + form := map[string]string{ + "parentId": parentID, + "name": dirName, + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[CreateDirData] + if err := d.postForm(ctx, createDirURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + newParentID := parentID + if resp.Data.ParentID != "" { + newParentID = resp.Data.ParentID + } + name := resp.Data.Name + if name == "" { + name = dirName + } + resource := Resource{ + ResourceID: resp.Data.DirID, + ResourceType: 1, + Name: name, + ParentID: newParentID, + } + obj, err := resource.toObject(newParentID, parentPath) + if err != nil { + return nil, err + } + if o, ok := obj.(*Object); ok { + o.ParentID = newParentID + } + return obj, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("create folder failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("create folder failed: retry limit reached") +} + +func (d *BitQiu) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + targetParentID := d.resolveParentID(dstDir) + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "parentId": targetParentID, + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["dirIds"] = srcObj.GetID() + } else { + form["fileIds"] = srcObj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, moveResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + dstPath := "" + if dstDir != nil { + dstPath = dstDir.GetPath() + } + if setter, ok := srcObj.(model.SetPath); ok { + setter.SetPath(path.Join(dstPath, srcObj.GetName())) + } + if o, ok := srcObj.(*Object); ok { + o.ParentID = targetParentID + } + return srcObj, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("move failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("move failed: retry limit reached") +} + +func (d *BitQiu) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + form := map[string]string{ + "resourceId": srcObj.GetID(), + "name": newName, + "type": "0", + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["type"] = "1" + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, renameResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + return updateObjectName(srcObj, newName), nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("rename failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("rename failed: retry limit reached") +} + +func (d *BitQiu) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + targetParentID := d.resolveParentID(dstDir) + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "parentId": targetParentID, + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["dirIds"] = srcObj.GetID() + } else { + form["fileIds"] = srcObj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, copyResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode, copySubmittedCode: + return d.waitForCopiedObject(ctx, srcObj, dstDir) + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("copy failed: %s", resp.Message) + } + } + + return nil, fmt.Errorf("copy failed: retry limit reached") +} + +func (d *BitQiu) Remove(ctx context.Context, obj model.Obj) error { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return err + } + } + + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "org_channel": orgChannel, + } + if obj.IsDir() { + form["dirIds"] = obj.GetID() + } else { + form["fileIds"] = obj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, deleteResourceURL, form, &resp); err != nil { + return err + } + switch resp.Code { + case successCode: + return nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return err + } + default: + return fmt.Errorf("remove failed: %s", resp.Message) + } + } + return fmt.Errorf("remove failed: retry limit reached") +} + +func (d *BitQiu) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + up(0) + tmpFile, md5sum, err := streamPkg.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return nil, err + } + defer tmpFile.Close() + + parentID := d.resolveParentID(dstDir) + parentPath := "" + if dstDir != nil { + parentPath = dstDir.GetPath() + } + form := map[string]string{ + "parentId": parentID, + "name": file.GetName(), + "size": strconv.FormatInt(file.GetSize(), 10), + "hash": md5sum, + "sampleMd5": md5sum, + "org_channel": orgChannel, + } + var resp Response[json.RawMessage] + if err = d.postForm(ctx, uploadInitializeURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != uploadSuccessCode { + switch resp.Code { + case successCode: + var initData UploadInitData + if err := json.Unmarshal(resp.Data, &initData); err != nil { + return nil, fmt.Errorf("parse upload init response failed: %w", err) + } + serverCode, err := d.uploadFileInChunks(ctx, tmpFile, file.GetSize(), md5sum, initData, up) + if err != nil { + return nil, err + } + obj, err := d.completeChunkUpload(ctx, initData, parentID, parentPath, file.GetName(), file.GetSize(), md5sum, serverCode) + if err != nil { + return nil, err + } + up(100) + return obj, nil + default: + return nil, fmt.Errorf("upload failed: %s", resp.Message) + } + } + + var resource Resource + if err := json.Unmarshal(resp.Data, &resource); err != nil { + return nil, fmt.Errorf("parse upload response failed: %w", err) + } + obj, err := resource.toObject(parentID, parentPath) + if err != nil { + return nil, err + } + up(100) + return obj, nil +} + +func (d *BitQiu) uploadFileInChunks(ctx context.Context, tmpFile model.File, size int64, md5sum string, initData UploadInitData, up driver.UpdateProgress) (string, error) { + if d.client == nil { + return "", fmt.Errorf("client not initialized") + } + if size <= 0 { + return "", fmt.Errorf("invalid file size") + } + buf := make([]byte, chunkSize) + offset := int64(0) + var finishedFlag string + + for offset < size { + chunkLen := chunkSize + remaining := size - offset + if remaining < chunkLen { + chunkLen = remaining + } + + reader := io.NewSectionReader(tmpFile, offset, chunkLen) + chunkBuf := buf[:chunkLen] + if _, err := io.ReadFull(reader, chunkBuf); err != nil { + return "", fmt.Errorf("read chunk failed: %w", err) + } + + headers := map[string]string{ + "accept": "*/*", + "content-type": "application/octet-stream", + "appid": initData.AppID, + "token": initData.Token, + "userid": strconv.FormatInt(initData.UserID, 10), + "serialnumber": initData.SerialNumber, + "hash": md5sum, + "len": strconv.FormatInt(chunkLen, 10), + "offset": strconv.FormatInt(offset, 10), + "user-agent": d.userAgent(), + } + + var chunkResp ChunkUploadResponse + req := d.client.R(). + SetContext(ctx). + SetHeaders(headers). + SetBody(chunkBuf). + SetResult(&chunkResp) + + if _, err := req.Post(initData.UploadURL); err != nil { + return "", err + } + if chunkResp.ErrCode != 0 { + return "", fmt.Errorf("chunk upload failed with code %d", chunkResp.ErrCode) + } + finishedFlag = chunkResp.FinishedFlag + offset += chunkLen + up(float64(offset) * 100 / float64(size)) + } + + if finishedFlag == "" { + return "", fmt.Errorf("upload finished without server code") + } + return finishedFlag, nil +} + +func (d *BitQiu) completeChunkUpload(ctx context.Context, initData UploadInitData, parentID, parentPath, name string, size int64, md5sum, serverCode string) (model.Obj, error) { + form := map[string]string{ + "currentPage": "1", + "limit": "1", + "userId": strconv.FormatInt(initData.UserID, 10), + "status": "0", + "parentId": parentID, + "name": name, + "fileUid": initData.FileUID, + "fileSid": initData.FileSID, + "size": strconv.FormatInt(size, 10), + "serverCode": serverCode, + "snapTime": "", + "hash": md5sum, + "sampleMd5": md5sum, + "org_channel": orgChannel, + } + + var resp Response[Resource] + if err := d.postForm(ctx, uploadCompleteURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != successCode { + return nil, fmt.Errorf("complete upload failed: %s", resp.Message) + } + + return resp.Data.toObject(parentID, parentPath) +} + +func (d *BitQiu) login(ctx context.Context) error { + if d.client == nil { + return fmt.Errorf("client not initialized") + } + + form := map[string]string{ + "passport": d.Username, + "password": utils.GetMD5EncodeStr(d.Password), + "remember": "0", + "captcha": "", + "org_channel": orgChannel, + } + var resp Response[LoginData] + if err := d.postForm(ctx, loginURL, form, &resp); err != nil { + return err + } + if resp.Code != successCode { + return fmt.Errorf("login failed: %s", resp.Message) + } + d.userID = strconv.FormatInt(resp.Data.UserID, 10) + return d.ensureRootFolderID(ctx) +} + +func (d *BitQiu) ensureRootFolderID(ctx context.Context) error { + rootID := d.Addition.GetRootId() + if rootID != "" && rootID != "0" { + return nil + } + + form := map[string]string{ + "org_channel": orgChannel, + } + var resp Response[UserInfoData] + if err := d.postForm(ctx, userInfoURL, form, &resp); err != nil { + return err + } + if resp.Code != successCode { + return fmt.Errorf("get user info failed: %s", resp.Message) + } + if resp.Data.RootDirID == "" { + return fmt.Errorf("get user info failed: empty root dir id") + } + if d.Addition.RootFolderID != resp.Data.RootDirID { + d.Addition.RootFolderID = resp.Data.RootDirID + op.MustSaveDriverStorage(d) + } + return nil +} + +func (d *BitQiu) postForm(ctx context.Context, url string, form map[string]string, result interface{}) error { + if d.client == nil { + return fmt.Errorf("client not initialized") + } + req := d.client.R(). + SetContext(ctx). + SetHeaders(d.commonHeaders()). + SetFormData(form) + if result != nil { + req = req.SetResult(result) + } + _, err := req.Post(url) + return err +} + +func (d *BitQiu) waitForCopiedObject(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + expectedName := srcObj.GetName() + expectedIsDir := srcObj.IsDir() + var lastListErr error + + for attempt := 0; attempt < copyPollMaxAttempts; attempt++ { + if attempt > 0 { + if err := waitWithContext(ctx, copyPollInterval); err != nil { + return nil, err + } + } + + if err := d.checkCopyFailure(ctx); err != nil { + return nil, err + } + + obj, err := d.findObjectInDir(ctx, dstDir, expectedName, expectedIsDir) + if err != nil { + lastListErr = err + continue + } + if obj != nil { + return obj, nil + } + } + if lastListErr != nil { + return nil, lastListErr + } + return nil, fmt.Errorf("copy task timed out waiting for completion") +} + +func (d *BitQiu) checkCopyFailure(ctx context.Context) error { + form := map[string]string{ + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[AsyncManagerData] + if err := d.postForm(ctx, copyManagerURL, form, &resp); err != nil { + return err + } + switch resp.Code { + case successCode: + if len(resp.Data.FailTasks) > 0 { + return fmt.Errorf("copy failed: %s", resp.Data.FailTasks[0].ErrorMessage()) + } + return nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return err + } + default: + return fmt.Errorf("query copy status failed: %s", resp.Message) + } + } + return fmt.Errorf("query copy status failed: retry limit reached") +} + +func (d *BitQiu) findObjectInDir(ctx context.Context, dir model.Obj, name string, isDir bool) (model.Obj, error) { + objs, err := d.List(ctx, dir, model.ListArgs{}) + if err != nil { + return nil, err + } + for _, obj := range objs { + if obj.GetName() == name && obj.IsDir() == isDir { + return obj, nil + } + } + return nil, nil +} + +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (d *BitQiu) commonHeaders() map[string]string { + headers := map[string]string{ + "accept": "application/json, text/plain, */*", + "accept-language": "en-US,en;q=0.9", + "cache-control": "no-cache", + "pragma": "no-cache", + "user-platform": d.Addition.UserPlatform, + "x-kl-saas-ajax-request": "Ajax_Request", + "x-requested-with": "XMLHttpRequest", + "referer": baseURL + "/", + "origin": baseURL, + "user-agent": d.userAgent(), + } + return headers +} + +func (d *BitQiu) userAgent() string { + if ua := strings.TrimSpace(d.Addition.UserAgent); ua != "" { + return ua + } + return defaultUserAgent +} + +func (d *BitQiu) resolveParentID(dir model.Obj) string { + if dir != nil && dir.GetID() != "" { + return dir.GetID() + } + if root := d.Addition.GetRootId(); root != "" { + return root + } + return config.DefaultRoot +} + +func (d *BitQiu) pageSize() int { + if size, err := strconv.Atoi(d.Addition.PageSize); err == nil && size > 0 { + return size + } + return 24 +} + +func (d *BitQiu) orderType() string { + if d.Addition.OrderType != "" { + return d.Addition.OrderType + } + return "updateTime" +} + +func (d *BitQiu) orderDesc() string { + if d.Addition.OrderDesc { + return "1" + } + return "0" +} + +var _ driver.Driver = (*BitQiu)(nil) +var _ driver.PutResult = (*BitQiu)(nil) diff --git a/drivers/bitqiu/meta.go b/drivers/bitqiu/meta.go new file mode 100644 index 00000000000..63cb03344c4 --- /dev/null +++ b/drivers/bitqiu/meta.go @@ -0,0 +1,28 @@ +package bitqiu + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + UserPlatform string `json:"user_platform" help:"Optional device identifier; auto-generated if empty."` + OrderType string `json:"order_type" type:"select" options:"updateTime,createTime,name,size" default:"updateTime"` + OrderDesc bool `json:"order_desc"` + PageSize string `json:"page_size" default:"24" help:"Number of entries to request per page."` + UserAgent string `json:"user_agent" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"` +} + +var config = driver.Config{ + Name: "BitQiu", + DefaultRoot: "0", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &BitQiu{} + }) +} diff --git a/drivers/bitqiu/types.go b/drivers/bitqiu/types.go new file mode 100644 index 00000000000..8fbec989135 --- /dev/null +++ b/drivers/bitqiu/types.go @@ -0,0 +1,107 @@ +package bitqiu + +import "encoding/json" + +type Response[T any] struct { + Code string `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type LoginData struct { + UserID int64 `json:"userId"` +} + +type ResourcePage struct { + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` + TotalCount int `json:"totalCount"` + TotalPageCount int `json:"totalPageCount"` + Data []Resource `json:"data"` + HasNext bool `json:"hasNext"` +} + +type Resource struct { + ResourceID string `json:"resourceId"` + ResourceUID string `json:"resourceUid"` + ResourceType int `json:"resourceType"` + ParentID string `json:"parentId"` + Name string `json:"name"` + ExtName string `json:"extName"` + Size *json.Number `json:"size"` + CreateTime *string `json:"createTime"` + UpdateTime *string `json:"updateTime"` + FileMD5 string `json:"fileMd5"` +} + +type DownloadData struct { + URL string `json:"url"` + MD5 string `json:"md5"` + Size int64 `json:"size"` +} + +type UserInfoData struct { + RootDirID string `json:"rootDirId"` +} + +type CreateDirData struct { + DirID string `json:"dirId"` + Name string `json:"name"` + ParentID string `json:"parentId"` +} + +type AsyncManagerData struct { + WaitTasks []AsyncTask `json:"waitTaskList"` + RunningTasks []AsyncTask `json:"runningTaskList"` + SuccessTasks []AsyncTask `json:"successTaskList"` + FailTasks []AsyncTask `json:"failTaskList"` + TaskList []AsyncTask `json:"taskList"` +} + +type AsyncTask struct { + TaskID string `json:"taskId"` + Status int `json:"status"` + ErrorMsg string `json:"errorMsg"` + Message string `json:"message"` + Result *AsyncTaskInfo `json:"result"` + TargetName string `json:"targetName"` + TargetDirID string `json:"parentId"` +} + +type AsyncTaskInfo struct { + Resource Resource `json:"resource"` + DirID string `json:"dirId"` + FileID string `json:"fileId"` + Name string `json:"name"` + ParentID string `json:"parentId"` +} + +func (t AsyncTask) ErrorMessage() string { + if t.ErrorMsg != "" { + return t.ErrorMsg + } + if t.Message != "" { + return t.Message + } + return "unknown error" +} + +type UploadInitData struct { + Name string `json:"name"` + Size int64 `json:"size"` + Token string `json:"token"` + FileUID string `json:"fileUid"` + FileSID string `json:"fileSid"` + ParentID string `json:"parentId"` + UserID int64 `json:"userId"` + SerialNumber string `json:"serialNumber"` + UploadURL string `json:"uploadUrl"` + AppID string `json:"appId"` +} + +type ChunkUploadResponse struct { + ErrCode int `json:"errCode"` + Offset int64 `json:"offset"` + Finished int `json:"finished"` + FinishedFlag string `json:"finishedFlag"` +} diff --git a/drivers/bitqiu/util.go b/drivers/bitqiu/util.go new file mode 100644 index 00000000000..bccd6815e9b --- /dev/null +++ b/drivers/bitqiu/util.go @@ -0,0 +1,102 @@ +package bitqiu + +import ( + "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Object struct { + model.Object + ParentID string +} + +func (r Resource) toObject(parentID, parentPath string) (model.Obj, error) { + id := r.ResourceID + if id == "" { + id = r.ResourceUID + } + obj := &Object{ + Object: model.Object{ + ID: id, + Name: r.Name, + IsFolder: r.ResourceType == 1, + }, + ParentID: parentID, + } + if r.Size != nil { + if size, err := (*r.Size).Int64(); err == nil { + obj.Size = size + } + } + if ct := parseBitQiuTime(r.CreateTime); !ct.IsZero() { + obj.Ctime = ct + } + if mt := parseBitQiuTime(r.UpdateTime); !mt.IsZero() { + obj.Modified = mt + } + if r.FileMD5 != "" { + obj.HashInfo = utils.NewHashInfo(utils.MD5, strings.ToLower(r.FileMD5)) + } + obj.SetPath(path.Join(parentPath, obj.Name)) + return obj, nil +} + +func parseBitQiuTime(value *string) time.Time { + if value == nil { + return time.Time{} + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return time.Time{} + } + if ts, err := time.ParseInLocation("2006-01-02 15:04:05", trimmed, time.Local); err == nil { + return ts + } + return time.Time{} +} + +func updateObjectName(obj model.Obj, newName string) model.Obj { + newPath := path.Join(parentPathOf(obj.GetPath()), newName) + + switch o := obj.(type) { + case *Object: + o.Name = newName + o.Object.Name = newName + o.SetPath(newPath) + return o + case *model.Object: + o.Name = newName + o.SetPath(newPath) + return o + } + + if setter, ok := obj.(model.SetPath); ok { + setter.SetPath(newPath) + } + + return &model.Object{ + ID: obj.GetID(), + Path: newPath, + Name: newName, + Size: obj.GetSize(), + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), + } +} + +func parentPathOf(p string) string { + if p == "" { + return "" + } + dir := path.Dir(p) + if dir == "." { + return "" + } + return dir +} From b4d9beb49cba399842a54fcc33bc95a4a09b7bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 23 Oct 2025 09:31:15 -0700 Subject: [PATCH 056/133] fix(Mediatrack): Add support for X-Device-Fingerprint header (#9354) Introduce a `DeviceFingerprint` field to the request metadata. This field is used to conditionally set the `X-Device-Fingerprint` HTTP header in outgoing requests if its value is not empty. --- drivers/mediatrack/meta.go | 5 +++-- drivers/mediatrack/util.go | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/mediatrack/meta.go b/drivers/mediatrack/meta.go index 47f112c3573..ade8ae1c8fe 100644 --- a/drivers/mediatrack/meta.go +++ b/drivers/mediatrack/meta.go @@ -9,8 +9,9 @@ type Addition struct { AccessToken string `json:"access_token" required:"true"` ProjectID string `json:"project_id"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"updated_at,title,size" default:"title"` - OrderDesc bool `json:"order_desc"` + OrderBy string `json:"order_by" type:"select" options:"updated_at,title,size" default:"title"` + OrderDesc bool `json:"order_desc"` + DeviceFingerprint string `json:"device_fingerprint" required:"true"` } var config = driver.Config{ diff --git a/drivers/mediatrack/util.go b/drivers/mediatrack/util.go index 37ca0b3d09c..f5b751111a1 100644 --- a/drivers/mediatrack/util.go +++ b/drivers/mediatrack/util.go @@ -17,6 +17,9 @@ import ( func (d *MediaTrack) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) + if d.DeviceFingerprint != "" { + req.SetHeader("X-Device-Fingerprint", d.DeviceFingerprint) + } if callback != nil { callback(req) } From 0cbc7ebc92b2d64299ab6014be4c3feffeb1c967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 11 Nov 2025 20:25:26 +0800 Subject: [PATCH 057/133] feat(driver): Added support for Gitee driver (#9368) * feat(driver): Added support for Gitee driver - Implemented core driver functions including initialization, file listing, and file linking - Added Gitee-specific API interaction and object mapping - Registered Gitee driver in the driver registry * feat(driver): Added cookie-based authentication support for Gitee driver - Extended request handling to include `Cookie` header if provided - Updated metadata to include `cookie` field with appropriate documentation - Adjusted file link generation to propagate `Cookie` headers in requests --- drivers/all.go | 1 + drivers/gitee/driver.go | 224 ++++++++++++++++++++++++++++++++++++++++ drivers/gitee/meta.go | 29 ++++++ drivers/gitee/types.go | 60 +++++++++++ drivers/gitee/util.go | 44 ++++++++ 5 files changed, 358 insertions(+) create mode 100644 drivers/gitee/driver.go create mode 100644 drivers/gitee/meta.go create mode 100644 drivers/gitee/types.go create mode 100644 drivers/gitee/util.go diff --git a/drivers/all.go b/drivers/all.go index efeb6f7716a..3eb7e813bcc 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -31,6 +31,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" + _ "github.com/alist-org/alist/v3/drivers/gitee" _ "github.com/alist-org/alist/v3/drivers/github" _ "github.com/alist-org/alist/v3/drivers/github_releases" _ "github.com/alist-org/alist/v3/drivers/gofile" diff --git a/drivers/gitee/driver.go b/drivers/gitee/driver.go new file mode 100644 index 00000000000..78a400941b9 --- /dev/null +++ b/drivers/gitee/driver.go @@ -0,0 +1,224 @@ +package gitee + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type Gitee struct { + model.Storage + Addition + client *resty.Client +} + +func (d *Gitee) Config() driver.Config { + return config +} + +func (d *Gitee) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Gitee) Init(ctx context.Context) error { + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + d.Endpoint = strings.TrimSpace(d.Endpoint) + if d.Endpoint == "" { + d.Endpoint = "https://gitee.com/api/v5" + } + d.Endpoint = strings.TrimSuffix(d.Endpoint, "/") + d.Owner = strings.TrimSpace(d.Owner) + d.Repo = strings.TrimSpace(d.Repo) + d.Token = strings.TrimSpace(d.Token) + d.DownloadProxy = strings.TrimSpace(d.DownloadProxy) + if d.Owner == "" || d.Repo == "" { + return errors.New("owner and repo are required") + } + d.client = base.NewRestyClient(). + SetBaseURL(d.Endpoint). + SetHeader("Accept", "application/json") + repo, err := d.getRepo() + if err != nil { + return err + } + d.Ref = strings.TrimSpace(d.Ref) + if d.Ref == "" { + d.Ref = repo.DefaultBranch + } + return nil +} + +func (d *Gitee) Drop(ctx context.Context) error { + return nil +} + +func (d *Gitee) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + relPath := d.relativePath(dir.GetPath()) + contents, err := d.listContents(relPath) + if err != nil { + return nil, err + } + objs := make([]model.Obj, 0, len(contents)) + for i := range contents { + objs = append(objs, contents[i].toModelObj()) + } + return objs, nil +} + +func (d *Gitee) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var downloadURL string + if obj, ok := file.(*Object); ok { + downloadURL = obj.DownloadURL + if downloadURL == "" { + relPath := d.relativePath(file.GetPath()) + content, err := d.getContent(relPath) + if err != nil { + return nil, err + } + if content.DownloadURL == "" { + return nil, errors.New("empty download url") + } + obj.DownloadURL = content.DownloadURL + downloadURL = content.DownloadURL + } + } else { + relPath := d.relativePath(file.GetPath()) + content, err := d.getContent(relPath) + if err != nil { + return nil, err + } + if content.DownloadURL == "" { + return nil, errors.New("empty download url") + } + downloadURL = content.DownloadURL + } + url := d.applyProxy(downloadURL) + return &model.Link{ + URL: url, + Header: http.Header{ + "Cookie": {d.Cookie}, + }, + }, nil +} + +func (d *Gitee) newRequest() *resty.Request { + req := d.client.R() + if d.Token != "" { + req.SetQueryParam("access_token", d.Token) + } + if d.Ref != "" { + req.SetQueryParam("ref", d.Ref) + } + return req +} + +func (d *Gitee) apiPath(path string) string { + escapedOwner := url.PathEscape(d.Owner) + escapedRepo := url.PathEscape(d.Repo) + if path == "" { + return fmt.Sprintf("/repos/%s/%s/contents", escapedOwner, escapedRepo) + } + return fmt.Sprintf("/repos/%s/%s/contents/%s", escapedOwner, escapedRepo, encodePath(path)) +} + +func (d *Gitee) listContents(path string) ([]Content, error) { + res, err := d.newRequest().Get(d.apiPath(path)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var contents []Content + if err := utils.Json.Unmarshal(res.Body(), &contents); err != nil { + var single Content + if err2 := utils.Json.Unmarshal(res.Body(), &single); err2 == nil && single.Type != "" { + if single.Type != "dir" { + return nil, errs.NotFolder + } + return []Content{}, nil + } + return nil, err + } + for i := range contents { + contents[i].Path = joinPath(path, contents[i].Name) + } + return contents, nil +} + +func (d *Gitee) getContent(path string) (*Content, error) { + res, err := d.newRequest().Get(d.apiPath(path)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var content Content + if err := utils.Json.Unmarshal(res.Body(), &content); err != nil { + return nil, err + } + if content.Type == "" { + return nil, errors.New("invalid response") + } + if content.Path == "" { + content.Path = path + } + return &content, nil +} + +func (d *Gitee) relativePath(full string) string { + full = utils.FixAndCleanPath(full) + root := utils.FixAndCleanPath(d.RootFolderPath) + if root == "/" { + return strings.TrimPrefix(full, "/") + } + if utils.PathEqual(full, root) { + return "" + } + prefix := utils.PathAddSeparatorSuffix(root) + if strings.HasPrefix(full, prefix) { + return strings.TrimPrefix(full, prefix) + } + return strings.TrimPrefix(full, "/") +} + +func (d *Gitee) applyProxy(raw string) string { + if raw == "" || d.DownloadProxy == "" { + return raw + } + proxy := d.DownloadProxy + if !strings.HasSuffix(proxy, "/") { + proxy += "/" + } + return proxy + strings.TrimLeft(raw, "/") +} + +func encodePath(p string) string { + if p == "" { + return "" + } + parts := strings.Split(p, "/") + for i, part := range parts { + parts[i] = url.PathEscape(part) + } + return strings.Join(parts, "/") +} + +func joinPath(base, name string) string { + if base == "" { + return name + } + return strings.TrimPrefix(stdpath.Join(base, name), "./") +} diff --git a/drivers/gitee/meta.go b/drivers/gitee/meta.go new file mode 100644 index 00000000000..2f926d635f3 --- /dev/null +++ b/drivers/gitee/meta.go @@ -0,0 +1,29 @@ +package gitee + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Endpoint string `json:"endpoint" type:"string" help:"Gitee API endpoint, default https://gitee.com/api/v5"` + Token string `json:"token" type:"string"` + Owner string `json:"owner" type:"string" required:"true"` + Repo string `json:"repo" type:"string" required:"true"` + Ref string `json:"ref" type:"string" help:"Branch, tag or commit SHA, defaults to repository default branch"` + DownloadProxy string `json:"download_proxy" type:"string" help:"Prefix added before download URLs, e.g. https://mirror.example.com/"` + Cookie string `json:"cookie" type:"string" help:"Cookie returned from user info request"` +} + +var config = driver.Config{ + Name: "Gitee", + LocalSort: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Gitee{} + }) +} diff --git a/drivers/gitee/types.go b/drivers/gitee/types.go new file mode 100644 index 00000000000..c10536a5d10 --- /dev/null +++ b/drivers/gitee/types.go @@ -0,0 +1,60 @@ +package gitee + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Links struct { + Self string `json:"self"` + Html string `json:"html"` +} + +type Content struct { + Type string `json:"type"` + Size *int64 `json:"size"` + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + URL string `json:"url"` + HtmlURL string `json:"html_url"` + DownloadURL string `json:"download_url"` + Links Links `json:"_links"` +} + +func (c Content) toModelObj() model.Obj { + size := int64(0) + if c.Size != nil { + size = *c.Size + } + return &Object{ + Object: model.Object{ + ID: c.Path, + Name: c.Name, + Size: size, + Modified: time.Unix(0, 0), + IsFolder: c.Type == "dir", + }, + DownloadURL: c.DownloadURL, + HtmlURL: c.HtmlURL, + } +} + +type Object struct { + model.Object + DownloadURL string + HtmlURL string +} + +func (o *Object) URL() string { + return o.DownloadURL +} + +type Repo struct { + DefaultBranch string `json:"default_branch"` +} + +type ErrResp struct { + Message string `json:"message"` +} diff --git a/drivers/gitee/util.go b/drivers/gitee/util.go new file mode 100644 index 00000000000..fbef972ad3d --- /dev/null +++ b/drivers/gitee/util.go @@ -0,0 +1,44 @@ +package gitee + +import ( + "fmt" + "net/url" + + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +func (d *Gitee) getRepo() (*Repo, error) { + req := d.client.R() + if d.Token != "" { + req.SetQueryParam("access_token", d.Token) + } + if d.Cookie != "" { + req.SetHeader("Cookie", d.Cookie) + } + escapedOwner := url.PathEscape(d.Owner) + escapedRepo := url.PathEscape(d.Repo) + res, err := req.Get(fmt.Sprintf("/repos/%s/%s", escapedOwner, escapedRepo)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var repo Repo + if err := utils.Json.Unmarshal(res.Body(), &repo); err != nil { + return nil, err + } + if repo.DefaultBranch == "" { + return nil, fmt.Errorf("failed to fetch default branch") + } + return &repo, nil +} + +func toErr(res *resty.Response) error { + var errMsg ErrResp + if err := utils.Json.Unmarshal(res.Body(), &errMsg); err == nil && errMsg.Message != "" { + return fmt.Errorf("%s: %s", res.Status(), errMsg.Message) + } + return fmt.Errorf(res.Status()) +} From ce41587095bb3a680b6bc926aacd0eecb61722a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 11 Nov 2025 20:26:51 +0800 Subject: [PATCH 058/133] feat(cloud189): Added sanitization for file and folder names (#9366) - Introduced `sanitizeName` function to remove four-byte characters (e.g., emojis) from names before upload or creation. - Added `StripEmoji` option in driver configurations for cloud189 and cloud189pc. - Updated file and folder operations (upload, rename, and creation) to use sanitized names. - Ensured compatibility with both cloud189 and cloud189pc implementations. --- drivers/189/driver.go | 6 ++++-- drivers/189/meta.go | 7 ++++--- drivers/189/util.go | 33 ++++++++++++++++++++++++++++++--- drivers/189pc/driver.go | 16 +++++++++++++--- drivers/189pc/meta.go | 7 ++++--- drivers/189pc/utils.go | 37 +++++++++++++++++++++++++++++++++---- 6 files changed, 88 insertions(+), 18 deletions(-) diff --git a/drivers/189/driver.go b/drivers/189/driver.go index 6fc4932640c..0c8db1134ee 100644 --- a/drivers/189/driver.go +++ b/drivers/189/driver.go @@ -80,9 +80,10 @@ func (d *Cloud189) Link(ctx context.Context, file model.Obj, args model.LinkArgs } func (d *Cloud189) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + safeName := d.sanitizeName(dirName) form := map[string]string{ "parentFolderId": parentDir.GetID(), - "folderName": dirName, + "folderName": safeName, } _, err := d.request("https://cloud.189.cn/api/open/file/createFolder.action", http.MethodPost, func(req *resty.Request) { req.SetFormData(form) @@ -126,9 +127,10 @@ func (d *Cloud189) Rename(ctx context.Context, srcObj model.Obj, newName string) idKey = "folderId" nameKey = "destFolderName" } + safeName := d.sanitizeName(newName) form := map[string]string{ idKey: srcObj.GetID(), - nameKey: newName, + nameKey: safeName, } _, err := d.request(url, http.MethodPost, func(req *resty.Request) { req.SetFormData(form) diff --git a/drivers/189/meta.go b/drivers/189/meta.go index ad621fb440d..da81406e683 100644 --- a/drivers/189/meta.go +++ b/drivers/189/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - Cookie string `json:"cookie" help:"Fill in the cookie if need captcha"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + Cookie string `json:"cookie" help:"Fill in the cookie if need captcha"` + StripEmoji bool `json:"strip_emoji" help:"Remove four-byte characters (e.g., emoji) before upload"` driver.RootID } diff --git a/drivers/189/util.go b/drivers/189/util.go index 16a5aa3996e..ee2b5061dd8 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -11,9 +11,11 @@ import ( "io" "math" "net/http" + "path" "strconv" "strings" "time" + "unicode/utf8" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -222,13 +224,37 @@ func (d *Cloud189) getFiles(fileId string) ([]model.Obj, error) { return res, nil } +func (d *Cloud189) sanitizeName(name string) string { + if !d.StripEmoji { + return name + } + b := strings.Builder{} + for _, r := range name { + if utf8.RuneLen(r) == 4 { + continue + } + b.WriteRune(r) + } + sanitized := b.String() + if sanitized == "" { + ext := path.Ext(name) + if ext != "" { + sanitized = "file" + ext + } else { + sanitized = "file" + } + } + return sanitized +} + func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error { + safeName := d.sanitizeName(file.GetName()) res, err := d.client.R().SetMultipartFormData(map[string]string{ "parentId": dstDir.GetID(), "sessionKey": "??", "opertype": "1", - "fname": file.GetName(), - }).SetMultipartField("Filedata", file.GetName(), file.GetMimetype(), file).Post("https://hb02.upload.cloud.189.cn/v1/DCIWebUploadAction") + "fname": safeName, + }).SetMultipartField("Filedata", safeName, file.GetMimetype(), file).Post("https://hb02.upload.cloud.189.cn/v1/DCIWebUploadAction") if err != nil { return err } @@ -313,9 +339,10 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F const DEFAULT int64 = 10485760 var count = int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT))) + safeName := d.sanitizeName(file.GetName()) res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ "parentFolderId": dstDir.GetID(), - "fileName": encode(file.GetName()), + "fileName": encode(safeName), "fileSize": strconv.FormatInt(file.GetSize(), 10), "sliceSize": strconv.FormatInt(DEFAULT, 10), "lazyCheck": "1", diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index c91caf2fb4f..9462cef6662 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -205,10 +205,11 @@ func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName s fullUrl += "/createFolder.action" var newFolder Cloud189Folder + safeName := y.sanitizeName(dirName) _, err := y.post(fullUrl, func(req *resty.Request) { req.SetContext(ctx) req.SetQueryParams(map[string]string{ - "folderName": dirName, + "folderName": safeName, "relativePath": "", }) if isFamily { @@ -225,6 +226,7 @@ func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName s if err != nil { return nil, err } + newFolder.Name = safeName return &newFolder, nil } @@ -258,21 +260,29 @@ func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName strin } var newObj model.Obj + safeName := y.sanitizeName(newName) switch f := srcObj.(type) { case *Cloud189File: fullUrl += "/renameFile.action" queryParam["fileId"] = srcObj.GetID() - queryParam["destFileName"] = newName + queryParam["destFileName"] = safeName newObj = &Cloud189File{Icon: f.Icon} // 复用预览 case *Cloud189Folder: fullUrl += "/renameFolder.action" queryParam["folderId"] = srcObj.GetID() - queryParam["destFolderName"] = newName + queryParam["destFolderName"] = safeName newObj = &Cloud189Folder{} default: return nil, errs.NotSupport } + switch obj := newObj.(type) { + case *Cloud189File: + obj.Name = safeName + case *Cloud189Folder: + obj.Name = safeName + } + _, err := y.request(fullUrl, method, func(req *resty.Request) { req.SetContext(ctx).SetQueryParams(queryParam) }, nil, newObj, isFamily) diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 1891c5c0ccd..d6edc063593 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - VCode string `json:"validate_code"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + VCode string `json:"validate_code"` + StripEmoji bool `json:"strip_emoji" help:"Remove four-byte characters (e.g., emoji) before upload"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index a8b444cb4b7..ca89251e278 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -12,11 +12,13 @@ import ( "net/http/cookiejar" "net/url" "os" + "path" "regexp" "sort" "strconv" "strings" "time" + "unicode/utf8" "golang.org/x/sync/semaphore" @@ -57,6 +59,29 @@ const ( CHANNEL_ID = "web_cloud.189.cn" ) +func (y *Cloud189PC) sanitizeName(name string) string { + if !y.StripEmoji { + return name + } + b := strings.Builder{} + for _, r := range name { + if utf8.RuneLen(r) == 4 { + continue + } + b.WriteRune(r) + } + sanitized := b.String() + if sanitized == "" { + ext := path.Ext(name) + if ext != "" { + sanitized = "file" + ext + } else { + sanitized = "file" + } + } + return sanitized +} + func (y *Cloud189PC) SignatureHeader(url, method, params string, isFamily bool) map[string]string { dateOfGmt := getHttpDateStr() sessionKey := y.getTokenInfo().SessionKey @@ -475,10 +500,11 @@ func (y *Cloud189PC) refreshSession() (err error) { func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { size := file.GetSize() sliceSize := partSize(size) + safeName := y.sanitizeName(file.GetName()) params := Params{ "parentFolderId": dstDir.GetID(), - "fileName": url.QueryEscape(file.GetName()), + "fileName": url.QueryEscape(safeName), "fileSize": fmt.Sprint(file.GetSize()), "sliceSize": fmt.Sprint(sliceSize), "lazyCheck": "1", @@ -596,7 +622,8 @@ func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream m return nil, errors.New("invalid hash") } - uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, stream.GetName(), fmt.Sprint(stream.GetSize()), isFamily) + safeName := y.sanitizeName(stream.GetName()) + uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, safeName, fmt.Sprint(stream.GetSize()), isFamily) if err != nil { return nil, err } @@ -615,6 +642,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode tmpF *os.File err error ) + safeName := y.sanitizeName(file.GetName()) size := file.GetSize() if _, ok := cache.(io.ReaderAt); !ok && size > 0 { tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") @@ -697,7 +725,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode //step.2 预上传 params := Params{ "parentFolderId": dstDir.GetID(), - "fileName": url.QueryEscape(file.GetName()), + "fileName": url.QueryEscape(safeName), "fileSize": fmt.Sprint(file.GetSize()), "fileMd5": fileMd5Hex, "sliceSize": fmt.Sprint(sliceSize), @@ -833,9 +861,10 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model return nil, err } rateLimited := driver.NewLimitedUploadStream(ctx, io.NopCloser(tempFile)) + safeName := y.sanitizeName(file.GetName()) // 创建上传会话 - uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, file.GetName(), fmt.Sprint(file.GetSize()), isFamily) + uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, safeName, fmt.Sprint(file.GetSize()), isFamily) if err != nil { return nil, err } From 3cddb6b7edd77b422b5c9c66462b58538d6c206b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 11 Nov 2025 20:27:20 +0800 Subject: [PATCH 059/133] fix(driver): Handle Lanzou anti-crawler challenge by recalculating cookies (#9364) - Detect and solve `acw_sc__v2` challenge to bypass anti-crawler validation - Refactored request header initialization logic for clarity --- drivers/lanzou/util.go | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index e66252bcc79..be53963c157 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -430,17 +430,35 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( file.Time = timeFindReg.FindString(sharePageData) // 重定向获取真实链接 - res, err := base.NoRedirectClient.R().SetHeaders(map[string]string{ + headers := map[string]string{ "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6", - }).Get(downloadUrl) + } + res, err := base.NoRedirectClient.R().SetHeaders(headers).Get(downloadUrl) if err != nil { return nil, err } + rPageData := res.String() + if findAcwScV2Reg.MatchString(rPageData) { + log.Debug("lanzou: detected acw_sc__v2 challenge, recalculating cookie") + acwScV2, err := CalcAcwScV2(rPageData) + if err != nil { + return nil, err + } + // retry with calculated cookie to bypass anti-crawler validation + res, err = base.NoRedirectClient.R(). + SetHeaders(headers). + SetCookie(&http.Cookie{Name: "acw_sc__v2", Value: acwScV2}). + Get(downloadUrl) + if err != nil { + return nil, err + } + rPageData = res.String() + } + file.Url = res.Header().Get("location") // 触发验证 - rPageData := res.String() if res.StatusCode() != 302 { param, err = htmlJsonToMap(rPageData) if err != nil { From 129895beecb87c278d227a2038a2056fa6bbf804 Mon Sep 17 00:00:00 2001 From: vxtls <187420201+vxtls@users.noreply.github.com> Date: Sat, 15 Nov 2025 12:33:58 -0500 Subject: [PATCH 060/133] fix(misskey): folderId format validation and root directory handling --- drivers/misskey/util.go | 57 +++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/drivers/misskey/util.go b/drivers/misskey/util.go index f8baeafa6cb..d301955ec42 100644 --- a/drivers/misskey/util.go +++ b/drivers/misskey/util.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io" + "net/http" "time" "github.com/go-resty/resty/v2" @@ -56,23 +57,27 @@ func setBody(body interface{}) base.ReqCallback { } func handleFolderId(dir model.Obj) interface{} { - if dir.GetID() == "" { - return nil + if isRootFolder(dir) { + return nil // Root folder doesn't need folderId } return dir.GetID() } +func isRootFolder(dir model.Obj) bool { + return dir.GetID() == "" +} + // API layer methods func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { var files []MFile var body map[string]string - if dir.GetPath() != "/" { + if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} } - err := d.request("/files", "POST", setBody(body), &files) + err := d.request("/files", http.MethodPost, setBody(body), &files) if err != nil { return []model.Obj{}, err } @@ -84,12 +89,12 @@ func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) { var folders []MFolder var body map[string]string - if dir.GetPath() != "/" { + if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} } - err := d.request("/folders", "POST", setBody(body), &folders) + err := d.request("/folders", http.MethodPost, setBody(body), &folders) if err != nil { return []model.Obj{}, err } @@ -106,7 +111,7 @@ func (d *Misskey) list(dir model.Obj) ([]model.Obj, error) { func (d *Misskey) link(file model.Obj) (*model.Link, error) { var mFile MFile - err := d.request("/files/show", "POST", setBody(map[string]string{"fileId": file.GetID()}), &mFile) + err := d.request("/files/show", http.MethodPost, setBody(map[string]string{"fileId": file.GetID()}), &mFile) if err != nil { return nil, err } @@ -117,7 +122,7 @@ func (d *Misskey) link(file model.Obj) (*model.Link, error) { func (d *Misskey) makeDir(parentDir model.Obj, dirName string) (model.Obj, error) { var folder MFolder - err := d.request("/folders/create", "POST", setBody(map[string]interface{}{"parentId": handleFolderId(parentDir), "name": dirName}), &folder) + err := d.request("/folders/create", http.MethodPost, setBody(map[string]interface{}{"parentId": handleFolderId(parentDir), "name": dirName}), &folder) if err != nil { return nil, err } @@ -127,11 +132,11 @@ func (d *Misskey) makeDir(parentDir model.Obj, dirName string) (model.Obj, error func (d *Misskey) move(srcObj, dstDir model.Obj) (model.Obj, error) { if srcObj.IsDir() { var folder MFolder - err := d.request("/folders/update", "POST", setBody(map[string]interface{}{"folderId": srcObj.GetID(), "parentId": handleFolderId(dstDir)}), &folder) + err := d.request("/folders/update", http.MethodPost, setBody(map[string]interface{}{"folderId": srcObj.GetID(), "parentId": handleFolderId(dstDir)}), &folder) return mFolder2Object(folder), err } else { var file MFile - err := d.request("/files/update", "POST", setBody(map[string]interface{}{"fileId": srcObj.GetID(), "folderId": handleFolderId(dstDir)}), &file) + err := d.request("/files/update", http.MethodPost, setBody(map[string]interface{}{"fileId": srcObj.GetID(), "folderId": handleFolderId(dstDir)}), &file) return mFile2Object(file), err } } @@ -139,11 +144,11 @@ func (d *Misskey) move(srcObj, dstDir model.Obj) (model.Obj, error) { func (d *Misskey) rename(srcObj model.Obj, newName string) (model.Obj, error) { if srcObj.IsDir() { var folder MFolder - err := d.request("/folders/update", "POST", setBody(map[string]string{"folderId": srcObj.GetID(), "name": newName}), &folder) + err := d.request("/folders/update", http.MethodPost, setBody(map[string]string{"folderId": srcObj.GetID(), "name": newName}), &folder) return mFolder2Object(folder), err } else { var file MFile - err := d.request("/files/update", "POST", setBody(map[string]string{"fileId": srcObj.GetID(), "name": newName}), &file) + err := d.request("/files/update", http.MethodPost, setBody(map[string]string{"fileId": srcObj.GetID(), "name": newName}), &file) return mFile2Object(file), err } } @@ -171,7 +176,7 @@ func (d *Misskey) copy(srcObj, dstDir model.Obj) (model.Obj, error) { if err != nil { return nil, err } - err = d.request("/files/upload-from-url", "POST", setBody(map[string]interface{}{"url": url.URL, "folderId": handleFolderId(dstDir)}), &file) + err = d.request("/files/upload-from-url", http.MethodPost, setBody(map[string]interface{}{"url": url.URL, "folderId": handleFolderId(dstDir)}), &file) if err != nil { return nil, err } @@ -181,10 +186,10 @@ func (d *Misskey) copy(srcObj, dstDir model.Obj) (model.Obj, error) { func (d *Misskey) remove(obj model.Obj) error { if obj.IsDir() { - err := d.request("/folders/delete", "POST", setBody(map[string]string{"folderId": obj.GetID()}), nil) + err := d.request("/folders/delete", http.MethodPost, setBody(map[string]string{"folderId": obj.GetID()}), nil) return err } else { - err := d.request("/files/delete", "POST", setBody(map[string]string{"fileId": obj.GetID()}), nil) + err := d.request("/files/delete", http.MethodPost, setBody(map[string]string{"fileId": obj.GetID()}), nil) return err } } @@ -196,16 +201,24 @@ func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileSt Reader: stream, UpdateProgress: up, }) + + // Build form data, only add folderId if not root folder + formData := map[string]string{ + "name": stream.GetName(), + "comment": "", + "isSensitive": "false", + "force": "false", + } + + folderId := handleFolderId(dstDir) + if folderId != nil { + formData["folderId"] = folderId.(string) + } + req := base.RestyClient.R(). SetContext(ctx). SetFileReader("file", stream.GetName(), reader). - SetFormData(map[string]string{ - "folderId": handleFolderId(dstDir).(string), - "name": stream.GetName(), - "comment": "", - "isSensitive": "false", - "force": "false", - }). + SetFormData(formData). SetResult(&file). SetAuthToken(d.AccessToken) From 998022e38be0de08e709a00f9769e8e22b0eaa81 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Sun, 21 Dec 2025 15:23:12 +0800 Subject: [PATCH 061/133] feat(settings): Add `SetToken` endpoint for updating token settings - Introduced `SetToken` handler to allow manual token updates via API - Added `SetTokenReq` struct for request validation --- server/handles/setting.go | 21 ++++++++++++++++++++- server/router.go | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/server/handles/setting.go b/server/handles/setting.go index e0dbb490033..f209b7c5dc5 100644 --- a/server/handles/setting.go +++ b/server/handles/setting.go @@ -29,9 +29,13 @@ func getRoleOptions() string { return strings.Join(names, ",") } +type SetTokenReq struct { + Token string `json:"token" form:"token" binding:"required"` +} + func ResetToken(c *gin.Context) { token := random.Token() - item := model.SettingItem{Key: "token", Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} + item := model.SettingItem{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} if err := op.SaveSettingItem(&item); err != nil { common.ErrorResp(c, err, 500) return @@ -40,6 +44,21 @@ func ResetToken(c *gin.Context) { common.SuccessResp(c, token) } +func SetToken(c *gin.Context) { + var req SetTokenReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + item := model.SettingItem{Key: conf.Token, Value: req.Token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} + if err := op.SaveSettingItem(&item); err != nil { + common.ErrorResp(c, err, 500) + return + } + sign.Instance() + common.SuccessResp(c, req.Token) +} + func GetSetting(c *gin.Context) { key := c.Query("key") keys := c.Query("keys") diff --git a/server/router.go b/server/router.go index 4d79c1fde52..63503838af7 100644 --- a/server/router.go +++ b/server/router.go @@ -154,6 +154,7 @@ func admin(g *gin.RouterGroup) { setting.POST("/save", handles.SaveSettings) setting.POST("/delete", handles.DeleteSetting) setting.POST("/reset_token", handles.ResetToken) + setting.POST("/set_token", handles.SetToken) setting.POST("/set_aria2", handles.SetAria2) setting.POST("/set_qbit", handles.SetQbittorrent) setting.POST("/set_transmission", handles.SetTransmission) From 5f244090708b3dd7f1a6b5cf7170ad71b561b795 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Sun, 21 Dec 2025 15:59:26 +0800 Subject: [PATCH 062/133] feat(proxy): Added configurable signature for down proxy URLs - Introduced `DownProxySign` field with default `true` in the `Proxy` struct - Added `BuildDownProxyURL` utility to generate proxy URLs with an optional signature - Updated proxy handling logic across multiple modules to support the new signature configuration --- internal/model/storage.go | 9 +++++---- internal/op/driver.go | 5 +++++ server/common/proxy.go | 9 +++++++++ server/handles/down.go | 7 +------ server/handles/fsread.go | 9 +++++---- server/webdav/webdav.go | 6 +----- 6 files changed, 26 insertions(+), 19 deletions(-) diff --git a/internal/model/storage.go b/internal/model/storage.go index e3c7e1f9731..4d9c062518d 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -28,10 +28,11 @@ type Sort struct { } type Proxy struct { - WebProxy bool `json:"web_proxy"` - WebdavPolicy string `json:"webdav_policy"` - ProxyRange bool `json:"proxy_range"` - DownProxyUrl string `json:"down_proxy_url"` + WebProxy bool `json:"web_proxy"` + WebdavPolicy string `json:"webdav_policy"` + ProxyRange bool `json:"proxy_range"` + DownProxyUrl string `json:"down_proxy_url"` + DownProxySign bool `json:"down_proxy_sign" gorm:"default:true"` } func (s *Storage) GetStorage() *Storage { diff --git a/internal/op/driver.go b/internal/op/driver.go index 41b6f6d42c7..4099fbbf5dd 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -117,6 +117,11 @@ func getMainItems(config driver.Config) []driver.Item { Name: "down_proxy_url", Type: conf.TypeText, }) + items = append(items, driver.Item{ + Name: "down_proxy_sign", + Type: conf.TypeBool, + Default: "true", + }) if config.LocalSort { items = append(items, []driver.Item{{ Name: "order_by", diff --git a/server/common/proxy.go b/server/common/proxy.go index ca7f6325d7d..97bf84efa12 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -13,6 +13,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" + "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" @@ -129,6 +130,14 @@ func ProxyRange(link *model.Link, size int64) { } } +func BuildDownProxyURL(downProxyURL, path string, useSign bool) string { + base := strings.Split(downProxyURL, "\n")[0] + if useSign { + return fmt.Sprintf("%s%s?sign=%s", base, utils.EncodePath(path, true), sign.Sign(path)) + } + return fmt.Sprintf("%s%s", base, utils.EncodePath(path, true)) +} + type InterceptResponseWriter struct { http.ResponseWriter io.Writer diff --git a/server/handles/down.go b/server/handles/down.go index 2c5c2fafc51..59c75530d3b 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -6,14 +6,12 @@ import ( "io" stdpath "path" "strconv" - "strings" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" @@ -62,10 +60,7 @@ func Proxy(c *gin.Context) { if downProxyUrl != "" { _, ok := c.GetQuery("d") if !ok { - URL := fmt.Sprintf("%s%s?sign=%s", - strings.Split(downProxyUrl, "\n")[0], - utils.EncodePath(rawPath, true), - sign.Sign(rawPath)) + URL := common.BuildDownProxyURL(downProxyUrl, rawPath, storage.GetStorage().DownProxySign) c.Redirect(302, URL) return } diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 676d64b166e..15dd9f1ce7e 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -340,10 +340,11 @@ func FsGet(c *gin.Context) { query = "?sign=" + sign.Sign(reqPath) } if storage.GetStorage().DownProxyUrl != "" { - rawURL = fmt.Sprintf("%s%s?sign=%s", - strings.Split(storage.GetStorage().DownProxyUrl, "\n")[0], - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)) + rawURL = common.BuildDownProxyURL( + storage.GetStorage().DownProxyUrl, + reqPath, + storage.GetStorage().DownProxySign, + ) } else { rawURL = fmt.Sprintf("%s/p%s%s", common.GetApiUrl(c.Request), diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 00c0471f743..df1d2045140 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -21,7 +21,6 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" ) @@ -253,10 +252,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta return http.StatusInternalServerError, fmt.Errorf("webdav proxy error: %+v", err) } } else if storage.GetStorage().WebdavProxy() && downProxyUrl != "" { - u := fmt.Sprintf("%s%s?sign=%s", - strings.Split(downProxyUrl, "\n")[0], - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)) + u := common.BuildDownProxyURL(downProxyUrl, reqPath, storage.GetStorage().DownProxySign) w.Header().Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") http.Redirect(w, r, u, http.StatusFound) } else { From e5662efad3efb8fea8a6057e537af439d4c7997b Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Sun, 21 Dec 2025 19:01:18 +0800 Subject: [PATCH 063/133] feat(driver): Enhanced Baidu Netdisk upload logic with dynamic URL retrieval - Added support for dynamically retrieving upload URLs, with fallback in case of failure - Improved token refresh and error handling during uploads - Prevented uploading of empty files and added error message for invalid operations - Refactored large file uploads with segment-level progress and retry handling logic - Introduced constants for upload settings, enabling better configurability - Improved logs to include the driver name for better debugging context --- drivers/baidu_netdisk/driver.go | 220 ++++++++++++++++++++++---------- drivers/baidu_netdisk/meta.go | 12 ++ drivers/baidu_netdisk/types.go | 29 +++++ drivers/baidu_netdisk/util.go | 91 ++++++++++++- 4 files changed, 276 insertions(+), 76 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index c33e0b32b05..64539dc7d89 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -5,23 +5,26 @@ import ( "crypto/md5" "encoding/hex" "errors" + "fmt" "io" "net/url" "os" stdpath "path" "strconv" + "strings" + "sync" "time" - "golang.org/x/sync/semaphore" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/errgroup" + "github.com/alist-org/alist/v3/pkg/singleflight" "github.com/alist-org/alist/v3/pkg/utils" "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -31,8 +34,16 @@ type BaiduNetdisk struct { uploadThread int vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) + + upClient *resty.Client // 上传文件使用的http客户端 + uploadUrlG singleflight.Group[string] + uploadUrlMu sync.RWMutex + uploadUrl string // 上传域名 + uploadUrlUpdateTime time.Time // 上传域名上次更新时间 } +var ErrUploadIDExpired = errors.New("uploadid expired") + func (d *BaiduNetdisk) Config() driver.Config { return config } @@ -42,19 +53,26 @@ func (d *BaiduNetdisk) GetAddition() driver.Additional { } func (d *BaiduNetdisk) Init(ctx context.Context) error { + d.upClient = base.NewRestyClient(). + SetTimeout(UPLOAD_TIMEOUT). + SetRetryCount(UPLOAD_RETRY_COUNT). + SetRetryWaitTime(UPLOAD_RETRY_WAIT_TIME). + SetRetryMaxWaitTime(UPLOAD_RETRY_MAX_WAIT_TIME) d.uploadThread, _ = strconv.Atoi(d.UploadThread) - if d.uploadThread < 1 || d.uploadThread > 32 { - d.uploadThread, d.UploadThread = 3, "3" + if d.uploadThread < 1 { + d.uploadThread, d.UploadThread = 1, "1" + } else if d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 32, "32" } if _, err := url.Parse(d.UploadAPI); d.UploadAPI == "" || err != nil { - d.UploadAPI = "https://d.pcs.baidu.com" + d.UploadAPI = UPLOAD_FALLBACK_API } res, err := d.get("/xpan/nas", map[string]string{ "method": "uinfo", }, nil) - log.Debugf("[baidu] get uinfo: %s", string(res)) + log.Debugf("[baidu_netdisk] get uinfo: %s", string(res)) if err != nil { return err } @@ -181,6 +199,11 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo // **注意**: 截至 2024/04/20 百度云盘 api 接口返回的时间永远是当前时间，而不是文件时间。 // 而实际上云盘存储的时间是文件时间，所以此处需要覆盖时间，保证缓存与云盘的数据一致 func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // 百度网盘不允许上传空文件 + if stream.GetSize() < 1 { + return nil, ErrBaiduEmptyFilesNotAllowed + } + // rapid upload if newObj, err := d.PutRapid(ctx, dstDir, stream); err == nil { return newObj, nil @@ -245,7 +268,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } if tmpF != nil { if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize) + return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) } _, err = tmpF.Seek(0, io.SeekStart) if err != nil { @@ -259,82 +282,97 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 预上传 - // 尝试获取之前的进度 + // step.1 尝试读取已保存进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { - params := map[string]string{ - "method": "precreate", - } - form := map[string]string{ - "path": path, - "size": strconv.FormatInt(streamSize, 10), - "isdir": "0", - "autoinit": "1", - "rtype": "3", - "block_list": blockListStr, - "content-md5": contentMd5, - "slice-md5": sliceMd5, - } - joinTime(form, ctime, mtime) - - log.Debugf("[baidu_netdisk] precreate data: %s", form) - _, err = d.postForm("/xpan/file", params, form, &precreateResp) + // 没有进度，走预上传 + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) if err != nil { return nil, err } - log.Debugf("%+v", precreateResp) if precreateResp.ReturnType == 2 { //rapid upload, since got md5 match from baidu server // 修复时间，具体原因见 Put 方法注释的 **注意** - precreateResp.File.Ctime = ctime - precreateResp.File.Mtime = mtime return fileToObj(precreateResp.File), nil } } + // step.2 上传分片 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(1), - retry.Delay(time.Second), - retry.DelayType(retry.BackOffDelay)) - sem := semaphore.NewWeighted(3) - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break +uploadLoop: + for attempt := 0; attempt < 2; attempt++ { + // 获取上传域名 + uploadUrl := d.getUploadUrl(path, precreateResp.Uploadid) + // 并发上传 + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + cacheReaderAt, okReaderAt := cache.(io.ReaderAt) + if !okReaderAt { + return nil, fmt.Errorf("cache object must implement io.ReaderAt interface for upload operations") } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - byteSize = lastBlockSize - } - threadG.Go(func(ctx context.Context) error { - if err = sem.Acquire(ctx, 1); err != nil { - return err - } - defer sem.Release(1) - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), + totalParts := len(precreateResp.BlockList) + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) || partseq < 0 { + continue } - err := d.uploadSlice(ctx, params, stream.GetName(), - driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize))) - if err != nil { - return err + + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize } - up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) - precreateResp.BlockList[i] = -1 - return nil - }) - } - if err = threadG.Wait(); err != nil { - // 如果属于用户主动取消，则保存上传进度 + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": precreateResp.Uploadid, + "partseq": strconv.Itoa(partseq), + } + section := io.NewSectionReader(cacheReaderAt, offset, size) + err := d.uploadSlice(ctx, uploadUrl, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section)) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + // 当前goroutine还没退出，+1才是真正成功的数量 + success := threadG.Success() + 1 + progress := float64(success) * 100 / float64(totalParts) + up(progress) + return nil + }) + } + + err = threadG.Wait() + if err == nil { + break uploadLoop + } + + // 保存进度（所有错误都会保存） + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + if errors.Is(err, context.Canceled) { - precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch") + // 重新 precreate（所有分片都要重传） + newPre, err2 := d.precreate(ctx, path, streamSize, blockListStr, "", "", ctime, mtime) + if err2 != nil { + return nil, err2 + } + if newPre.ReturnType == 2 { + return fileToObj(newPre.File), nil + } + precreateResp = newPre + // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + continue uploadLoop } return nil, err } @@ -348,23 +386,67 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // 修复时间，具体原因见 Put 方法注释的 **注意** newFile.Ctime = ctime newFile.Mtime = mtime + // 上传成功清理进度 + base.SaveUploadProgress(d, nil, d.AccessToken, contentMd5) return fileToObj(newFile), nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { - res, err := base.RestyClient.R(). +// precreate 执行预上传操作，支持首次上传和 uploadid 过期重试 +func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + params := map[string]string{"method": "precreate"} + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + } + + // 只有在首次上传时才包含 content-md5 和 slice-md5 + if contentMd5 != "" && sliceMd5 != "" { + form["content-md5"] = contentMd5 + form["slice-md5"] = sliceMd5 + } + + joinTime(form, ctime, mtime) + + var precreateResp PrecreateResp + _, err := d.postForm("/xpan/file", params, form, &precreateResp) + if err != nil { + return nil, err + } + + // 修复时间，具体原因见 Put 方法注释的 **注意** + if precreateResp.ReturnType == 2 { + precreateResp.File.Ctime = ctime + precreateResp.File.Mtime = mtime + } + + return &precreateResp, nil +} + +func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file io.Reader) error { + res, err := d.upClient.R(). SetContext(ctx). SetQueryParams(params). SetFileReader("file", fileName, file). - Post(d.UploadAPI + "/rest/2.0/pcs/superfile2") + Post(uploadUrl + "/rest/2.0/pcs/superfile2") if err != nil { return err } log.Debugln(res.RawResponse.Status + res.String()) errCode := utils.Json.Get(res.Body(), "error_code").ToInt() errNo := utils.Json.Get(res.Body(), "errno").ToInt() + respStr := res.String() + lower := strings.ToLower(respStr) + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error in uploading to baidu, will retry. response=%s", res.String()) + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", res.String()) } return nil } diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 7577c747fe3..b75650ef063 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -1,6 +1,8 @@ package baidu_netdisk import ( + "time" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/op" ) @@ -17,11 +19,21 @@ type Addition struct { AccessToken string UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` + UseDynamicUploadAPI bool `json:"use_dynamic_upload_api" default:"true" help:"dynamically get upload api domain, when enabled, the 'Upload API' setting will be used as a fallback if failed to get"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` LowBandwithUploadMode bool `json:"low_bandwith_upload_mode" default:"false"` OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` } +const ( + UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 + UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) + UPLOAD_TIMEOUT = time.Minute * 30 // 上传请求超时时间 + UPLOAD_RETRY_COUNT = 3 + UPLOAD_RETRY_WAIT_TIME = time.Second * 1 + UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 +) + var config = driver.Config{ Name: "BaiduNetdisk", DefaultRoot: "/", diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index ed9b09df8ee..a158956d09b 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -1,6 +1,7 @@ package baidu_netdisk import ( + "errors" "path" "strconv" "time" @@ -9,6 +10,10 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" ) +var ( + ErrBaiduEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu netdisk") +) + type TokenErrResp struct { ErrorDescription string `json:"error_description"` Error string `json:"error"` @@ -189,3 +194,27 @@ type PrecreateResp struct { // return_type=2 File File `json:"info"` } + +type UploadServerResp struct { + BakServer []any `json:"bak_server"` + BakServers []struct { + Server string `json:"server"` + } `json:"bak_servers"` + ClientIP string `json:"client_ip"` + ErrorCode int `json:"error_code"` + ErrorMsg string `json:"error_msg"` + Expire int `json:"expire"` + Host string `json:"host"` + Newno string `json:"newno"` + QuicServer []any `json:"quic_server"` + QuicServers []struct { + Server string `json:"server"` + } `json:"quic_servers"` + RequestID int64 `json:"request_id"` + Server []any `json:"server"` + ServerTime int `json:"server_time"` + Servers []struct { + Server string `json:"server"` + } `json:"servers"` + Sl int `json:"sl"` +} diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 1249b3f470f..c5a7334315d 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -73,7 +73,7 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall errno := utils.Json.Get(res.Body(), "errno").ToInt() if errno != 0 { if utils.SliceContains([]int{111, -6}, errno) { - log.Info("refreshing baidu_netdisk token.") + log.Info("[baidu_netdisk] refreshing baidu_netdisk token.") err2 := d.refreshToken() if err2 != nil { return retry.Unrecoverable(err2) @@ -284,10 +284,10 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { // 非会员固定为 4MB if d.vipType == 0 { if d.CustomUploadPartSize != 0 { - log.Warnf("CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") + log.Warnf("[baidu_netdisk] CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") } if filesize > MaxSliceNum*DefaultSliceSize { - log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return DefaultSliceSize @@ -295,17 +295,17 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { if d.CustomUploadPartSize != 0 { if d.CustomUploadPartSize < DefaultSliceSize { - log.Warnf("CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) return DefaultSliceSize } if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize { - log.Warnf("CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) return VipSliceSize } if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize { - log.Warnf("CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) return SVipSliceSize } @@ -335,12 +335,89 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { } if filesize > MaxSliceNum*maxSliceSize { - log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return maxSliceSize } +// getUploadUrl 从开放平台获取上传域名/地址，并发请求会被合并，结果会被缓存1h。 +// 如果获取失败，则返回 Upload API设置项。 +func (d *BaiduNetdisk) getUploadUrl(path, uploadId string) string { + if !d.UseDynamicUploadAPI { + return d.UploadAPI + } + getCachedUrlFunc := func() string { + d.uploadUrlMu.RLock() + defer d.uploadUrlMu.RUnlock() + if d.uploadUrl != "" && time.Since(d.uploadUrlUpdateTime) < UPLOAD_URL_EXPIRE_TIME { + return d.uploadUrl + } + return "" + } + // 检查地址缓存 + if uploadUrl := getCachedUrlFunc(); uploadUrl != "" { + return uploadUrl + } + + uploadUrlGetFunc := func() (string, error) { + // 双重检查缓存 + if uploadUrl := getCachedUrlFunc(); uploadUrl != "" { + return uploadUrl, nil + } + + uploadUrl, err := d.requestForUploadUrl(path, uploadId) + if err != nil { + return "", err + } + + d.uploadUrlMu.Lock() + defer d.uploadUrlMu.Unlock() + d.uploadUrl = uploadUrl + d.uploadUrlUpdateTime = time.Now() + return uploadUrl, nil + } + + uploadUrl, err, _ := d.uploadUrlG.Do("", uploadUrlGetFunc) + if err != nil { + fallback := d.UploadAPI + log.Warnf("[baidu_netdisk] get upload URL failed (%v), will use fallback URL: %s", err, fallback) + return fallback + } + return uploadUrl +} + +// requestForUploadUrl 请求获取上传地址。 +// 实测此接口不需要认证，传method和upload_version就行，不过还是按文档规范调用。 +// https://pan.baidu.com/union/doc/Mlvw5hfnr +func (d *BaiduNetdisk) requestForUploadUrl(path, uploadId string) (string, error) { + params := map[string]string{ + "method": "locateupload", + "appid": "250528", + "path": path, + "uploadid": uploadId, + "upload_version": "2.0", + } + apiUrl := "https://d.pcs.baidu.com/rest/2.0/pcs/file" + var resp UploadServerResp + _, err := d.request(apiUrl, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(params) + }, &resp) + if err != nil { + return "", err + } + var uploadUrl string + if len(resp.Servers) > 0 { + uploadUrl = resp.Servers[0].Server + } else if len(resp.BakServers) > 0 { + uploadUrl = resp.BakServers[0].Server + } + if uploadUrl == "" { + return "", errors.New("upload URL is empty") + } + return uploadUrl, nil +} + // func encodeURIComponent(str string) string { // r := url.QueryEscape(str) // r = strings.ReplaceAll(r, "+", "%20") From 66a52b820dede9ccbc55eee4e1f300d9e11aec22 Mon Sep 17 00:00:00 2001 From: qianshi Date: Fri, 30 Jan 2026 14:55:09 +0800 Subject: [PATCH 064/133] fix(tls): harden defaults and warn on insecure mode default TLS verification to enabled to avoid insecure skips use the global TLS skip flag for WebDAV and LDAP transports log a clear warning when insecure TLS is explicitly enabled --- drivers/webdav/meta.go | 1 - drivers/webdav/util.go | 3 ++- internal/bootstrap/config.go | 9 +++++++++ internal/conf/config.go | 2 +- server/handles/ldap_login.go | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/webdav/meta.go b/drivers/webdav/meta.go index 2294d482a6e..d66499bc3f9 100644 --- a/drivers/webdav/meta.go +++ b/drivers/webdav/meta.go @@ -11,7 +11,6 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` driver.RootPath - TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" default:"false"` } var config = driver.Config{ diff --git a/drivers/webdav/util.go b/drivers/webdav/util.go index 23dc909ff88..dfd6e5b2457 100644 --- a/drivers/webdav/util.go +++ b/drivers/webdav/util.go @@ -6,6 +6,7 @@ import ( "net/http/cookiejar" "github.com/alist-org/alist/v3/drivers/webdav/odrvcookie" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/gowebdav" ) @@ -20,7 +21,7 @@ func (d *WebDav) setClient() error { c := gowebdav.NewClient(d.Address, d.Username, d.Password) c.SetTransport(&http.Transport{ Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: d.TlsInsecureSkipVerify}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, }) if d.isSharepoint() { cookie, err := odrvcookie.GetCookie(d.Username, d.Password, d.Address) diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index db3e20942b6..ac36059a076 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -70,6 +70,15 @@ func InitConfig() { if !conf.Conf.Force { confFromEnv() } + if conf.Conf.TlsInsecureSkipVerify { + log.Warn("SECURITY WARNING / 安全警告:") + log.Warn("TLS certificate verification is disabled.") + log.Warn("TLS 证书校验已被禁用。") + log.Warn("This exposes all storage traffic to MitM attacks and may leak credentials or allow data tampering.") + log.Warn("这会使所有存储通信暴露于中间人攻击（MitM），可能导致凭据泄露和数据被篡改。") + log.Warn("Only use this setting if you fully understand the risks.") + log.Warn("仅在你完全理解风险的情况下使用该配置。") + } // convert abs path if !filepath.IsAbs(conf.Conf.TempDir) { absPath, err := filepath.Abs(conf.Conf.TempDir) diff --git a/internal/conf/config.go b/internal/conf/config.go index cf7cde0bcf0..15bd7feba0a 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -156,7 +156,7 @@ func DefaultConfig() *Config { }, MaxConnections: 0, MaxConcurrency: 64, - TlsInsecureSkipVerify: true, + TlsInsecureSkipVerify: false, Tasks: TasksConfig{ Download: TaskConfig{ Workers: 5, diff --git a/server/handles/ldap_login.go b/server/handles/ldap_login.go index 2a85dc03d0b..fb8417b68b5 100644 --- a/server/handles/ldap_login.go +++ b/server/handles/ldap_login.go @@ -150,7 +150,7 @@ func dial(ldapServer string) (*ldap.Conn, error) { } if tlsEnabled { - return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: true}) + return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) } else { return ldap.Dial("tcp", ldapServer) } From 1328690016c3ffdc06e33949d3595113b59645eb Mon Sep 17 00:00:00 2001 From: qianshi Date: Fri, 30 Jan 2026 16:04:25 +0800 Subject: [PATCH 065/133] fix(fs): block path traversal in handlers --- internal/errs/operate.go | 1 + pkg/utils/path.go | 35 ++++++++++++++++++++++++ pkg/utils/path_test.go | 46 +++++++++++++++++++++++++++++++ server/handles/archive.go | 9 +++++-- server/handles/fsbatch.go | 21 +++++++++++++-- server/handles/fsmanage.go | 55 +++++++++++++++++++++++++++++++++----- 6 files changed, 157 insertions(+), 10 deletions(-) diff --git a/internal/errs/operate.go b/internal/errs/operate.go index 92fbd6a1a49..d2df47ddb9c 100644 --- a/internal/errs/operate.go +++ b/internal/errs/operate.go @@ -4,4 +4,5 @@ import "errors" var ( PermissionDenied = errors.New("permission denied") + InvalidName = errors.New("invalid file name") ) diff --git a/pkg/utils/path.go b/pkg/utils/path.go index 6f3a55fc3d3..fe4ff2fd96a 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -101,3 +101,38 @@ func JoinBasePath(basePath, reqPath string) (string, error) { func GetFullPath(mountPath, path string) string { return stdpath.Join(GetActualMountPath(mountPath), path) } + +// ValidateNameComponent validates a single path component. +// It rejects empty names, dot segments, separators, ".." sequences, and NUL bytes. +func ValidateNameComponent(name string) error { + if name == "" { + return errs.InvalidName + } + if name == "." || name == ".." { + return errs.InvalidName + } + if strings.Contains(name, "/") || strings.Contains(name, "\\") { + return errs.InvalidName + } + if strings.Contains(name, "..") { + return errs.InvalidName + } + if strings.ContainsRune(name, 0) { + return errs.InvalidName + } + return nil +} + +// JoinUnderBase safely joins baseDir with a single name component and ensures the +// result stays under baseDir after normalization. +func JoinUnderBase(baseDir, name string) (string, error) { + if err := ValidateNameComponent(name); err != nil { + return "", err + } + base := FixAndCleanPath(baseDir) + joined := FixAndCleanPath(stdpath.Join(base, name)) + if !IsSubPath(base, joined) { + return "", errs.InvalidName + } + return joined, nil +} diff --git a/pkg/utils/path_test.go b/pkg/utils/path_test.go index f42f2f8bb5d..def286b8038 100644 --- a/pkg/utils/path_test.go +++ b/pkg/utils/path_test.go @@ -20,3 +20,49 @@ func TestFixAndCleanPath(t *testing.T) { } } } + +func TestValidateNameComponent(t *testing.T) { + validNames := []string{ + "file.txt", + "abc", + "file_name-1", + } + for _, name := range validNames { + if err := ValidateNameComponent(name); err != nil { + t.Fatalf("expected valid name %q, got error: %v", name, err) + } + } + + invalidNames := []string{ + "", + ".", + "..", + "a/b", + `a\b`, + "a..b", + string([]byte{'a', 0, 'b'}), + } + for _, name := range invalidNames { + if err := ValidateNameComponent(name); err == nil { + t.Fatalf("expected invalid name %q to be rejected", name) + } + } +} + +func TestJoinUnderBase(t *testing.T) { + base := "/lanzou-y/shared/test1" + out, err := JoinUnderBase(base, "file.txt") + if err != nil { + t.Fatalf("expected join success, got error: %v", err) + } + if out != "/lanzou-y/shared/test1/file.txt" { + t.Fatalf("unexpected join result: %s", out) + } + + if _, err := JoinUnderBase(base, "../admin/screts.txt"); err == nil { + t.Fatalf("expected traversal to be rejected") + } + if _, err := JoinUnderBase(base, "sub/child"); err == nil { + t.Fatalf("expected nested path to be rejected") + } +} diff --git a/server/handles/archive.go b/server/handles/archive.go index 5787897cc90..844947408be 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -254,11 +254,16 @@ func FsArchiveDecompress(c *gin.Context) { return } user := c.MustGet("user").(*model.User) + srcDir, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } srcPaths := make([]string, 0, len(req.Name)) for _, name := range req.Name { - srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) + srcPath, err := utils.JoinUnderBase(srcDir, name) if err != nil { - common.ErrorResp(c, err, 403) + common.ErrorResp(c, err, 400) return } if !common.CheckPathLimitWithRoles(user, srcPath) { diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 7ff07c6df28..35cec645e1b 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -10,6 +10,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/generic" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" @@ -185,7 +186,15 @@ func FsBatchRename(c *gin.Context) { if renameObject.SrcName == "" || renameObject.NewName == "" { continue } - filePath := fmt.Sprintf("%s/%s", reqPath, renameObject.SrcName) + if err := utils.ValidateNameComponent(renameObject.NewName); err != nil { + common.ErrorResp(c, err, 400) + return + } + filePath, err := utils.JoinUnderBase(reqPath, renameObject.SrcName) + if err != nil { + common.ErrorResp(c, err, 400) + return + } if err := fs.Rename(c, filePath, renameObject.NewName); err != nil { common.ErrorResp(c, err, 500) return @@ -247,8 +256,16 @@ func FsRegexRename(c *gin.Context) { for _, file := range files { if srcRegexp.MatchString(file.GetName()) { - filePath := fmt.Sprintf("%s/%s", reqPath, file.GetName()) + filePath, err := utils.JoinUnderBase(reqPath, file.GetName()) + if err != nil { + common.ErrorResp(c, err, 500) + return + } newFileName := srcRegexp.ReplaceAllString(file.GetName(), req.NewNameRegex) + if err := utils.ValidateNameComponent(newFileName); err != nil { + common.ErrorResp(c, err, 400) + return + } if err := fs.Rename(c, filePath, newFileName); err != nil { common.ErrorResp(c, err, 500) return diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 87be6e4106b..dcb5a7b9a09 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -103,14 +103,29 @@ func FsMove(c *gin.Context) { } if !req.Overwrite { for _, name := range req.Names { - if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { + dstPath, err := utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } } } for i, name := range req.Names { - err := fs.Move(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, err = utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + err = fs.Move(c, srcPath, dstDir, len(req.Names) > i+1) if err != nil { common.ErrorResp(c, err, 500) return @@ -155,7 +170,12 @@ func FsCopy(c *gin.Context) { } if !req.Overwrite { for _, name := range req.Names { - if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { + dstPath, err := utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } @@ -163,7 +183,17 @@ func FsCopy(c *gin.Context) { } var addedTasks []task.TaskExtensionInfo for i, name := range req.Names { - t, err := fs.Copy(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, err = utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + t, err := fs.Copy(c, srcPath, dstDir, len(req.Names) > i+1) if t != nil { addedTasks = append(addedTasks, t) } @@ -204,8 +234,16 @@ func FsRename(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } + if err := utils.ValidateNameComponent(req.Name); err != nil { + common.ErrorResp(c, err, 400) + return + } if !req.Overwrite { - dstPath := stdpath.Join(stdpath.Dir(reqPath), req.Name) + dstPath, err := utils.JoinUnderBase(stdpath.Dir(reqPath), req.Name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } if dstPath != reqPath { if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", req.Name), 403) @@ -251,7 +289,12 @@ func FsRemove(c *gin.Context) { return } for _, name := range req.Names { - err := fs.Remove(c, stdpath.Join(reqDir, name)) + removePath, err := utils.JoinUnderBase(reqDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + err = fs.Remove(c, removePath) if err != nil { common.ErrorResp(c, err, 500) return From c258b93dbf01f8a343dca7130cc24367cf533081 Mon Sep 17 00:00:00 2001 From: qianshi Date: Sat, 31 Jan 2026 15:07:03 +0800 Subject: [PATCH 066/133] fix(archive): prevent Zip Slip in extraction --- internal/archive/archives/archives.go | 46 +++++++-- internal/archive/archives/utils.go | 10 +- internal/archive/rardecode/rardecode.go | 59 ++++++++++-- internal/archive/rardecode/utils.go | 40 ++++---- internal/archive/tool/helper.go | 113 +++++++++++++++++------ internal/archive/tool/securepath.go | 62 +++++++++++++ internal/archive/tool/securepath_test.go | 48 ++++++++++ 7 files changed, 310 insertions(+), 68 deletions(-) create mode 100644 internal/archive/tool/securepath.go create mode 100644 internal/archive/tool/securepath_test.go diff --git a/internal/archive/archives/archives.go b/internal/archive/archives/archives.go index 0a42cd0c512..e3a48b15757 100644 --- a/internal/archive/archives/archives.go +++ b/internal/archive/archives/archives.go @@ -1,10 +1,12 @@ package archives import ( + "fmt" "io" "io/fs" "os" stdpath "path" + "path/filepath" "strings" "github.com/alist-org/alist/v3/internal/archive/tool" @@ -106,7 +108,7 @@ func (Archives) Decompress(ss []*stream.SeekableStream, outputPath string, args } if stat.IsDir() { isDir = true - outputPath = stdpath.Join(outputPath, stat.Name()) + outputPath = filepath.Join(outputPath, stat.Name()) err = os.Mkdir(outputPath, 0700) if err != nil { return filterPassword(err) @@ -118,18 +120,46 @@ func (Archives) Decompress(ss []*stream.SeekableStream, outputPath string, args if err != nil { return err } + if p == path { + if d.IsDir() { + return nil + } + } relPath := strings.TrimPrefix(p, path+"/") - dstPath := stdpath.Join(outputPath, relPath) + if relPath == "" || relPath == "." { + if d.IsDir() { + return nil + } + } + dstPath, err := tool.SecureJoin(outputPath, relPath) + if err != nil { + return err + } if d.IsDir() { - err = os.MkdirAll(dstPath, 0700) - } else { - dir := stdpath.Dir(dstPath) - err = decompress(fsys, p, dir, func(_ float64) {}) + return os.MkdirAll(dstPath, 0700) } - return err + info, err := d.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, p) + } + if err := os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + return decompress(fsys, p, dstPath, func(_ float64) {}) }) } else { - err = decompress(fsys, path, outputPath, up) + entryName := stdpath.Base(path) + dstPath, e := tool.SecureJoin(outputPath, entryName) + if e != nil { + return e + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = decompress(fsys, path, dstPath, up) } return filterPassword(err) } diff --git a/internal/archive/archives/utils.go b/internal/archive/archives/utils.go index 2f499a10feb..5249862ce4b 100644 --- a/internal/archive/archives/utils.go +++ b/internal/archive/archives/utils.go @@ -1,12 +1,13 @@ package archives import ( + "fmt" "io" fs2 "io/fs" "os" - stdpath "path" "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" @@ -59,7 +60,7 @@ func filterPassword(err error) error { return err } -func decompress(fsys fs2.FS, filePath, targetPath string, up model.UpdateProgress) error { +func decompress(fsys fs2.FS, filePath, dstPath string, up model.UpdateProgress) error { rc, err := fsys.Open(filePath) if err != nil { return err @@ -69,7 +70,10 @@ func decompress(fsys fs2.FS, filePath, targetPath string, up model.UpdateProgres if err != nil { return err } - f, err := os.OpenFile(stdpath.Join(targetPath, stat.Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if !stat.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, filePath) + } + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } diff --git a/internal/archive/rardecode/rardecode.go b/internal/archive/rardecode/rardecode.go index cd31d1a40e0..2848c704bee 100644 --- a/internal/archive/rardecode/rardecode.go +++ b/internal/archive/rardecode/rardecode.go @@ -1,15 +1,18 @@ package rardecode import ( + "fmt" + "io" + "os" + stdpath "path" + "path/filepath" + "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" "github.com/nwaples/rardecode/v2" - "io" - "os" - stdpath "path" - "strings" ) type RarDecoder struct{} @@ -85,7 +88,11 @@ func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, arg if header.IsDir { name = name + "/" } - err = decompress(reader, header, name, outputPath) + dstPath, e := tool.SecureJoin(outputPath, name) + if e != nil { + return e + } + err = decompress(reader, header, dstPath) if err != nil { return err } @@ -94,6 +101,7 @@ func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, arg innerPath := strings.TrimPrefix(args.InnerPath, "/") innerBase := stdpath.Base(innerPath) createdBaseDir := false + var baseDirPath string for { var header *rardecode.FileHeader header, err = reader.Next() @@ -108,22 +116,55 @@ func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, arg name = name + "/" } if name == innerPath { - err = _decompress(reader, header, outputPath, up) + if header.IsDir { + if !createdBaseDir { + baseDirPath, err = tool.SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + if err = os.MkdirAll(baseDirPath, 0700); err != nil { + return err + } + createdBaseDir = true + } + continue + } + if !header.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, header.Name) + } + dstPath, e := tool.SecureJoin(outputPath, stdpath.Base(innerPath)) + if e != nil { + return e + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(reader, header, dstPath, up) if err != nil { return err } break } else if strings.HasPrefix(name, innerPath+"/") { - targetPath := stdpath.Join(outputPath, innerBase) if !createdBaseDir { - err = os.Mkdir(targetPath, 0700) + baseDirPath, err = tool.SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + err = os.MkdirAll(baseDirPath, 0700) if err != nil { return err } createdBaseDir = true } restPath := strings.TrimPrefix(name, innerPath+"/") - err = decompress(reader, header, restPath, targetPath) + if restPath == "" || restPath == "." { + continue + } + dstPath, e := tool.SecureJoin(baseDirPath, restPath) + if e != nil { + return e + } + err = decompress(reader, header, dstPath) if err != nil { return err } diff --git a/internal/archive/rardecode/utils.go b/internal/archive/rardecode/utils.go index 5790ec58a22..e3612b363df 100644 --- a/internal/archive/rardecode/utils.go +++ b/internal/archive/rardecode/utils.go @@ -2,18 +2,20 @@ package rardecode import ( "fmt" - "github.com/alist-org/alist/v3/internal/archive/tool" - "github.com/alist-org/alist/v3/internal/errs" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/nwaples/rardecode/v2" "io" "io/fs" "os" stdpath "path" + "path/filepath" "sort" "strings" "time" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/nwaples/rardecode/v2" ) type VolumeFile struct { @@ -179,27 +181,21 @@ func getReader(ss []*stream.SeekableStream, password string) (*rardecode.Reader, return &rc.Reader, nil } -func decompress(reader *rardecode.Reader, header *rardecode.FileHeader, filePath, outputPath string) error { - targetPath := outputPath - dir, base := stdpath.Split(filePath) - if dir != "" { - targetPath = stdpath.Join(targetPath, dir) - err := os.MkdirAll(targetPath, 0700) - if err != nil { - return err - } +func decompress(reader *rardecode.Reader, header *rardecode.FileHeader, dstPath string) error { + if header.IsDir { + return os.MkdirAll(dstPath, 0700) } - if base != "" { - err := _decompress(reader, header, targetPath, func(_ float64) {}) - if err != nil { - return err - } + if !header.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, header.Name) } - return nil + if err := os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + return _decompress(reader, header, dstPath, func(_ float64) {}) } -func _decompress(reader *rardecode.Reader, header *rardecode.FileHeader, targetPath string, up model.UpdateProgress) error { - f, err := os.OpenFile(stdpath.Join(targetPath, stdpath.Base(header.Name)), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) +func _decompress(reader *rardecode.Reader, header *rardecode.FileHeader, dstPath string, up model.UpdateProgress) error { + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } diff --git a/internal/archive/tool/helper.go b/internal/archive/tool/helper.go index 20da34467b0..80254b8fd04 100644 --- a/internal/archive/tool/helper.go +++ b/internal/archive/tool/helper.go @@ -1,10 +1,12 @@ package tool import ( + "fmt" "io" "io/fs" "os" stdpath "path" + "path/filepath" "strings" "github.com/alist-org/alist/v3/internal/model" @@ -119,7 +121,30 @@ func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args mode if args.InnerPath == "/" { for i, file := range files { name := file.Name() - err = decompress(file, name, outputPath, args.Password) + info := file.FileInfo() + if info.IsDir() { + var dirPath string + dirPath, err = SecureJoin(outputPath, name) + if err != nil { + return err + } + if err = os.MkdirAll(dirPath, 0700); err != nil { + return err + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(outputPath, name) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, func(_ float64) {}) if err != nil { return err } @@ -129,25 +154,80 @@ func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args mode innerPath := strings.TrimPrefix(args.InnerPath, "/") innerBase := stdpath.Base(innerPath) createdBaseDir := false + var baseDirPath string for _, file := range files { name := file.Name() if name == innerPath { - err = _decompress(file, outputPath, args.Password, up) + info := file.FileInfo() + if info.IsDir() { + if !createdBaseDir { + baseDirPath, err = SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + if err = os.MkdirAll(baseDirPath, 0700); err != nil { + return err + } + createdBaseDir = true + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(outputPath, stdpath.Base(innerPath)) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, up) if err != nil { return err } break } else if strings.HasPrefix(name, innerPath+"/") { - targetPath := stdpath.Join(outputPath, innerBase) if !createdBaseDir { - err = os.Mkdir(targetPath, 0700) + baseDirPath, err = SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + err = os.MkdirAll(baseDirPath, 0700) if err != nil { return err } createdBaseDir = true } restPath := strings.TrimPrefix(name, innerPath+"/") - err = decompress(file, restPath, targetPath, args.Password) + if restPath == "" || restPath == "." { + continue + } + info := file.FileInfo() + if info.IsDir() { + var dirPath string + dirPath, err = SecureJoin(baseDirPath, restPath) + if err != nil { + return err + } + if err = os.MkdirAll(dirPath, 0700); err != nil { + return err + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(baseDirPath, restPath) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, func(_ float64) {}) if err != nil { return err } @@ -157,26 +237,7 @@ func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args mode return nil } -func decompress(file SubFile, filePath, outputPath, password string) error { - targetPath := outputPath - dir, base := stdpath.Split(filePath) - if dir != "" { - targetPath = stdpath.Join(targetPath, dir) - err := os.MkdirAll(targetPath, 0700) - if err != nil { - return err - } - } - if base != "" { - err := _decompress(file, targetPath, password, func(_ float64) {}) - if err != nil { - return err - } - } - return nil -} - -func _decompress(file SubFile, targetPath, password string, up model.UpdateProgress) error { +func _decompress(file SubFile, dstPath, password string, up model.UpdateProgress) error { if encrypt, ok := file.(CanEncryptSubFile); ok && encrypt.IsEncrypted() { encrypt.SetPassword(password) } @@ -185,7 +246,7 @@ func _decompress(file SubFile, targetPath, password string, up model.UpdateProgr return err } defer func() { _ = rc.Close() }() - f, err := os.OpenFile(stdpath.Join(targetPath, file.FileInfo().Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } diff --git a/internal/archive/tool/securepath.go b/internal/archive/tool/securepath.go new file mode 100644 index 00000000000..45f5e749d53 --- /dev/null +++ b/internal/archive/tool/securepath.go @@ -0,0 +1,62 @@ +package tool + +import ( + "errors" + "fmt" + "path" + "path/filepath" + "strings" +) + +// ErrArchiveIllegalPath indicates an archive entry path is unsafe for extraction. +var ErrArchiveIllegalPath = errors.New("archive entry has illegal path") + +// SecureJoin returns a safe extraction path for an archive entry. +// It rejects absolute paths, traversal, Windows drive/UNC paths, and NUL bytes. +func SecureJoin(baseDir, entryName string) (string, error) { + if strings.Contains(entryName, "\x00") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + normalized := strings.ReplaceAll(entryName, "\\", "/") + if strings.HasPrefix(normalized, "//") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + cleaned := path.Clean(normalized) + + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + if strings.HasPrefix(cleaned, "/") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + rel := filepath.FromSlash(cleaned) + if filepath.IsAbs(rel) || filepath.VolumeName(rel) != "" { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + if strings.HasPrefix(rel, `\\`) { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + base := filepath.Clean(baseDir) + dst := filepath.Join(base, rel) + + baseAbs, err := filepath.Abs(base) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + dstAbs, err := filepath.Abs(dst) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + + relCheck, err := filepath.Rel(baseAbs, dstAbs) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + if relCheck == ".." || strings.HasPrefix(relCheck, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + return dst, nil +} diff --git a/internal/archive/tool/securepath_test.go b/internal/archive/tool/securepath_test.go new file mode 100644 index 00000000000..78be52ee5d5 --- /dev/null +++ b/internal/archive/tool/securepath_test.go @@ -0,0 +1,48 @@ +package tool + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestSecureJoin(t *testing.T) { + baseDir := t.TempDir() + tests := []struct { + name string + entry string + wantErr bool + }{ + {name: "ok", entry: "a/b/c.txt", wantErr: false}, + {name: "parent", entry: "../evil.txt", wantErr: true}, + {name: "parent-backslash", entry: "..\\evil.txt", wantErr: true}, + {name: "abs", entry: "/tmp/evil.txt", wantErr: true}, + {name: "drive", entry: "C:\\evil.txt", wantErr: true}, + {name: "unc", entry: "\\\\server\\share\\evil.txt", wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dst, err := SecureJoin(baseDir, tc.entry) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for %q, got nil", tc.entry) + } + if !strings.Contains(err.Error(), tc.entry) { + t.Fatalf("error should include entry name %q, got %q", tc.entry, err.Error()) + } + return + } + if err != nil { + t.Fatalf("unexpected error for %q: %v", tc.entry, err) + } + rel, err := filepath.Rel(baseDir, dst) + if err != nil { + t.Fatalf("Rel failed: %v", err) + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + t.Fatalf("path escaped baseDir: %q", dst) + } + }) + } +} From 42fce722a90c1fa33695969df2fb54faae089d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 3 Feb 2026 22:33:50 +0800 Subject: [PATCH 067/133] feat: add doubao new driver (#9416) * feat(driver): add doubao_new driver * fix(driver): improve doubao_new move with csrf * feat(driver): add doubao_new remove task polling --- drivers/all.go | 1 + drivers/doubao_new/driver.go | 600 +++++++++++++++++++++++ drivers/doubao_new/meta.go | 36 ++ drivers/doubao_new/types.go | 182 +++++++ drivers/doubao_new/util.go | 909 +++++++++++++++++++++++++++++++++++ server/handles/down.go | 14 + 6 files changed, 1742 insertions(+) create mode 100644 drivers/doubao_new/driver.go create mode 100644 drivers/doubao_new/meta.go create mode 100644 drivers/doubao_new/types.go create mode 100644 drivers/doubao_new/util.go diff --git a/drivers/all.go b/drivers/all.go index 3eb7e813bcc..3b0435595d2 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -27,6 +27,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" _ "github.com/alist-org/alist/v3/drivers/crypt" _ "github.com/alist-org/alist/v3/drivers/doubao" + _ "github.com/alist-org/alist/v3/drivers/doubao_new" _ "github.com/alist-org/alist/v3/drivers/doubao_share" _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" diff --git a/drivers/doubao_new/driver.go b/drivers/doubao_new/driver.go new file mode 100644 index 00000000000..7846551790a --- /dev/null +++ b/drivers/doubao_new/driver.go @@ -0,0 +1,600 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" +) + +type DoubaoNew struct { + model.Storage + Addition + TtLogid string +} + +func (d *DoubaoNew) Config() driver.Config { + return config +} + +func (d *DoubaoNew) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *DoubaoNew) Init(ctx context.Context) error { + // TODO login / refresh token + //op.MustSaveDriverStorage(d) + return nil +} + +func (d *DoubaoNew) Drop(ctx context.Context) error { + return nil +} + +func (d *DoubaoNew) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + nodes, err := d.listAllChildren(ctx, dir.GetID()) + if err != nil { + return nil, err + } + + objs := make([]model.Obj, 0, len(nodes)) + for _, node := range nodes { + size := parseSize(node.Extra.Size) + isFolder := node.Type == 0 + obj := &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: dir.GetID(), + Name: node.Name, + Size: size, + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: isFolder, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + } + objs = append(objs, obj) + } + + return objs, nil +} + +func (d *DoubaoNew) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj, ok := file.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + if obj.IsFolder { + return nil, errs.LinkIsDir + } + if args.Type == "preview" || args.Type == "thumb" { + if link, err := d.previewLink(ctx, obj, args); err == nil { + return link, nil + } + } + auth := d.resolveAuthorization() + dpop := d.resolveDpop() + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + + query := url.Values{} + query.Set("authorization", auth) + query.Set("dpop", dpop) + + downloadURL := DownloadBaseURL + "/space/api/box/stream/download/all/" + obj.ObjToken + "/?" + query.Encode() + + headers := http.Header{ + "Referer": []string{"https://www.doubao.com/"}, + "User-Agent": []string{base.UserAgent}, + } + + return &model.Link{ + URL: downloadURL, + Header: headers, + }, nil +} + +func (d *DoubaoNew) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + node, err := d.createFolder(ctx, parentDir.GetID(), dirName) + if err != nil { + return nil, err + } + return &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: parentDir.GetID(), + Name: node.Name, + Size: parseSize(node.Extra.Size), + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: true, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + }, nil +} + +func (d *DoubaoNew) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if dstDir == nil { + return nil, errors.New("nil destination dir") + } + srcToken := srcObj.GetID() + if srcToken == "" { + if obj, ok := srcObj.(*Object); ok { + srcToken = obj.ObjToken + } + } + if srcToken == "" { + return nil, errors.New("missing source token") + } + if err := d.moveObj(ctx, srcToken, dstDir.GetID()); err != nil { + return nil, err + } + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Path = dstDir.GetID() + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if srcObj.IsDir() { + if err := d.renameFolder(ctx, srcObj.GetID(), newName); err != nil { + return nil, err + } + } else { + fileToken := "" + if obj, ok := srcObj.(*Object); ok { + fileToken = obj.ObjToken + } + if fileToken == "" { + fileToken = srcObj.GetID() + } + if err := d.renameFile(ctx, fileToken, newName); err != nil { + return nil, err + } + } + + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Name = newName + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Remove(ctx context.Context, obj model.Obj) error { + if obj == nil { + return errors.New("nil object") + } + token := obj.GetID() + if token == "" { + if o, ok := obj.(*Object); ok { + token = o.ObjToken + } + } + if token == "" { + return errors.New("missing object token") + } + return d.removeObj(ctx, []string{token}) +} + +func (d *DoubaoNew) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if file == nil { + return nil, errors.New("nil file") + } + if file.GetSize() <= 0 { + return nil, errors.New("invalid file size") + } + + uploadPrep, err := d.prepareUpload(ctx, file.GetName(), file.GetSize(), dstDir.GetID()) + if err != nil { + return nil, err + } + if uploadPrep.BlockSize <= 0 { + return nil, errors.New("invalid block size from prepare") + } + + tmpFile, err := file.CacheFullInTempFile() + if err != nil { + return nil, err + } + defer tmpFile.Close() + + blockSize := uploadPrep.BlockSize + totalSize := file.GetSize() + numBlocks := int((totalSize + blockSize - 1) / blockSize) + blocks := make([]UploadBlock, 0, numBlocks) + blockMeta := make(map[int]UploadBlock, numBlocks) + + for seq := 0; seq < numBlocks; seq++ { + offset := int64(seq) * blockSize + length := blockSize + if remain := totalSize - offset; remain < length { + length = remain + } + buf := make([]byte, int(length)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + buf = buf[:n] + sum := sha256.Sum256(buf) + hash := base64.StdEncoding.EncodeToString(sum[:]) + checksum := adler32String(buf) + + block := UploadBlock{ + Hash: hash, + Seq: seq, + Size: int64(n), + Checksum: checksum, + IsUploaded: true, + } + blocks = append(blocks, block) + blockMeta[seq] = block + } + + needed, err := d.uploadBlocks(ctx, uploadPrep.UploadID, blocks, "explorer") + if err != nil { + return nil, err + } + + if len(needed.NeededUploadBlocks) > 0 { + sort.Slice(needed.NeededUploadBlocks, func(i, j int) bool { + return needed.NeededUploadBlocks[i].Seq < needed.NeededUploadBlocks[j].Seq + }) + const maxMergeBlockCount = 20 + var ( + groupSeqs []int + groupChecksums []string + groupSizes []int64 + groupRealSize int64 + groupExpectSum int64 + groupBuf bytes.Buffer + uploadedBytes int64 + ) + + flushGroup := func() error { + if len(groupSeqs) == 0 { + return nil + } + data := groupBuf.Bytes() + expectLen := groupExpectSum + if len(data) > 0 { + headLen := 32 + if len(data) < headLen { + headLen = len(data) + } + tailLen := 32 + if len(data) < tailLen { + tailLen = len(data) + } + fmt.Printf("head32 = %x\n", data[:headLen]) + fmt.Printf("tail32 = %x\n", data[len(data)-tailLen:]) + } + if int64(len(data)) != expectLen { + return fmt.Errorf("[doubao_new] merge blocks invalid body len: got=%d expect=%d seqs=%v", len(data), expectLen, groupSeqs) + } + mergeResp, err := d.mergeUploadBlocks(ctx, uploadPrep.UploadID, groupSeqs, groupChecksums, groupSizes, blockSize, data) + if err != nil { + return err + } + if len(mergeResp.SuccessSeqList) != len(groupSeqs) { + return fmt.Errorf("[doubao_new] merge blocks incomplete: %v", mergeResp.SuccessSeqList) + } + success := make(map[int]bool, len(mergeResp.SuccessSeqList)) + for _, seq := range mergeResp.SuccessSeqList { + success[seq] = true + } + for _, seq := range groupSeqs { + if !success[seq] { + return fmt.Errorf("[doubao_new] merge blocks missing seq %d", seq) + } + } + + uploadedBytes += groupRealSize + groupSeqs = groupSeqs[:0] + groupChecksums = groupChecksums[:0] + groupSizes = groupSizes[:0] + groupRealSize = 0 + groupExpectSum = 0 + groupBuf.Reset() + if up != nil { + percent := float64(uploadedBytes) / float64(totalSize) * 100 + up(percent) + } + return nil + } + + for _, item := range needed.NeededUploadBlocks { + if _, ok := blockMeta[item.Seq]; !ok { + return nil, fmt.Errorf("[doubao_new] missing block meta for seq %d", item.Seq) + } + if item.Size <= 0 { + return nil, fmt.Errorf("[doubao_new] invalid block size from needed list: seq=%d size=%d", item.Seq, item.Size) + } + offset := int64(item.Seq) * blockSize + buf := make([]byte, int(item.Size)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + if n != len(buf) { + return nil, fmt.Errorf("[doubao_new] short read: seq=%d want=%d got=%d", item.Seq, len(buf), n) + } + buf = buf[:n] + realAdler := adler32String(buf) + if realAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] block checksum mismatch: seq=%d offset=%d adler32=%s step2=%s", item.Seq, offset, realAdler, item.Checksum) + } + payloadStart := groupBuf.Len() + groupBuf.Write(buf) + payloadEnd := groupBuf.Len() + payloadAdler := adler32String(groupBuf.Bytes()[payloadStart:payloadEnd]) + if payloadAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] payload checksum mismatch: seq=%d start=%d end=%d adler32=%s step2=%s", item.Seq, payloadStart, payloadEnd, payloadAdler, item.Checksum) + } + groupSeqs = append(groupSeqs, item.Seq) + groupChecksums = append(groupChecksums, item.Checksum) + groupSizes = append(groupSizes, item.Size) + groupRealSize += int64(n) + groupExpectSum += item.Size + if len(groupSeqs) >= maxMergeBlockCount { + if err := flushGroup(); err != nil { + return nil, err + } + } + } + + if err := flushGroup(); err != nil { + return nil, err + } + if up != nil { + up(100) + } + } else if up != nil { + up(100) + } + + numBlocksFinish := uploadPrep.NumBlocks + if numBlocksFinish <= 0 { + numBlocksFinish = numBlocks + } + finish, err := d.finishUpload(ctx, uploadPrep.UploadID, numBlocksFinish, "explorer") + if err != nil { + return nil, err + } + + nodeToken := finish.Extra.NodeToken + if nodeToken == "" { + nodeToken = finish.FileToken + } + now := time.Now() + return &Object{ + Object: model.Object{ + ID: nodeToken, + Path: dstDir.GetID(), + Name: file.GetName(), + Size: file.GetSize(), + Modified: now, + Ctime: now, + IsFolder: false, + }, + ObjToken: finish.FileToken, + }, nil +} + +func (d *DoubaoNew) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch args.Method { + case "doubao_preview", "preview": + obj, ok := args.Obj.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errs.NotSupport + } + + imgExt := ".webp" + pageNums := 1 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + if extra.PageNums > 0 { + pageNums = extra.PageNums + } + } + } + + return base.Json{ + "version": info.Version, + "img_ext": imgExt, + "page_nums": pageNums, + }, nil + default: + return nil, errs.NotSupport + } +} + +func (d *DoubaoNew) listAllChildren(ctx context.Context, parentToken string) ([]Node, error) { + nodes := make([]Node, 0, 50) + lastLabel := "" + for page := 0; page < 100; page++ { + data, err := d.listChildren(ctx, parentToken, lastLabel) + if err != nil { + return nil, err + } + + if len(data.NodeList) > 0 { + for _, token := range data.NodeList { + node, ok := data.Entities.Nodes[token] + if !ok { + continue + } + nodes = append(nodes, node) + } + } else { + for _, node := range data.Entities.Nodes { + nodes = append(nodes, node) + } + } + + if !data.HasMore || data.LastLabel == "" || data.LastLabel == lastLabel { + break + } + lastLabel = data.LastLabel + } + + if len(nodes) == 0 { + return nil, nil + } + return nodes, nil +} + +func (d *DoubaoNew) previewLink(ctx context.Context, obj *Object, args model.LinkArgs) (*model.Link, error) { + auth := d.resolveAuthorization() + dpop := d.resolveDpop() + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errors.New("preview not available") + } + + subID := "" + pageIndex := 0 + if args.HttpReq != nil { + query := args.HttpReq.URL.Query() + if v := query.Get("sub_id"); v != "" { + subID = v + } else if v := query.Get("page"); v != "" { + if p, err := strconv.Atoi(v); err == nil && p >= 0 { + pageIndex = p + } + } + } + if subID == "" { + imgExt := ".webp" + pageNums := 0 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + pageNums = extra.PageNums + } + } + if pageNums > 0 && pageIndex >= pageNums { + pageIndex = pageNums - 1 + } + subID = fmt.Sprintf("img_%d%s", pageIndex, imgExt) + } + + query := url.Values{} + query.Set("preview_type", "22") + query.Set("sub_id", subID) + if info.Version != "" { + query.Set("version", info.Version) + } + previewURL := fmt.Sprintf("%s/space/api/box/stream/download/preview_sub/%s?%s", BaseURL, obj.ObjToken, query.Encode()) + + headers := http.Header{ + "Referer": []string{"https://www.doubao.com/"}, + "User-Agent": []string{base.UserAgent}, + "Authorization": []string{auth}, + "Dpop": []string{dpop}, + } + + return &model.Link{ + URL: previewURL, + Header: headers, + }, nil +} + +func parseSize(size string) int64 { + if size == "" { + return 0 + } + val, err := strconv.ParseInt(size, 10, 64) + if err != nil { + return 0 + } + return val +} + +var _ driver.Driver = (*DoubaoNew)(nil) diff --git a/drivers/doubao_new/meta.go b/drivers/doubao_new/meta.go new file mode 100644 index 00000000000..5a8050a2653 --- /dev/null +++ b/drivers/doubao_new/meta.go @@ -0,0 +1,36 @@ +package doubao_new + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootID + // define other + Authorization string `json:"authorization" help:"DPoP access token (Authorization header value); optional if present in cookie"` + Dpop string `json:"dpop" help:"DPoP header value; optional if present in cookie"` + Cookie string `json:"cookie" help:"Optional cookie; only used to extract authorization/dpop tokens"` + Debug bool `json:"debug" help:"Enable debug logs for upload"` +} + +var config = driver.Config{ + Name: "DoubaoNew", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &DoubaoNew{} + }) +} diff --git a/drivers/doubao_new/types.go b/drivers/doubao_new/types.go new file mode 100644 index 00000000000..ea8acfc5b18 --- /dev/null +++ b/drivers/doubao_new/types.go @@ -0,0 +1,182 @@ +package doubao_new + +import "github.com/alist-org/alist/v3/internal/model" + +type BaseResp struct { + Code int `json:"code"` + Msg string `json:"msg,omitempty"` + Message string `json:"message,omitempty"` +} + +type ListResp struct { + BaseResp + Data ListData `json:"data"` +} + +type ListData struct { + HasMore bool `json:"has_more"` + LastLabel string `json:"last_label"` + NodeList []string `json:"node_list"` + Entities struct { + Nodes map[string]Node `json:"nodes"` + Users map[string]User `json:"users"` + } `json:"entities"` +} + +type Node struct { + Token string `json:"token"` + NodeToken string `json:"node_token"` + ObjToken string `json:"obj_token"` + Name string `json:"name"` + Type int `json:"type"` + NodeType int `json:"node_type"` + OwnerID string `json:"owner_id"` + EditUID string `json:"edit_uid"` + CreateTime int64 `json:"create_time"` + EditTime int64 `json:"edit_time"` + URL string `json:"url"` + Extra struct { + Size string `json:"size"` + } `json:"extra"` +} + +type User struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type Object struct { + model.Object + ObjToken string + NodeType int + ObjType int + URL string +} + +type CreateFolderResp struct { + BaseResp + Data struct { + Entities struct { + Nodes map[string]Node `json:"nodes"` + } `json:"entities"` + NodeList []string `json:"node_list"` + } `json:"data"` +} + +type FileInfoResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data FileInfo `json:"data"` +} + +type FileInfo struct { + Name string `json:"name"` + NumBlocks int `json:"num_blocks"` + Version string `json:"version"` + MimeType string `json:"mime_type"` + MountPoint string `json:"mount_point"` + PreviewMeta PreviewMeta `json:"preview_meta"` +} + +type PreviewMeta struct { + Data map[string]PreviewMetaEntry `json:"data"` +} + +type PreviewMetaEntry struct { + Status int `json:"status"` + Extra string `json:"extra"` + PreviewFileSize int64 `json:"preview_file_size"` +} + +type PreviewImageExtra struct { + ImgExt string `json:"img_ext"` + PageNums int `json:"page_nums"` +} + +type UserStorageResp struct { + BaseResp + Data UserStorageData `json:"data"` +} + +type UserStorageData struct { + ShowSizeLimit bool `json:"show_size_limit"` + TotalSizeLimitBytes int64 `json:"total_size_limit_bytes"` + UsedSizeBytes int64 `json:"used_size_bytes"` +} + +type UploadPrepareResp struct { + BaseResp + Data UploadPrepareData `json:"data"` +} + +type UploadPrepareData struct { + BlockSize int64 `json:"block_size"` + NumBlocks int `json:"num_blocks"` + OptionBlockSize int64 `json:"option_block_size"` + DedupeSupport bool `json:"dedupe_support"` + UploadID string `json:"upload_id"` +} + +type UploadBlock struct { + Hash string `json:"hash"` + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + IsUploaded bool `json:"isUploaded"` +} + +type UploadBlocksResp struct { + BaseResp + Data UploadBlocksData `json:"data"` +} + +type UploadBlocksData struct { + NeededUploadBlocks []UploadBlockNeed `json:"needed_upload_blocks"` +} + +type UploadBlockNeed struct { + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + Hash string `json:"hash"` +} + +type UploadMergeResp struct { + BaseResp + Data UploadMergeData `json:"data"` +} + +type UploadMergeData struct { + SuccessSeqList []int `json:"success_seq_list"` +} + +type UploadFinishResp struct { + BaseResp + Data UploadFinishData `json:"data"` +} + +type UploadFinishData struct { + Version string `json:"version"` + DataVersion string `json:"data_version"` + Extra struct { + NodeToken string `json:"node_token"` + } `json:"extra"` + FileToken string `json:"file_token"` +} + +type RemoveResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + } `json:"data"` +} + +type TaskStatusResp struct { + BaseResp + Data TaskStatusData `json:"data"` +} + +type TaskStatusData struct { + IsFinish bool `json:"is_finish"` + IsFail bool `json:"is_fail"` +} diff --git a/drivers/doubao_new/util.go b/drivers/doubao_new/util.go new file mode 100644 index 00000000000..5c21ca090db --- /dev/null +++ b/drivers/doubao_new/util.go @@ -0,0 +1,909 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "hash/adler32" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/go-resty/resty/v2" +) + +const ( + BaseURL = "https://my.feishu.cn" + DownloadBaseURL = "https://internal-api-drive-stream.feishu.cn" +) + +var defaultObjTypes = []string{"124", "0", "12", "30", "123", "22"} + +func (d *DoubaoNew) request(ctx context.Context, path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + + if callback != nil { + callback(req) + } + + res, err := req.Execute(method, BaseURL+path) + if err != nil { + return nil, err + } + if res != nil { + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + } + + body := res.Body() + var common BaseResp + if err = json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return body, fmt.Errorf(msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return body, fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + if resp != nil { + if err = json.Unmarshal(body, resp); err != nil { + return body, err + } + } + + return body, nil +} + +func getCookieValue(cookie, name string) string { + parts := strings.Split(cookie, ";") + prefix := name + "=" + for _, part := range parts { + part = strings.TrimSpace(part) + if strings.HasPrefix(part, prefix) { + return strings.TrimPrefix(part, prefix) + } + } + return "" +} + +func adler32String(data []byte) string { + sum := adler32.Checksum(data) + return strconv.FormatUint(uint64(sum), 10) +} + +func buildCommaHeader(items []string) string { + return strings.Join(items, ",") +} + +func joinIntComma(items []int) string { + if len(items) == 0 { + return "" + } + var sb strings.Builder + for i, v := range items { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(strconv.Itoa(v)) + } + return sb.String() +} + +func previewList(items []string, n int) string { + if n <= 0 || len(items) == 0 { + return "" + } + if len(items) < n { + n = len(items) + } + return strings.Join(items[:n], ",") +} + +func (d *DoubaoNew) resolveAuthorization() string { + auth := strings.TrimSpace(d.Authorization) + if auth == "" && d.Cookie != "" { + if token := getCookieValue(d.Cookie, "LARK_SUITE_ACCESS_TOKEN"); token != "" { + auth = token + } + } + if auth == "" { + return "" + } + if !strings.HasPrefix(auth, "DPoP ") && !strings.HasPrefix(auth, "dpop ") { + auth = "DPoP " + auth + } + return auth +} + +func (d *DoubaoNew) resolveDpop() string { + dpop := strings.TrimSpace(d.Dpop) + if dpop == "" && d.Cookie != "" { + dpop = getCookieValue(d.Cookie, "LARK_SUITE_DPOP") + } + return dpop +} + +func (d *DoubaoNew) listChildren(ctx context.Context, parentToken string, lastLabel string) (ListData, error) { + var resp ListResp + _, err := d.request(ctx, "/space/api/explorer/doubao/children/list/", http.MethodGet, func(req *resty.Request) { + values := url.Values{} + for _, t := range defaultObjTypes { + values.Add("obj_type", t) + } + values.Set("length", "50") + values.Set("rank", "0") + values.Set("asc", "0") + values.Set("min_length", "40") + values.Set("thumbnail_width", "1028") + values.Set("thumbnail_height", "1028") + values.Set("thumbnail_policy", "4") + if parentToken != "" { + values.Set("token", parentToken) + } + if lastLabel != "" { + values.Set("last_label", lastLabel) + } + req.SetQueryParamsFromValues(values) + }, &resp) + if err != nil { + return ListData{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) getFileInfo(ctx context.Context, fileToken string) (FileInfo, error) { + var resp FileInfoResp + _, err := d.request(ctx, "/space/api/box/file/info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "caller": "explorer", + "file_token": fileToken, + "mount_point": "explorer", + "option_params": []string{"preview_meta", "check_cipher"}, + }) + }, &resp) + if err != nil { + return FileInfo{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) createFolder(ctx context.Context, parentToken, name string) (Node, error) { + data := url.Values{} + data.Set("name", name) + data.Set("source", "0") + if parentToken != "" { + data.Set("parent_token", parentToken) + } + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/create/folder/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return Node{}, err + } + if err := decodeBaseResp(body, res); err != nil { + return Node{}, err + } + + var resp CreateFolderResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return Node{}, fmt.Errorf(msg) + } + + var node Node + if len(resp.Data.NodeList) > 0 { + if n, ok := resp.Data.Entities.Nodes[resp.Data.NodeList[0]]; ok { + node = n + } + } + if node.Token == "" { + for _, n := range resp.Data.Entities.Nodes { + node = n + break + } + } + if node.Token == "" && node.ObjToken == "" && node.NodeToken == "" { + return Node{}, fmt.Errorf("[doubao_new] create folder failed: empty response") + } + if node.NodeToken == "" { + if node.Token != "" { + node.NodeToken = node.Token + } else if node.ObjToken != "" { + node.NodeToken = node.ObjToken + } + } + if node.ObjToken == "" && node.Token != "" { + node.ObjToken = node.Token + } + return node, nil +} + +func (d *DoubaoNew) renameFolder(ctx context.Context, token, name string) error { + if token == "" { + return fmt.Errorf("[doubao_new] rename folder missing token") + } + data := url.Values{} + data.Set("token", token) + data.Set("name", name) + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/rename/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func isCsrfTokenError(body []byte, res *resty.Response) bool { + if len(body) == 0 { + return false + } + if strings.Contains(strings.ToLower(string(body)), "csrf token error") { + return true + } + if res != nil && res.StatusCode() == http.StatusForbidden { + return true + } + return false +} + +func doRequestWithCsrf(doRequest func(csrfToken string) (*resty.Response, []byte, error)) (*resty.Response, []byte, error) { + res, body, err := doRequest("") + if err != nil { + return res, body, err + } + if isCsrfTokenError(body, res) { + csrfToken := extractCsrfTokenFromResponse(res) + if csrfToken != "" { + return doRequest(csrfToken) + } + } + return res, body, err +} + +func extractCsrfTokenFromResponse(res *resty.Response) string { + if res == nil || res.Request == nil { + return "" + } + if res.Request.RawRequest != nil { + if csrf := getCookieValue(res.Request.RawRequest.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + } + if csrf := getCookieValue(res.Request.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + for _, c := range res.Cookies() { + if c.Name == "_csrf_token" { + return c.Value + } + } + return "" +} + +func decodeBaseResp(body []byte, res *resty.Response) error { + var common BaseResp + if err := json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf(msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + return nil +} + +func (d *DoubaoNew) renameFile(ctx context.Context, fileToken, name string) error { + if fileToken == "" { + return fmt.Errorf("[doubao_new] rename file missing file token") + } + _, err := d.request(ctx, "/space/api/box/file/update_info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "file_token": fileToken, + "name": name, + }) + }, nil) + return err +} + +func (d *DoubaoNew) moveObj(ctx context.Context, srcToken, destToken string) error { + if srcToken == "" { + return fmt.Errorf("[doubao_new] move missing src token") + } + data := url.Values{} + data.Set("src_token", srcToken) + if destToken != "" { + data.Set("dest_token", destToken) + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/move/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func (d *DoubaoNew) removeObj(ctx context.Context, tokens []string) error { + if len(tokens) == 0 { + return fmt.Errorf("[doubao_new] remove missing tokens") + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "tokens": tokens, + "apply": 1, + }) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v3/remove/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + var resp RemoveResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + if resp.Data.TaskID == "" { + return nil + } + return d.waitTask(ctx, resp.Data.TaskID) +} + +func (d *DoubaoNew) getUserStorage(ctx context.Context) (UserStorageData, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("agw-js-conv", "str") + req.SetHeader("content-type", "application/json") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if d.Cookie != "" { + req.SetHeader("cookie", d.Cookie) + } + req.SetBody(base.Json{}) + + res, err := req.Execute(http.MethodPost, "https://www.doubao.com/alice/aispace/facade/get_user_storage") + if err != nil { + return UserStorageData{}, err + } + + body := res.Body() + var resp UserStorageResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UserStorageData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UserStorageData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) waitTask(ctx context.Context, taskID string) error { + const ( + taskPollInterval = time.Second + taskPollMaxAttempts = 120 + ) + var lastErr error + for attempt := 0; attempt < taskPollMaxAttempts; attempt++ { + if attempt > 0 { + if err := waitWithContext(ctx, taskPollInterval); err != nil { + return err + } + } + status, err := d.getTaskStatus(ctx, taskID) + if err != nil { + lastErr = err + continue + } + if status.IsFail { + return fmt.Errorf("[doubao_new] remove task failed: %s", taskID) + } + if status.IsFinish { + return nil + } + } + if lastErr != nil { + return lastErr + } + return fmt.Errorf("[doubao_new] remove task timed out: %s", taskID) +} + +func (d *DoubaoNew) getTaskStatus(ctx context.Context, taskID string) (TaskStatusData, error) { + if taskID == "" { + return TaskStatusData{}, fmt.Errorf("[doubao_new] task status missing task_id") + } + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + req.SetQueryParam("task_id", taskID) + res, err := req.Execute(http.MethodGet, BaseURL+"/space/api/explorer/v2/task/") + if err != nil { + return TaskStatusData{}, err + } + body := res.Body() + var resp TaskStatusResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return TaskStatusData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return TaskStatusData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + return resp.Data, nil +} + +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (d *DoubaoNew) prepareUpload(ctx context.Context, name string, size int64, mountNodeToken string) (UploadPrepareData, error) { + var resp UploadPrepareResp + _, err := d.request(ctx, "/space/api/box/upload/prepare/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.prepare") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + body := base.Json{ + "mount_point": "explorer", + "mount_node_token": "", + "name": name, + "size": size, + "size_checker": true, + } + if mountNodeToken != "" { + body["mount_node_token"] = mountNodeToken + } + req.SetBody(body) + }, &resp) + if err != nil { + return UploadPrepareData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlocks(ctx context.Context, uploadID string, blocks []UploadBlock, mountPoint string) (UploadBlocksData, error) { + if uploadID == "" { + return UploadBlocksData{}, fmt.Errorf("[doubao_new] upload blocks missing upload_id") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadBlocksResp + _, err := d.request(ctx, "/space/api/box/upload/blocks/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.blocks") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetBody(base.Json{ + "blocks": blocks, + "upload_id": uploadID, + "mount_point": mountPoint, + }) + }, &resp) + if err != nil { + return UploadBlocksData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) mergeUploadBlocks(ctx context.Context, uploadID string, seqList []int, checksumList []string, sizeList []int64, blockOriginSize int64, data []byte) (UploadMergeData, error) { + if uploadID == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing upload_id") + } + if len(seqList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty seq list") + } + if len(checksumList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty checksum list") + } + if len(sizeList) != len(seqList) { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks size list mismatch") + } + if blockOriginSize <= 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks invalid block origin size") + } + if len(data) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty data") + } + + seqHeader := joinIntComma(seqList) + checksumHeader := buildCommaHeader(checksumList) + + client := base.NewRestyClient() + client.SetCookieJar(nil) + req := client.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("content-type", "application/octet-stream") + req.Header.Set("x-block-list-checksum", checksumHeader) + req.Header.Set("x-seq-list", seqHeader) + req.SetHeader("x-block-origin-size", strconv.FormatInt(blockOriginSize, 10)) + req.SetHeader("x-command", "space.api.box.stream.upload.merge_block") + req.SetHeader("x-csrftoken", "") + reqID := "" + if buf := make([]byte, 16); true { + if _, err := rand.Read(buf); err == nil { + reqID = hex.EncodeToString(buf) + } + } + if reqID != "" { + req.SetHeader("x-request-id", reqID) + } + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + req.Header.Del("Cookie") + req.Header.Del("cookie") + if req.Header.Get("x-command") == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing x-command header") + } + req.SetBody(data) + + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + urlStr := "https://internal-api-drive-stream.feishu.cn/space/api/box/stream/upload/merge_block/?" + values.Encode() + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return UploadMergeData{}, err + } + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + body := res.Body() + var resp UploadMergeResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UploadMergeData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + if res != nil && res.StatusCode() == http.StatusBadRequest && resp.Code == 2 { + success := make([]int, 0, len(seqList)) + offset := 0 + for i, seq := range seqList { + size := sizeList[i] + if size <= 0 { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback invalid size: seq=%d size=%d", seq, size) + } + if offset+int(size) > len(data) { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback payload out of range: seq=%d offset=%d size=%d total=%d", seq, offset, size, len(data)) + } + payload := data[offset : offset+int(size)] + block := UploadBlockNeed{ + Seq: seq, + Size: size, + Checksum: checksumList[i], + } + if err := d.uploadBlockV3(ctx, uploadID, block, payload); err != nil { + return UploadMergeData{SuccessSeqList: success}, err + } + success = append(success, seq) + offset += int(size) + } + return UploadMergeData{SuccessSeqList: success}, nil + } + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UploadMergeData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlockV3(ctx context.Context, uploadID string, block UploadBlockNeed, data []byte) error { + if uploadID == "" { + return fmt.Errorf("[doubao_new] upload v3 block missing upload_id") + } + if block.Seq < 0 { + return fmt.Errorf("[doubao_new] upload v3 block invalid seq") + } + if len(data) == 0 { + return fmt.Errorf("[doubao_new] upload v3 block empty data") + } + + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("x-block-seq", strconv.Itoa(block.Seq)) + req.SetHeader("x-block-checksum", block.Checksum) + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + + req.SetMultipartFormData(map[string]string{ + "upload_id": uploadID, + "size": strconv.FormatInt(int64(len(data)), 10), + }) + req.SetMultipartField("file", "blob", "application/octet-stream", bytes.NewReader(data)) + + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("seq", strconv.Itoa(block.Seq)) + values.Set("size", strconv.FormatInt(int64(len(data)), 10)) + values.Set("checksum", block.Checksum) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + urlStr := "https://internal-api-drive-stream.feishu.cn/space/api/box/stream/upload/v3/block/?" + values.Encode() + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return err + } + body := res.Body() + if err := decodeBaseResp(body, res); err != nil { + return err + } + return nil +} + +func (d *DoubaoNew) finishUpload(ctx context.Context, uploadID string, numBlocks int, mountPoint string) (UploadFinishData, error) { + if uploadID == "" { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload missing upload_id") + } + if numBlocks <= 0 { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload invalid num_blocks") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadFinishResp + _, err := d.request(ctx, "/space/api/box/upload/finish/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.finish") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetHeader("biz-scene", "file_upload") + req.SetHeader("biz-ua-type", "Web") + req.SetBody(base.Json{ + "upload_id": uploadID, + "num_blocks": numBlocks, + "mount_point": mountPoint, + "push_open_history_record": 1, + }) + }, &resp) + if err != nil { + return UploadFinishData{}, err + } + return resp.Data, nil +} diff --git a/server/handles/down.go b/server/handles/down.go index 59c75530d3b..e93ed1eb103 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -55,6 +55,20 @@ func Proxy(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if c.Query("type") == "preview" && storage.GetStorage().Driver == "DoubaoNew" { + // Force proxy for DoubaoNew preview so headers are preserved. + link, file, err := fs.Link(c, rawPath, model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + localProxy(c, link, file, storage.GetStorage().ProxyRange) + return + } if canProxy(storage, filename) { downProxyUrl := storage.GetStorage().DownProxyUrl if downProxyUrl != "" { From 7323583af8c1f32b76cfc5270b7fd9b275ab6969 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 4 Feb 2026 13:20:11 +0800 Subject: [PATCH 068/133] chore(driver, archive): Remove debug logs and add missing `os` import --- drivers/doubao_new/driver.go | 2 -- internal/archive/tool/securepath.go | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/doubao_new/driver.go b/drivers/doubao_new/driver.go index 7846551790a..af0f9dd2ac0 100644 --- a/drivers/doubao_new/driver.go +++ b/drivers/doubao_new/driver.go @@ -304,8 +304,6 @@ func (d *DoubaoNew) Put(ctx context.Context, dstDir model.Obj, file model.FileSt if len(data) < tailLen { tailLen = len(data) } - fmt.Printf("head32 = %x\n", data[:headLen]) - fmt.Printf("tail32 = %x\n", data[len(data)-tailLen:]) } if int64(len(data)) != expectLen { return fmt.Errorf("[doubao_new] merge blocks invalid body len: got=%d expect=%d seqs=%v", len(data), expectLen, groupSeqs) diff --git a/internal/archive/tool/securepath.go b/internal/archive/tool/securepath.go index 45f5e749d53..f9bd89a914a 100644 --- a/internal/archive/tool/securepath.go +++ b/internal/archive/tool/securepath.go @@ -3,6 +3,7 @@ package tool import ( "errors" "fmt" + "os" "path" "path/filepath" "strings" From a0389e2777644e339716ff462c76179b698da199 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 4 Feb 2026 16:39:15 +0800 Subject: [PATCH 069/133] feat(iso9660): Refactor path handling for secure file extraction - Introduced `SecureJoin` to prevent path traversal vulnerabilities - Refactored `decompress` logic into `decompressEntry` for improved modularity - Ensured secure directory creation with `MkdirAll` in all operations --- internal/archive/iso9660/iso9660.go | 11 +++++++---- internal/archive/iso9660/utils.go | 26 +++++++++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/internal/archive/iso9660/iso9660.go b/internal/archive/iso9660/iso9660.go index be107d7b4c4..7de8da6f393 100644 --- a/internal/archive/iso9660/iso9660.go +++ b/internal/archive/iso9660/iso9660.go @@ -1,14 +1,14 @@ package iso9660 import ( + "io" + "os" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" "github.com/kdomanski/iso9660" - "io" - "os" - stdpath "path" ) type ISO9660 struct { @@ -78,7 +78,10 @@ func (ISO9660) Decompress(ss []*stream.SeekableStream, outputPath string, args m } if obj.IsDir() { if args.InnerPath != "/" { - outputPath = stdpath.Join(outputPath, obj.Name()) + outputPath, err = tool.SecureJoin(outputPath, obj.Name()) + if err != nil { + return err + } if err = os.MkdirAll(outputPath, 0700); err != nil { return err } diff --git a/internal/archive/iso9660/utils.go b/internal/archive/iso9660/utils.go index 0e4cfb1caf3..dd2e00354f8 100644 --- a/internal/archive/iso9660/utils.go +++ b/internal/archive/iso9660/utils.go @@ -1,10 +1,12 @@ package iso9660 import ( + "io" "os" - stdpath "path" + "path/filepath" "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" @@ -62,15 +64,26 @@ func toModelObj(file *iso9660.File) model.Obj { } func decompress(f *iso9660.File, path string, up model.UpdateProgress) error { - file, err := os.OpenFile(stdpath.Join(path, f.Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + return decompressEntry(f.Reader(), f.Size(), path, f.Name(), up) +} + +func decompressEntry(reader io.Reader, size int64, path, entryName string, up model.UpdateProgress) error { + dstPath, err := tool.SecureJoin(path, entryName) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + file, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } defer file.Close() _, err = utils.CopyWithBuffer(file, &stream.ReaderUpdatingProgress{ Reader: &stream.SimpleReaderWithSize{ - Reader: f.Reader(), - Size: f.Size(), + Reader: reader, + Size: size, }, UpdateProgress: up, }) @@ -84,7 +97,10 @@ func decompressAll(children []*iso9660.File, path string) error { if err != nil { return err } - nextPath := stdpath.Join(path, child.Name()) + nextPath, err := tool.SecureJoin(path, child.Name()) + if err != nil { + return err + } if err = os.MkdirAll(nextPath, 0700); err != nil { return err } From 1880b5afb8f8d47dc37f2b07b9366f533d4ae562 Mon Sep 17 00:00:00 2001 From: RedwindA Date: Sat, 7 Feb 2026 21:15:15 +0800 Subject: [PATCH 070/133] feat(drivers): add FTPS driver with TLS support - Implement FTPS (FTP over SSL/TLS) driver for secure file transfers - Support both Explicit (STARTTLS on port 21) and Implicit (direct TLS on port 990) TLS modes - Configurable TLS certificate verification skip for self-signed certificates - Character encoding support via mahonia library - Full CRUD operations: list, create directories, upload, download, move, rename, delete --- drivers/all.go | 1 + drivers/ftps/driver.go | 127 +++++++++++++++++++++++++++++++++++++++ drivers/ftps/meta.go | 46 ++++++++++++++ drivers/ftps/types.go | 1 + drivers/ftps/util.go | 132 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 307 insertions(+) create mode 100644 drivers/ftps/driver.go create mode 100644 drivers/ftps/meta.go create mode 100644 drivers/ftps/types.go create mode 100644 drivers/ftps/util.go diff --git a/drivers/all.go b/drivers/all.go index 3b0435595d2..c53ba6caddc 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -32,6 +32,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" + _ "github.com/alist-org/alist/v3/drivers/ftps" _ "github.com/alist-org/alist/v3/drivers/gitee" _ "github.com/alist-org/alist/v3/drivers/github" _ "github.com/alist-org/alist/v3/drivers/github_releases" diff --git a/drivers/ftps/driver.go b/drivers/ftps/driver.go new file mode 100644 index 00000000000..4467380f09c --- /dev/null +++ b/drivers/ftps/driver.go @@ -0,0 +1,127 @@ +package ftps + +import ( + "context" + stdpath "path" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/jlaffaye/ftp" +) + +type FTPS struct { + model.Storage + Addition + conn *ftp.ServerConn +} + +func (d *FTPS) Config() driver.Config { + return config +} + +func (d *FTPS) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *FTPS) Init(ctx context.Context) error { + return d.login() +} + +func (d *FTPS) Drop(ctx context.Context) error { + if d.conn != nil { + _ = d.conn.Logout() + } + return nil +} + +func (d *FTPS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.login(); err != nil { + return nil, err + } + entries, err := d.conn.List(encode(dir.GetPath(), d.Encoding)) + if err != nil { + return nil, err + } + res := make([]model.Obj, 0) + for _, entry := range entries { + if entry.Name == "." || entry.Name == ".." { + continue + } + f := model.Object{ + Name: decode(entry.Name, d.Encoding), + Size: int64(entry.Size), + Modified: entry.Time, + IsFolder: entry.Type == ftp.EntryTypeFolder, + } + res = append(res, &f) + } + return res, nil +} + +func (d *FTPS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.login(); err != nil { + return nil, err + } + r := NewFileReader(d.conn, encode(file.GetPath(), d.Encoding), file.GetSize()) + link := &model.Link{ + MFile: r, + } + return link, nil +} + +func (d *FTPS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.login(); err != nil { + return err + } + return d.conn.MakeDir(encode(stdpath.Join(parentDir.GetPath(), dirName), d.Encoding)) +} + +func (d *FTPS) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.login(); err != nil { + return err + } + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(dstDir.GetPath(), srcObj.GetName()), d.Encoding), + ) +} + +func (d *FTPS) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.login(); err != nil { + return err + } + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), d.Encoding), + ) +} + +func (d *FTPS) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *FTPS) Remove(ctx context.Context, obj model.Obj) error { + if err := d.login(); err != nil { + return err + } + path := encode(obj.GetPath(), d.Encoding) + if obj.IsDir() { + return d.conn.RemoveDirRecur(path) + } else { + return d.conn.Delete(path) + } +} + +func (d *FTPS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + if err := d.login(); err != nil { + return err + } + path := stdpath.Join(dstDir.GetPath(), s.GetName()) + return d.conn.Stor(encode(path, d.Encoding), driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + })) +} + +var _ driver.Driver = (*FTPS)(nil) diff --git a/drivers/ftps/meta.go b/drivers/ftps/meta.go new file mode 100644 index 00000000000..a752ec01aa6 --- /dev/null +++ b/drivers/ftps/meta.go @@ -0,0 +1,46 @@ +package ftps + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" + "github.com/axgle/mahonia" +) + +func encode(str string, encoding string) string { + if encoding == "" { + return str + } + encoder := mahonia.NewEncoder(encoding) + return encoder.ConvertString(str) +} + +func decode(str string, encoding string) string { + if encoding == "" { + return str + } + decoder := mahonia.NewDecoder(encoding) + return decoder.ConvertString(str) +} + +type Addition struct { + Address string `json:"address" required:"true"` + Encoding string `json:"encoding" required:"false"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + TLSMode string `json:"tls_mode" type:"select" options:"Explicit,Implicit" default:"Explicit" required:"true" help:"Explicit: STARTTLS on port 21; Implicit: direct TLS on port 990"` + TLSInsecureSkipVerify bool `json:"tls_insecure_skip_verify" default:"false" help:"Allow insecure TLS connections (e.g. self-signed certificates)"` + driver.RootPath +} + +var config = driver.Config{ + Name: "FTPS", + LocalSort: true, + OnlyLocal: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &FTPS{} + }) +} diff --git a/drivers/ftps/types.go b/drivers/ftps/types.go new file mode 100644 index 00000000000..d01dc417a16 --- /dev/null +++ b/drivers/ftps/types.go @@ -0,0 +1 @@ +package ftps diff --git a/drivers/ftps/util.go b/drivers/ftps/util.go new file mode 100644 index 00000000000..37591dcbb78 --- /dev/null +++ b/drivers/ftps/util.go @@ -0,0 +1,132 @@ +package ftps + +import ( + "crypto/tls" + "io" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/jlaffaye/ftp" +) + +func (d *FTPS) login() error { + if d.conn != nil { + _, err := d.conn.CurrentDir() + if err == nil { + return nil + } + } + + tlsConfig := &tls.Config{ + InsecureSkipVerify: d.TLSInsecureSkipVerify, + } + + opts := []ftp.DialOption{ + ftp.DialWithShutTimeout(10 * time.Second), + } + if d.TLSMode == "Implicit" { + opts = append(opts, ftp.DialWithTLS(tlsConfig)) + } else { + opts = append(opts, ftp.DialWithExplicitTLS(tlsConfig)) + } + + conn, err := ftp.Dial(d.Address, opts...) + if err != nil { + return err + } + err = conn.Login(d.Username, d.Password) + if err != nil { + _ = conn.Quit() + return err + } + d.conn = conn + return nil +} + +type FileReader struct { + conn *ftp.ServerConn + resp *ftp.Response + offset atomic.Int64 + readAtOffset int64 + mu sync.Mutex + path string + size int64 +} + +func NewFileReader(conn *ftp.ServerConn, path string, size int64) *FileReader { + return &FileReader{ + conn: conn, + path: path, + size: size, + } +} + +func (r *FileReader) Read(buf []byte) (n int, err error) { + r.mu.Lock() + defer r.mu.Unlock() + off := r.offset.Load() + n, err = r.readAtLocked(buf, off) + r.offset.Add(int64(n)) + return +} + +func (r *FileReader) ReadAt(buf []byte, off int64) (n int, err error) { + if off < 0 { + return 0, os.ErrInvalid + } + r.mu.Lock() + defer r.mu.Unlock() + return r.readAtLocked(buf, off) +} + +func (r *FileReader) readAtLocked(buf []byte, off int64) (n int, err error) { + if r.resp != nil && off != r.readAtOffset { + _ = r.resp.Close() + r.resp = nil + } + + if r.resp == nil { + r.resp, err = r.conn.RetrFrom(r.path, uint64(off)) + r.readAtOffset = off + if err != nil { + return 0, err + } + } + + n, err = r.resp.Read(buf) + r.readAtOffset += int64(n) + return +} + +func (r *FileReader) Seek(offset int64, whence int) (int64, error) { + oldOffset := r.offset.Load() + var newOffset int64 + switch whence { + case io.SeekStart: + newOffset = offset + case io.SeekCurrent: + newOffset = oldOffset + offset + case io.SeekEnd: + newOffset = r.size + offset + default: + return -1, os.ErrInvalid + } + + if newOffset < 0 { + return oldOffset, os.ErrInvalid + } + if newOffset == oldOffset { + return oldOffset, nil + } + r.offset.Store(newOffset) + return newOffset, nil +} + +func (r *FileReader) Close() error { + if r.resp != nil { + return r.resp.Close() + } + return nil +} From 426d594c5178e02cfa5ab5dd73dc76a13210094e Mon Sep 17 00:00:00 2001 From: RedwindA Date: Sat, 7 Feb 2026 21:29:20 +0800 Subject: [PATCH 071/133] fix(ftps): add host parsing for TLS configuration in login function --- drivers/ftps/util.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/ftps/util.go b/drivers/ftps/util.go index 37591dcbb78..10680fa8b80 100644 --- a/drivers/ftps/util.go +++ b/drivers/ftps/util.go @@ -3,6 +3,7 @@ package ftps import ( "crypto/tls" "io" + "net" "os" "sync" "sync/atomic" @@ -19,7 +20,13 @@ func (d *FTPS) login() error { } } + host, _, err := net.SplitHostPort(d.Address) + if err != nil { + host = d.Address + } + tlsConfig := &tls.Config{ + ServerName: host, InsecureSkipVerify: d.TLSInsecureSkipVerify, } From 39962d67f64847a647a7855f9783ea5513123c90 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Mon, 9 Feb 2026 14:08:56 +0800 Subject: [PATCH 072/133] feat(driver): Upgrade dependencies and enhance 115 cloud functionality - Updated `github.com/SheltonZhu/115driver` to v1.2.3-1 and `golang.org/x/oauth2` to v0.30.0 - Added support for `ThumbURL` handling in file objects, enabling thumbnail retrieval - Refactored 115 share operations with new methods for enhanced User-Agent configuration - Improved file fetching logic to include thumbnails and support paginated results - Consolidated download and snapshot logic with User-Agent handling for better compatibility --- drivers/115/driver.go | 8 +- drivers/115/types.go | 5 ++ drivers/115/util.go | 161 +++++++++++++++++++++--------------- drivers/115_share/driver.go | 20 +++-- drivers/115_share/utils.go | 101 +++++++++++++++++++++- go.mod | 6 +- go.sum | 8 +- 7 files changed, 222 insertions(+), 87 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 0dcb64d8284..60fe60e68e7 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -66,9 +66,11 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err := d.WaitLimit(ctx); err != nil { return nil, err } - userAgent := args.Header.Get("User-Agent") - downloadInfo, err := d. - DownloadWithUA(file.(*FileObj).PickCode, userAgent) + userAgent := "" + if args.Header != nil { + userAgent = args.Header.Get("User-Agent") + } + downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err } diff --git a/drivers/115/types.go b/drivers/115/types.go index 40b951d80ce..7a80e3ef047 100644 --- a/drivers/115/types.go +++ b/drivers/115/types.go @@ -12,6 +12,7 @@ var _ model.Obj = (*FileObj)(nil) type FileObj struct { driver.File + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -22,6 +23,10 @@ func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + type UploadResult struct { driver.BasicResp Data struct { diff --git a/drivers/115/util.go b/drivers/115/util.go index fc17fe3cebf..79d869178cb 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "net/url" "strconv" "strings" @@ -25,11 +24,28 @@ import ( "github.com/aliyun/aliyun-oss-go-sdk/oss" cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" - crypto "github.com/SheltonZhu/115driver/pkg/crypto/m115" driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/pkg/errors" ) +type fileInfoWithThumb struct { + driver115.FileInfo + ThumbURL string `json:"u"` +} + +type fileListRespWithThumb struct { + driver115.BasicResp + CategoryID driver115.IntString `json:"cid"` + Count int `json:"count"` + Offset int `json:"offset"` + Files []fileInfoWithThumb `json:"data"` +} + +type getFileInfoResponseWithThumb struct { + driver115.BasicResp + Files []*fileInfoWithThumb `json:"data"` +} + // var UserAgent = driver115.UA115Browser func (d *Pan115) login() error { var err error @@ -66,100 +82,113 @@ func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { if d.PageSize <= 0 { d.PageSize = driver115.FileListLimit } - files, err := d.client.ListWithLimit(fileId, d.PageSize, driver115.WithMultiUrls()) - if err != nil { - return nil, err + limit := d.PageSize + if limit > driver115.MaxDirPageLimit { + limit = driver115.MaxDirPageLimit } - for _, file := range *files { - res = append(res, FileObj{file}) + + opts := driver115.DefaultListOptions() + driver115.WithMultiUrls()(opts) + if len(opts.ApiURLs) == 0 { + opts.ApiURLs = []string{driver115.ApiFileList} } + + offset := int64(0) + for i := 0; ; i++ { + result, err := d.getFilesPageWithThumb(fileId, opts.ApiURLs[i%len(opts.ApiURLs)], limit, offset) + if err != nil { + return nil, err + } + for _, fileInfo := range result.Files { + res = append(res, fileObjFromInfo(&fileInfo)) + } + offset = int64(result.Offset) + limit + if offset >= int64(result.Count) { + break + } + } + return res, nil } func (d *Pan115) getNewFile(fileId string) (*FileObj, error) { - file, err := d.client.GetFile(fileId) + fileInfo, err := d.getFileInfoWithThumb("file_id", fileId) if err != nil { return nil, err } - return &FileObj{*file}, nil + file := fileObjFromInfo(fileInfo) + return &file, nil } func (d *Pan115) getNewFileByPickCode(pickCode string) (*FileObj, error) { - result := driver115.GetFileInfoResponse{} - req := d.client.NewRequest(). - SetQueryParam("pick_code", pickCode). - ForceContentType("application/json;charset=UTF-8"). - SetResult(&result) - resp, err := req.Get(driver115.ApiFileInfo) - if err := driver115.CheckErr(err, &result, resp); err != nil { + fileInfo, err := d.getFileInfoWithThumb("pick_code", pickCode) + if err != nil { return nil, err } - if len(result.Files) == 0 { - return nil, errors.New("not get file info") - } - fileInfo := result.Files[0] - - f := &FileObj{} - f.From(fileInfo) - return f, nil + file := fileObjFromInfo(fileInfo) + return &file, nil } func (d *Pan115) getUA() string { return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } -func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { - key := crypto.GenerateKey() - result := driver115.DownloadResp{} - params, err := utils.Json.Marshal(map[string]string{"pick_code": pickCode}) - if err != nil { - return nil, err - } - - data := crypto.Encode(params, key) - - bodyReader := strings.NewReader(url.Values{"data": []string{data}}.Encode()) - reqUrl := fmt.Sprintf("%s?t=%s", driver115.AndroidApiDownloadGetUrl, driver115.Now().String()) - req, _ := http.NewRequest(http.MethodPost, reqUrl, bodyReader) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Cookie", d.Cookie) - req.Header.Set("User-Agent", ua) - - resp, err := d.client.Client.GetClient().Do(req) - if err != nil { - return nil, err +func fileObjFromInfo(fileInfo *fileInfoWithThumb) FileObj { + file := &driver115.File{} + file.From(&fileInfo.FileInfo) + return FileObj{ + File: *file, + ThumbURL: fileInfo.ThumbURL, } - defer resp.Body.Close() +} - body, err := io.ReadAll(resp.Body) - if err != nil { +func (d *Pan115) getFileInfoWithThumb(queryKey, queryVal string) (*fileInfoWithThumb, error) { + result := getFileInfoResponseWithThumb{} + req := d.client.NewRequest(). + SetQueryParam(queryKey, queryVal). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&result) + resp, err := req.Get(driver115.ApiFileInfo) + if err := driver115.CheckErr(err, &result, resp); err != nil { return nil, err } - if err := utils.Json.Unmarshal(body, &result); err != nil { - return nil, err + if len(result.Files) == 0 { + return nil, errors.New("not get file info") } + return result.Files[0], nil +} - if err = result.Err(string(body)); err != nil { - return nil, err +func (d *Pan115) getFilesPageWithThumb(dirID, apiURL string, limit, offset int64) (*fileListRespWithThumb, error) { + if dirID == "" { + dirID = "0" + } + result := fileListRespWithThumb{} + params := map[string]string{ + "aid": "1", + "cid": dirID, + "o": driver115.FileOrderByTime, + "asc": "1", + "offset": strconv.FormatInt(offset, 10), + "show_dir": "1", + "limit": strconv.FormatInt(limit, 10), + "snap": "0", + "natsort": "0", + "record_open_time": "1", + "format": "json", + "fc_mix": "0", } - - b, err := crypto.Decode(string(result.EncodedData), key) - if err != nil { + req := d.client.NewRequest(). + ForceContentType("application/json;charset=UTF-8"). + SetQueryParams(params). + SetResult(&result) + resp, err := req.Get(apiURL) + if err := driver115.CheckErr(err, &result, resp); err != nil { return nil, err } - - downloadInfo := struct { - Url string `json:"url"` - }{} - if err := utils.Json.Unmarshal(b, &downloadInfo); err != nil { - return nil, err + if dirID != string(result.CategoryID) { + return nil, driver115.ErrUnexpected } - - info := &driver115.DownloadInfo{} - info.PickCode = pickCode - info.Header = resp.Request.Header - info.Url.Url = downloadInfo.Url - return info, nil + return &result, nil } func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index 886a369c1b8..322b64afd45 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -4,6 +4,7 @@ import ( "context" driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -50,8 +51,9 @@ func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListAr return nil, err } - files := make([]driver115.ShareFile, 0) - fileResp, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) + ua := base.UserAgent + files := make([]shareFile, 0) + fileResp, err := d.getShareSnapWithUA(ua, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) if err != nil { return nil, err } @@ -59,10 +61,7 @@ func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListAr total := fileResp.Data.Count count := len(fileResp.Data.List) for total > count { - fileResp, err := d.client.GetShareSnap( - d.ShareCode, d.ReceiveCode, dir.GetID(), - driver115.QueryLimit(int(d.PageSize)), driver115.QueryOffset(count), - ) + fileResp, err := d.getShareSnapWithUA(ua, dir.GetID(), driver115.QueryLimit(int(d.PageSize)), driver115.QueryOffset(count)) if err != nil { return nil, err } @@ -77,7 +76,14 @@ func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkA if err := d.WaitLimit(ctx); err != nil { return nil, err } - downloadInfo, err := d.client.DownloadByShareCode(d.ShareCode, d.ReceiveCode, file.GetID()) + ua := "" + if args.Header != nil { + ua = args.Header.Get("User-Agent") + } + if ua == "" { + ua = base.UserAgent + } + downloadInfo, err := d.downloadByShareCodeWithUA(ua, file.GetID()) if err != nil { return nil, err } diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go index 1f9e112deef..e36a5ef8ccb 100644 --- a/drivers/115_share/utils.go +++ b/drivers/115_share/utils.go @@ -6,6 +6,7 @@ import ( "time" driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" @@ -20,6 +21,7 @@ type FileObj struct { FileName string isDir bool FileID string + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -54,7 +56,39 @@ func (f *FileObj) GetPath() string { return "" } -func transFunc(sf driver115.ShareFile) (model.Obj, error) { +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + +type shareFile struct { + FileID string `json:"fid"` + UID int `json:"uid"` + CategoryID driver115.IntString `json:"cid"` + FileName string `json:"n"` + Type string `json:"ico"` + Sha1 string `json:"sha"` + Size driver115.StringInt64 `json:"s"` + Labels []*driver115.LabelInfo `json:"fl"` + UpdateTime string `json:"t"` + IsFile int `json:"fc"` + ParentID string `json:"pid"` + ThumbURL string `json:"u"` +} + +type shareSnapResp struct { + driver115.BasicResp + Data struct { + Count int `json:"count"` + List []shareFile `json:"list"` + } `json:"data"` +} + +type downloadShareResp struct { + driver115.BasicResp + Data driver115.SharedDownloadInfo `json:"data"` +} + +func transFunc(sf shareFile) (model.Obj, error) { timeInt, err := strconv.ParseInt(sf.UpdateTime, 10, 64) if err != nil { return nil, err @@ -74,18 +108,77 @@ func transFunc(sf driver115.ShareFile) (model.Obj, error) { FileName: string(sf.FileName), isDir: isDir, FileID: fileID, + ThumbURL: sf.ThumbURL, }, nil } -var UserAgent = driver115.UA115Browser +func buildShareReferer(shareCode, receiveCode string) string { + return fmt.Sprintf("https://115cdn.com/s/%s?password=%s&", shareCode, receiveCode) +} + +func (d *Pan115Share) getShareSnapWithUA(ua, dirID string, queries ...driver115.Query) (*shareSnapResp, error) { + result := shareSnapResp{} + query := map[string]string{ + "share_code": d.ShareCode, + "receive_code": d.ReceiveCode, + "cid": dirID, + "limit": "20", + "asc": "0", + "offset": "0", + "format": "json", + } + for _, q := range queries { + q(&query) + } + + req := d.client.NewRequest(). + SetQueryParams(query). + SetHeader("referer", buildShareReferer(d.ShareCode, d.ReceiveCode)). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&result) + if ua != "" { + req = req.SetHeader("User-Agent", ua) + } + + resp, err := req.Get(driver115.ApiShareSnap) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + return &result, nil +} + +func (d *Pan115Share) downloadByShareCodeWithUA(ua, fileID string) (*driver115.SharedDownloadInfo, error) { + result := downloadShareResp{} + params := map[string]string{ + "share_code": d.ShareCode, + "receive_code": d.ReceiveCode, + "file_id": fileID, + "dl": "1", + } + + req := d.client.NewRequest(). + SetQueryParams(params). + ForceContentType("application/json"). + SetHeader("referer", buildShareReferer(d.ShareCode, d.ReceiveCode)). + SetResult(&result) + if ua != "" { + req = req.SetHeader("User-Agent", ua) + } + + resp, err := req.Get(driver115.ApiDownloadGetShareUrl) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + return &result.Data, nil +} func (d *Pan115Share) login() error { var err error opts := []driver115.Option{ - driver115.UA(UserAgent), + driver115.UA(base.UserAgent), } d.client = driver115.New(opts...) - if _, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, ""); err != nil { + if _, err = d.getShareSnapWithUA(base.UserAgent, ""); err != nil { return errors.Wrap(err, "failed to get share snap") } cr := &driver115.Credential{} diff --git a/go.mod b/go.mod index 5772806a7b4..2e202bc9de0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 github.com/ProtonMail/gopenpgp/v2 v2.7.4 - github.com/SheltonZhu/115driver v1.1.2 + github.com/SheltonZhu/115driver v1.2.3-1 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 @@ -74,7 +74,7 @@ require ( golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 golang.org/x/net v0.38.0 - golang.org/x/oauth2 v0.22.0 + golang.org/x/oauth2 v0.30.0 golang.org/x/time v0.8.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 @@ -286,4 +286,4 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/SheltonZhu/115driver => github.com/okatu-loli/115driver v1.1.2 +replace github.com/SheltonZhu/115driver => github.com/okatu-loli/115driver v1.2.3-1 diff --git a/go.sum b/go.sum index 4b21f881331..af3a6b3f96c 100644 --- a/go.sum +++ b/go.sum @@ -524,8 +524,8 @@ github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= -github.com/okatu-loli/115driver v1.1.2 h1:XZT3r/51SZRQGzre2IeA+0/k4T1FneqArdhE4Wd600Q= -github.com/okatu-loli/115driver v1.1.2/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= +github.com/okatu-loli/115driver v1.2.3-1 h1:UoBEREqh6RD6WlxiJ2Z29JxNZ/UcoChvdHn9r9Tx7nI= +github.com/okatu-loli/115driver v1.2.3-1/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= @@ -797,8 +797,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= -golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 3bc7de34ee53153eade8532ed212d70bc1273860 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 10 Feb 2026 16:46:57 +0800 Subject: [PATCH 073/133] feat(auth): Add consistent error messaging for invalid login credentials - Introduced `invalidLoginCredentialsMsg` constant for reusability - Updated error responses in login flow to provide a consistent message --- server/handles/auth.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/handles/auth.go b/server/handles/auth.go index 3520a459c3d..f59b4fb1ed7 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -26,8 +26,9 @@ import ( var loginCache = cache.NewMemCache[int]() var ( - defaultDuration = time.Minute * 5 - defaultTimes = 5 + defaultDuration = time.Minute * 5 + defaultTimes = 5 + invalidLoginCredentialsMsg = "username or password is incorrect" ) type LoginReq struct { @@ -69,13 +70,13 @@ func loginHash(c *gin.Context, req *LoginReq) { // check username user, err := op.GetUserByName(req.Username) if err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, invalidLoginCredentialsMsg, 400) loginCache.Set(ip, count+1) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, invalidLoginCredentialsMsg, 400) loginCache.Set(ip, count+1) return } From 338569fdc253979cb87fb31be27c68ff399fd755 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 20 Feb 2026 22:01:17 +0800 Subject: [PATCH 074/133] feat(fs): Add pagination support and enhance response structure - Introduced new pagination fields: `page`, `per_page`, `has_more`, and `pages_total` in `FsListResp` - Added `normalizeListPage` and `calcPagesTotal` for effective pagination handling - Updated default and max page size values; implemented validations for page parameters - Refactored `FsList` handler to include paginated responses and filtered totals - Adjusted default page size in site settings from 30 to 50 --- internal/bootstrap/data/setting.go | 2 +- server/handles/fsread.go | 81 +++++++++++++++++++++++------- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index bbb633e33a6..c5541fa38d4 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -100,7 +100,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.SiteTitle, Value: "AList", Type: conf.TypeString, Group: model.SITE}, {Key: conf.Announcement, Value: "### repo\nhttps://github.com/alist-org/alist", Type: conf.TypeText, Group: model.SITE}, {Key: "pagination_type", Value: "all", Type: conf.TypeSelect, Options: "all,pagination,load_more,auto_load_more", Group: model.SITE}, - {Key: "default_page_size", Value: "30", Type: conf.TypeNumber, Group: model.SITE}, + {Key: "default_page_size", Value: "50", Type: conf.TypeNumber, Group: model.SITE}, {Key: conf.AllowIndexed, Value: "false", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 15dd9f1ce7e..eb984698b16 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -49,12 +49,17 @@ type ObjResp struct { } type FsListResp struct { - Content []ObjLabelResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` + Content []ObjLabelResp `json:"content"` + Total int64 `json:"total"` + FilteredTotal int64 `json:"filtered_total"` + Page int `json:"page"` + PerPage int `json:"per_page"` + HasMore bool `json:"has_more"` + PagesTotal int `json:"pages_total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + Provider string `json:"provider"` } type ObjLabelResp struct { @@ -74,13 +79,20 @@ type ObjLabelResp struct { StorageClass string `json:"storage_class,omitempty"` } +const ( + DefaultPerPage = 200 + MaxPerPage = 500 +) + func FsList(c *gin.Context) { var req ListReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } - req.Validate() + effPage, effPerPage := normalizeListPage(req.Page, req.PerPage) + req.Page = effPage + req.PerPage = effPerPage user := c.MustGet("user").(*model.User) reqPath, err := user.JoinPath(req.Path) if err != nil { @@ -104,6 +116,11 @@ func FsList(c *gin.Context) { common.ErrorStrResp(c, "Refresh without permission", 403) return } + provider := "unknown" + storage, storageErr := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if storageErr == nil { + provider = storage.GetStorage().Driver + } objs, err := fs.List(c, reqPath, &fs.ListArgs{Refresh: req.Refresh}) if err != nil { common.ErrorResp(c, err, 500) @@ -116,19 +133,23 @@ func FsList(c *gin.Context) { filtered = append(filtered, obj) } } - total, objs := pagination(filtered, &req.PageReq) - provider := "unknown" - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) - if err == nil { - provider = storage.GetStorage().Driver - } + total, pageObjs := pagination(filtered, &req.PageReq) + respContent := toObjsResp(pageObjs, reqPath, isEncrypt(meta, reqPath)) + pagesTotal := calcPagesTotal(total, req.PerPage) + hasMore := req.Page*req.PerPage < total + common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), - Total: int64(total), - Readme: getReadme(meta, reqPath), - Header: getHeader(meta, reqPath), - Write: common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, reqPath), - Provider: provider, + Content: respContent, + Total: int64(total), + FilteredTotal: int64(total), + Page: req.Page, + PerPage: req.PerPage, + HasMore: hasMore, + PagesTotal: pagesTotal, + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Write: common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, reqPath), + Provider: provider, }) } @@ -226,6 +247,28 @@ func isEncrypt(meta *model.Meta, path string) bool { return true } +func normalizeListPage(page, perPage int) (int, int) { + effPage := page + if effPage <= 0 { + effPage = 1 + } + effPerPage := perPage + if effPerPage <= 0 { + effPerPage = DefaultPerPage + } + if effPerPage > MaxPerPage { + effPerPage = MaxPerPage + } + return effPage, effPerPage +} + +func calcPagesTotal(total, perPage int) int { + if total <= 0 || perPage <= 0 { + return 0 + } + return (total + perPage - 1) / perPage +} + func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { pageIndex, pageSize := req.Page, req.PerPage total := len(objs) From 3f9933a0c275956dca2a25e8730b77a1b2bb465b Mon Sep 17 00:00:00 2001 From: Stanislav Chernov Date: Fri, 20 Feb 2026 23:23:12 +0300 Subject: [PATCH 075/133] fix: unescape URL path to handle '#' in filenames Fixes AlistGo/alist#9361 --- server/middlewares/down.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/middlewares/down.go b/server/middlewares/down.go index d015672de80..7e67663b74b 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -1,6 +1,7 @@ package middlewares import ( + "net/url" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -41,9 +42,8 @@ func Down(verifyFunc func(string, string) error) func(c *gin.Context) { } } -// TODO: implement -// path maybe contains # ? etc. func parsePath(path string) string { + path, _ = url.PathUnescape(path) return utils.FixAndCleanPath(path) } From cf01ff6f4e40253a5bbccb082a0f1b982c630849 Mon Sep 17 00:00:00 2001 From: Sky_slience Date: Sun, 1 Mar 2026 00:38:25 +0800 Subject: [PATCH 076/133] feat(thumbnail): add configurable thumbnail size setting and update thumbnail generation --- drivers/local/driver.go | 13 +++++++++++++ drivers/local/util.go | 4 ++-- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 7ff72d11cdf..5ba82786622 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -18,6 +18,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" @@ -31,6 +32,7 @@ type Local struct { model.Storage Addition mkdirPerm int32 + thumbSize int // zero means no limit thumbConcurrency int @@ -71,6 +73,17 @@ func (d *Local) Init(ctx context.Context) error { return err } } + d.thumbSize = 144 + if item, err := op.GetSettingItemByKey(conf.ThumbnailSize); err == nil && item != nil && strings.TrimSpace(item.Value) != "" { + v, err := strconv.ParseUint(item.Value, 10, 32) + if err != nil { + return fmt.Errorf("invalid setting %s value: %s, err: %s", conf.ThumbnailSize, item.Value, err) + } + if v == 0 { + return fmt.Errorf("invalid setting %s value: %s, the value must be a positive integer", conf.ThumbnailSize, item.Value) + } + d.thumbSize = int(v) + } if d.ThumbConcurrency != "" { v, err := strconv.ParseUint(d.ThumbConcurrency, 10, 32) if err != nil { diff --git a/drivers/local/util.go b/drivers/local/util.go index b9df717fb40..5a4a2be9107 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -111,7 +111,7 @@ func readDir(dirname string) ([]fs.FileInfo, error) { func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { fullPath := file.GetPath() thumbPrefix := "alist_thumb_" - thumbName := thumbPrefix + utils.GetMD5EncodeStr(fullPath) + ".png" + thumbName := thumbPrefix + utils.GetMD5EncodeStr(fmt.Sprintf("%s:%d", fullPath, d.thumbSize)) + ".png" if d.ThumbCacheFolder != "" { // skip if the file is a thumbnail if strings.HasPrefix(file.GetName(), thumbPrefix) { @@ -142,7 +142,7 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { if err != nil { return nil, nil, err } - thumbImg := imaging.Resize(image, 144, 0, imaging.Lanczos) + thumbImg := imaging.Resize(image, d.thumbSize, 0, imaging.Lanczos) var buf bytes.Buffer err = imaging.Encode(&buf, thumbImg, imaging.PNG) if err != nil { diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index bbb633e33a6..006580888c6 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -146,6 +146,7 @@ func InitialSettings() []model.SettingItem { {Key: "audio_cover", Value: "https://jsd.nn.ci/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW}, {Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.ThumbnailSize, Value: "144", Type: conf.TypeNumber, Group: model.PREVIEW, Help: "Thumbnail width in pixels. Height is scaled proportionally."}, {Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 1a5581633a0..db2c84dc4ae 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -33,6 +33,7 @@ const ( ProxyIgnoreHeaders = "proxy_ignore_headers" AudioAutoplay = "audio_autoplay" VideoAutoplay = "video_autoplay" + ThumbnailSize = "thumbnail_size" PreviewArchivesByDefault = "preview_archives_by_default" ReadMeAutoRender = "readme_autorender" FilterReadMeScripts = "filter_readme_scripts" From 7dc8231e177bb1e602e18de5f4d18a113c4b6f30 Mon Sep 17 00:00:00 2001 From: Sky_slience Date: Sun, 1 Mar 2026 17:10:51 +0800 Subject: [PATCH 077/133] fix(rename): block rename for password-protected paths --- server/handles/fsbatch.go | 6 ++++++ server/handles/fsmanage.go | 22 +++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 35cec645e1b..bccbee72d29 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -195,6 +195,9 @@ func FsBatchRename(c *gin.Context) { common.ErrorResp(c, err, 400) return } + if !canRenamePath(c, filePath) { + return + } if err := fs.Rename(c, filePath, renameObject.NewName); err != nil { common.ErrorResp(c, err, 500) return @@ -261,6 +264,9 @@ func FsRegexRename(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if !canRenamePath(c, filePath) { + return + } newFileName := srcRegexp.ReplaceAllString(file.GetName(), req.NewNameRegex) if err := utils.ValidateNameComponent(newFileName); err != nil { common.ErrorResp(c, err, 400) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index dcb5a7b9a09..31976edd9ad 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -2,10 +2,11 @@ package handles import ( "fmt" - "github.com/alist-org/alist/v3/internal/task" "io" stdpath "path" + "github.com/alist-org/alist/v3/internal/task" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" @@ -213,6 +214,22 @@ type RenameReq struct { Overwrite bool `json:"overwrite"` } +func canRenamePath(c *gin.Context, reqPath string) bool { + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return false + } + return true + } + if meta != nil && meta.Password != "" && common.IsApply(meta.Path, reqPath, meta.PSub) { + common.ErrorStrResp(c, "Path is password-protected and cannot be renamed.", 403) + return false + } + return true +} + func FsRename(c *gin.Context) { var req RenameReq if err := c.ShouldBind(&req); err != nil { @@ -229,6 +246,9 @@ func FsRename(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } + if !canRenamePath(c, reqPath) { + return + } perm := common.MergeRolePermissions(user, reqPath) if !common.HasPermission(perm, common.PermRename) { common.ErrorResp(c, errs.PermissionDenied, 403) From d4cc6efd4ef6dce7e832a43c13d1de280aca6f43 Mon Sep 17 00:00:00 2001 From: hgkdzbf6 Date: Fri, 6 Mar 2026 16:14:23 +0800 Subject: [PATCH 078/133] fix: reduce WebDAV logging for NotFoundError Skip logging full error stack trace for NotFoundError when a file doesn't exist, as this is a normal case and not a program error. Use DEBUG level instead of ERROR to reduce log noise. Fixes #9141 --- server/webdav.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/webdav.go b/server/webdav.go index ac520070686..d188cb8010d 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -9,6 +9,7 @@ import ( "path" "strings" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/server/middlewares" @@ -31,6 +32,12 @@ func WebDav(dav *gin.RouterGroup) { Prefix: path.Join(conf.URL.Path, "/dav"), LockSystem: webdav.NewMemLS(), Logger: func(request *http.Request, err error) { + // Skip logging for NotFoundError as it's not a program error + // but a normal case when a file doesn't exist + if errs.IsNotFoundError(err) { + log.Debugf("%s %s %v", request.Method, request.URL.Path, err) + return + } log.Errorf("%s %s %+v", request.Method, request.URL.Path, err) }, } From 1175f8ac3e2896ef714b0d3c0fdf1e3c1748e381 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 6 Mar 2026 21:44:36 +0800 Subject: [PATCH 079/133] feat: add streamtape driver --- drivers/all.go | 1 + drivers/streamtape/driver.go | 400 +++++++++++++++++++++++++++++++++++ drivers/streamtape/meta.go | 38 ++++ drivers/streamtape/types.go | 54 +++++ drivers/streamtape/util.go | 146 +++++++++++++ 5 files changed, 639 insertions(+) create mode 100644 drivers/streamtape/driver.go create mode 100644 drivers/streamtape/meta.go create mode 100644 drivers/streamtape/types.go create mode 100644 drivers/streamtape/util.go diff --git a/drivers/all.go b/drivers/all.go index c53ba6caddc..b40a7fd677a 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -66,6 +66,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/seafile" _ "github.com/alist-org/alist/v3/drivers/sftp" _ "github.com/alist-org/alist/v3/drivers/smb" + _ "github.com/alist-org/alist/v3/drivers/streamtape" _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go new file mode 100644 index 00000000000..042df42cbe6 --- /dev/null +++ b/drivers/streamtape/driver.go @@ -0,0 +1,400 @@ +package streamtape + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + log "github.com/sirupsen/logrus" +) + +type Streamtape struct { + model.Storage + Addition +} + +var waitMoreSecondsRe = regexp.MustCompile(`wait\s+(\d+)\s+more\s+seconds?`) + +func (d *Streamtape) Config() driver.Config { + return config +} + +func (d *Streamtape) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Streamtape) Init(ctx context.Context) error { + if strings.TrimSpace(d.APILogin) == "" || strings.TrimSpace(d.APIKey) == "" { + return errors.New("api_login and api_key are required") + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + + var account accountInfo + if err := d.callAPI(ctx, "/account/info", nil, &account); err != nil { + return err + } + + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Streamtape) Drop(ctx context.Context) error { + return nil +} + +func (d *Streamtape) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := d.RootFolderID + if dir.GetID() != "" { + folderID = folderIDFromObjID(dir.GetID()) + } + + params := map[string]string{} + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + + var result listFolderResult + if err := d.callAPI(ctx, "/file/listfolder", params, &result); err != nil { + return nil, err + } + + objects := make([]model.Obj, 0, len(result.Folders)+len(result.Files)) + for _, f := range result.Folders { + objects = append(objects, &model.Object{ + ID: encodeFolderID(f.ID), + Name: f.Name, + IsFolder: true, + }) + } + for _, f := range result.Files { + objects = append(objects, buildFileObj(f)) + } + return objects, nil +} + +func (d *Streamtape) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + fileID := fileIDFromObjID(file.GetID()) + if fileID == "" { + return nil, errors.New("empty file id") + } + + var ticket dlTicketResult + if err := d.callAPI(ctx, "/file/dlticket", map[string]string{"file": fileID}, &ticket); err != nil { + return nil, err + } + + var dl dlResult + waitSeconds := ticket.WaitTime + if waitSeconds > 0 { + timer := time.NewTimer(time.Duration(waitSeconds+1) * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + + var err error + for i := 0; i < 3; i++ { + err = d.callAPI(ctx, "/file/dl", map[string]string{ + "file": fileID, + "ticket": ticket.Ticket, + }, &dl) + if err == nil { + break + } + waitSeconds = extractWaitSecondsFromErr(err) + if waitSeconds <= 0 { + return nil, err + } + timer := time.NewTimer(time.Duration(waitSeconds+1) * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + if err != nil { + return nil, err + } + + finalURL := ensureStreamQuery(dl.URL) + log.Infof("streamtape direct link file=%s url=%s", fileID, finalURL) + link := &model.Link{ + URL: finalURL, + Header: http.Header{ + "Referer": []string{"https://streamtape.com/"}, + "Origin": []string{"https://streamtape.com"}, + }, + } + d.applyRangeStrategy(link, file.GetSize()) + return link, nil +} + +func extractWaitSecondsFromErr(err error) int { + if err == nil { + return 0 + } + matches := waitMoreSecondsRe.FindStringSubmatch(strings.ToLower(err.Error())) + if len(matches) < 2 { + return 0 + } + seconds, convErr := strconv.Atoi(matches[1]) + if convErr != nil || seconds < 0 { + return 0 + } + return seconds +} + +func ensureStreamQuery(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + q := u.Query() + if q.Get("stream") == "" { + q.Set("stream", "1") + u.RawQuery = q.Encode() + } + return u.String() +} + +func (d *Streamtape) applyRangeStrategy(link *model.Link, size int64) { + if !d.EnableRangeControl || size <= 0 { + return + } + + mode := strings.ToLower(strings.TrimSpace(d.RangeMode)) + if mode == "" { + mode = "chunk" + } + + switch mode { + case "full": + // Keep single full-tail behavior while still using ranged requests. + link.Concurrency = 1 + link.PartSize = int(size) + case "percent": + percent := d.RangePercent + if percent <= 0 { + percent = 15 + } + if percent > 100 { + percent = 100 + } + partSize := size * int64(percent) / 100 + if partSize < 1*1024*1024 { + partSize = 1 * 1024 * 1024 + } + if partSize > size { + partSize = size + } + link.Concurrency = 1 + link.PartSize = int(partSize) + default: + chunkMB := d.RangeChunkMB + if chunkMB <= 0 { + chunkMB = 8 + } + partSize := int64(chunkMB) * 1024 * 1024 + if partSize > size { + partSize = size + } + concurrency := d.RangeConcurrency + if concurrency <= 0 { + concurrency = 4 + } + link.Concurrency = concurrency + link.PartSize = int(partSize) + } +} + +func (d *Streamtape) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + pid := d.RootFolderID + if parentDir.GetID() != "" { + pid = folderIDFromObjID(parentDir.GetID()) + } + + params := map[string]string{"name": dirName} + if pid != "" && pid != "0" { + params["pid"] = pid + } + + var result createFolderResult + if err := d.callAPI(ctx, "/file/createfolder", params, &result); err != nil { + return nil, err + } + + return &model.Object{ + ID: encodeFolderID(result.FolderID), + Name: dirName, + IsFolder: true, + }, nil +} + +func (d *Streamtape) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, errs.NotImplement + } + fileID := fileIDFromObjID(srcObj.GetID()) + if fileID == "" { + return nil, errors.New("empty file id") + } + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + if folderID == "" || folderID == "0" { + return nil, fmt.Errorf("streamtape move to root is not supported by API") + } + + if err := d.callAPI(ctx, "/file/move", map[string]string{ + "file": fileID, + "folder": folderID, + }, nil); err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: false, + }, nil +} + +func (d *Streamtape) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + endpoint := "/file/rename" + params := map[string]string{"name": newName} + if srcObj.IsDir() { + endpoint = "/file/renamefolder" + params["folder"] = folderIDFromObjID(srcObj.GetID()) + } else { + params["file"] = fileIDFromObjID(srcObj.GetID()) + } + + if err := d.callAPI(ctx, endpoint, params, nil); err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Streamtape) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) Remove(ctx context.Context, obj model.Obj) error { + endpoint := "/file/delete" + params := map[string]string{} + if obj.IsDir() { + endpoint = "/file/deletefolder" + params["folder"] = folderIDFromObjID(obj.GetID()) + } else { + params["file"] = fileIDFromObjID(obj.GetID()) + } + return d.callAPI(ctx, endpoint, params, nil) +} + +func (d *Streamtape) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + params := map[string]string{} + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + + var uploadURL uploadURLResult + if err := d.callAPI(ctx, "/file/ul", params, &uploadURL); err != nil { + return nil, err + } + + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }) + + res, err := base.RestyClient.R(). + SetContext(ctx). + SetFileReader("file1", file.GetName(), reader). + Post(uploadURL.URL) + if err != nil { + return nil, err + } + if res.StatusCode() >= http.StatusBadRequest { + return nil, fmt.Errorf("streamtape upload failed: http %d", res.StatusCode()) + } + + uploadedID := extractFileIDFromUploadBody(res.Body()) + if uploadedID == "" { + list, listErr := d.List(ctx, &model.Object{ID: encodeFolderID(folderID), IsFolder: true}, model.ListArgs{}) + if listErr == nil { + for _, obj := range list { + if obj.IsDir() { + continue + } + if obj.GetName() == file.GetName() && (file.GetSize() <= 0 || obj.GetSize() == file.GetSize()) { + return obj, nil + } + } + } + return &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil + } + + return &model.Object{ + ID: encodeFileID(uploadedID), + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +func (d *Streamtape) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*Streamtape)(nil) diff --git a/drivers/streamtape/meta.go b/drivers/streamtape/meta.go new file mode 100644 index 00000000000..50b09f14d41 --- /dev/null +++ b/drivers/streamtape/meta.go @@ -0,0 +1,38 @@ +package streamtape + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APILogin string `json:"api_login" required:"true" help:"API Login from Streamtape account settings"` + APIKey string `json:"api_key" required:"true" help:"API Key from Streamtape account settings"` + RangeMode string `json:"range_mode" type:"select" options:"chunk,full,percent" default:"chunk" help:"Range strategy for preview: chunk=bounded ranges, full=single full-tail range, percent=part size by file percentage"` + RangeChunkMB int `json:"range_chunk_mb" type:"number" default:"8" help:"Chunk mode part size in MB"` + RangeConcurrency int `json:"range_concurrency" type:"number" default:"4" help:"Chunk mode concurrent upstream requests"` + RangePercent int `json:"range_percent" type:"number" default:"15" help:"Percent mode part size percentage (1-100)"` + EnableRangeControl bool `json:"enable_range_control" default:"true" help:"Enable driver-level range shaping for smoother streaming"` +} + +var config = driver.Config{ + Name: "Streamtape", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: true, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, + ProxyRangeOption: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Streamtape{} + }) +} diff --git a/drivers/streamtape/types.go b/drivers/streamtape/types.go new file mode 100644 index 00000000000..4e470ff927e --- /dev/null +++ b/drivers/streamtape/types.go @@ -0,0 +1,54 @@ +package streamtape + +import "encoding/json" + +type apiResponse struct { + Status int `json:"status"` + Msg string `json:"msg"` + Result json.RawMessage `json:"result"` +} + +type accountInfo struct { + APIID string `json:"apiid"` + Email string `json:"email"` + SignupAt string `json:"signup_at"` +} + +type listFolderResult struct { + Folders []folderItem `json:"folders"` + Files []fileItem `json:"files"` +} + +type folderItem struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type fileItem struct { + Name string `json:"name"` + Size int64 `json:"size"` + Link string `json:"link"` + CreatedAt int64 `json:"created_at"` + Downloads int64 `json:"downloads"` + LinkID string `json:"linkid"` + Convert string `json:"convert"` +} + +type dlTicketResult struct { + Ticket string `json:"ticket"` + WaitTime int `json:"wait_time"` +} + +type dlResult struct { + Name string `json:"name"` + Size int64 `json:"size"` + URL string `json:"url"` +} + +type createFolderResult struct { + FolderID string `json:"folderid"` +} + +type uploadURLResult struct { + URL string `json:"url"` +} diff --git a/drivers/streamtape/util.go b/drivers/streamtape/util.go new file mode 100644 index 00000000000..f6d8373763e --- /dev/null +++ b/drivers/streamtape/util.go @@ -0,0 +1,146 @@ +package streamtape + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" +) + +const apiBase = "https://api.streamtape.com" + +func (d *Streamtape) callAPI(ctx context.Context, endpoint string, params map[string]string, out any) error { + query := map[string]string{ + "login": d.APILogin, + "key": d.APIKey, + } + for k, v := range params { + if strings.TrimSpace(v) == "" { + continue + } + query[k] = v + } + + var resp apiResponse + r, err := base.RestyClient.R(). + SetContext(ctx). + SetQueryParams(query). + SetResult(&resp). + Get(apiBase + endpoint) + if err != nil { + return err + } + if r.StatusCode() != http.StatusOK { + return fmt.Errorf("streamtape http error: %d", r.StatusCode()) + } + if resp.Status != 200 { + return fmt.Errorf("streamtape api error: status=%d msg=%s", resp.Status, resp.Msg) + } + if out == nil || len(resp.Result) == 0 || string(resp.Result) == "null" { + return nil + } + if err := json.Unmarshal(resp.Result, out); err != nil { + return fmt.Errorf("decode streamtape result failed: %w", err) + } + return nil +} + +func folderIDFromObjID(id string) string { + if id == "" || id == "0" || id == "/" { + return "0" + } + if strings.HasPrefix(id, "d:") { + return strings.TrimPrefix(id, "d:") + } + return id +} + +func fileIDFromObjID(id string) string { + if strings.HasPrefix(id, "f:") { + return strings.TrimPrefix(id, "f:") + } + return id +} + +func encodeFolderID(id string) string { + if id == "" || id == "0" || id == "/" { + return "d:0" + } + return "d:" + id +} + +func encodeFileID(id string) string { + if strings.HasPrefix(id, "f:") { + return id + } + return "f:" + id +} + +func extractFileIDFromLink(link string) string { + if link == "" { + return "" + } + u, err := url.Parse(link) + if err != nil { + return "" + } + parts := strings.Split(strings.Trim(path.Clean(u.Path), "/"), "/") + for i := 0; i < len(parts)-1; i++ { + if parts[i] == "v" { + return parts[i+1] + } + } + return "" +} + +func buildFileObj(f fileItem) model.Obj { + id := f.LinkID + if id == "" { + id = extractFileIDFromLink(f.Link) + } + mod := time.Now() + if f.CreatedAt > 0 { + mod = time.Unix(f.CreatedAt, 0) + } + return &model.Object{ + ID: encodeFileID(id), + Name: f.Name, + Size: f.Size, + Modified: mod, + IsFolder: false, + } +} + +func extractFileIDFromUploadBody(body []byte) string { + if len(body) == 0 { + return "" + } + + var resp apiResponse + if err := json.Unmarshal(body, &resp); err != nil { + return "" + } + if resp.Status != 200 || len(resp.Result) == 0 { + return "" + } + + var result map[string]any + if err := json.Unmarshal(resp.Result, &result); err != nil { + return "" + } + for _, key := range []string{"file", "fileid", "id", "linkid"} { + if v, ok := result[key]; ok { + if s, ok := v.(string); ok && s != "" { + return s + } + } + } + return "" +} From 5773c637e2a5331596cf1311165f3f01fd73888a Mon Sep 17 00:00:00 2001 From: Zhongwen Luo <75424880+Muione@users.noreply.github.com> Date: Sat, 7 Mar 2026 10:32:29 +0000 Subject: [PATCH 080/133] fix: handle nil buffer case in resizeImageToBufferWithFFmpegGo --- drivers/local/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/local/util.go b/drivers/local/util.go index 6c59911b4c8..fa95ddd24a0 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -80,7 +80,7 @@ func resizeImageToBufferWithFFmpegGo(inputFile string, width int, outputFormat s if err != nil { return nil, fmt.Errorf("ffmpeg-go failed to resize image %s to buffer: %w", inputFile, err) } - if outBuffer.Len() == 0 { + if outBuffer == nil || outBuffer.Len() == 0 { return nil, fmt.Errorf("ffmpeg-go produced empty buffer for %s", inputFile) } From 5852a81773e506323710d4a41cf3d8a2c70942b2 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Mon, 9 Mar 2026 19:45:16 +0800 Subject: [PATCH 081/133] feat(strm): add STRM driver with independent sync flow - implement STRM mount and generate behavior with alias mapping - support local save modes: insert, update, sync - keep config compatibility and remove previous intermediate history --- drivers/all.go | 1 + drivers/strm/driver.go | 237 ++++++++++++++++++++++++++++++++ drivers/strm/hook.go | 3 + drivers/strm/meta.go | 42 ++++++ drivers/strm/util.go | 305 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 588 insertions(+) create mode 100644 drivers/strm/driver.go create mode 100644 drivers/strm/hook.go create mode 100644 drivers/strm/meta.go create mode 100644 drivers/strm/util.go diff --git a/drivers/all.go b/drivers/all.go index c53ba6caddc..a9130555fd6 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -66,6 +66,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/seafile" _ "github.com/alist-org/alist/v3/drivers/sftp" _ "github.com/alist-org/alist/v3/drivers/smb" + _ "github.com/alist-org/alist/v3/drivers/strm" _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go new file mode 100644 index 00000000000..c0f3a27d0ac --- /dev/null +++ b/drivers/strm/driver.go @@ -0,0 +1,237 @@ +package strm + +import ( + "context" + "errors" + stdpath "path" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" +) + +type Strm struct { + model.Storage + Addition + + aliases map[string][]string + autoFlatten bool + singleRootKey string + + mediaExtSet map[string]struct{} + downloadExtSet map[string]struct{} + normalizedMode string + normalizedPrefix string +} + +func (d *Strm) Config() driver.Config { + return config +} + +func (d *Strm) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Strm) Init(ctx context.Context) error { + if strings.TrimSpace(d.Paths) == "" { + return errors.New("paths is required") + } + if d.SaveStrmToLocal && strings.TrimSpace(d.SaveStrmLocalPath) == "" { + return errors.New("SaveStrmLocalPath is required") + } + + d.aliases = parseAliases(d.Paths) + if len(d.aliases) == 0 { + return errors.New("no valid path mapping found") + } + + d.autoFlatten = len(d.aliases) == 1 + d.singleRootKey = "" + if d.autoFlatten { + for k := range d.aliases { + d.singleRootKey = k + } + } + + d.mediaExtSet = parseExtSet(defaultIfEmpty(d.FilterFileTypes, defaultMediaExt)) + d.downloadExtSet = parseExtSet(defaultIfEmpty(d.DownloadFileTypes, defaultDownloadExt)) + d.normalizedPrefix = normalizePrefix(defaultIfEmpty(d.PathPrefix, "/d")) + d.normalizedMode = normalizeSaveMode(d.SaveLocalMode) + + if d.Version != 5 { + d.FilterFileTypes = mergeDefaultExtCSV(d.FilterFileTypes, defaultMediaExt) + d.DownloadFileTypes = mergeDefaultExtCSV(d.DownloadFileTypes, defaultDownloadExt) + d.PathPrefix = "/d" + d.Version = 5 + } + if d.SaveLocalMode == "" { + d.SaveLocalMode = SaveLocalInsertMode + } + return nil +} + +func (d *Strm) Drop(ctx context.Context) error { + d.aliases = nil + d.mediaExtSet = nil + d.downloadExtSet = nil + return nil +} + +func (Addition) GetRootPath() string { + return "/" +} + +func (d *Strm) Get(ctx context.Context, path string) (model.Obj, error) { + path = cleanPath(path) + root, sub := d.splitVirtualPath(path) + targets, ok := d.aliases[root] + if !ok { + return nil, errs.ObjectNotFound + } + + for _, targetRoot := range targets { + realPath := stdpath.Join(targetRoot, sub) + obj, err := fs.Get(ctx, realPath, &fs.GetArgs{NoLog: true}) + if err != nil { + continue + } + if obj.IsDir() { + return wrapObj(path, obj, 0), nil + } + return wrapObj(realPath, obj, obj.GetSize()), nil + } + + if strings.HasSuffix(strings.ToLower(path), ".strm") { + return nil, errs.NotSupport + } + return nil, errs.ObjectNotFound +} + +func (d *Strm) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + virtualDir := cleanPath(dir.GetPath()) + if virtualDir == "/" && !d.autoFlatten { + objs := d.listVirtualRoots() + d.syncLocalDir(ctx, virtualDir, objs) + return objs, nil + } + + root, sub := d.splitVirtualPath(virtualDir) + targets, ok := d.aliases[root] + if !ok { + return nil, errs.ObjectNotFound + } + + out := make([]model.Obj, 0) + for _, targetRoot := range targets { + realDir := stdpath.Join(targetRoot, sub) + objs, err := fs.List(ctx, realDir, &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) + if err != nil { + continue + } + out = append(out, d.mapListedObjects(ctx, realDir, objs)...) + } + + d.syncLocalDir(ctx, virtualDir, out) + return out, nil +} + +func (d *Strm) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.GetID() == "strm" { + line := d.buildStrmLine(ctx, file.GetPath()) + return &model.Link{MFile: model.NewNopMFile(strings.NewReader(line + "\n"))}, nil + } + return d.linkRealFile(ctx, file.GetPath(), args) +} + +func (d *Strm) listVirtualRoots() []model.Obj { + objs := make([]model.Obj, 0, len(d.aliases)) + for k := range d.aliases { + objs = append(objs, &model.Object{ + Path: "/" + k, + Name: k, + IsFolder: true, + Modified: d.Modified, + }) + } + return objs +} + +func (d *Strm) mapListedObjects(ctx context.Context, realDir string, listed []model.Obj) []model.Obj { + ret := make([]model.Obj, 0, len(listed)) + for _, obj := range listed { + if obj.IsDir() { + ret = append(ret, &model.Object{ + Name: obj.GetName(), + Path: "", + IsFolder: true, + Modified: obj.ModTime(), + }) + continue + } + + realPath := stdpath.Join(realDir, obj.GetName()) + ext := fileExt(obj.GetName()) + + if _, ok := d.downloadExtSet[ext]; ok { + ret = append(ret, d.cloneWithPath(obj, realPath, obj.GetName(), "", obj.GetSize())) + continue + } + if _, ok := d.mediaExtSet[ext]; ok { + strmName := strings.TrimSuffix(obj.GetName(), stdpath.Ext(obj.GetName())) + ".strm" + size := int64(len(d.buildStrmLine(ctx, realPath)) + 1) + ret = append(ret, d.cloneWithPath(obj, realPath, strmName, "strm", size)) + } + } + return ret +} + +func (d *Strm) cloneWithPath(src model.Obj, realPath, name, id string, size int64) model.Obj { + baseObj := model.Object{ + ID: id, + Path: realPath, + Name: name, + Size: size, + Modified: src.ModTime(), + IsFolder: src.IsDir(), + } + thumb, ok := model.GetThumb(src) + if !ok { + return &baseObj + } + return &model.ObjThumb{Object: baseObj, Thumbnail: model.Thumbnail{Thumbnail: thumb}} +} + +func (d *Strm) splitVirtualPath(path string) (string, string) { + if d.autoFlatten { + return d.singleRootKey, path + } + trimmed := strings.TrimPrefix(path, "/") + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) == 1 { + return parts[0], "" + } + return parts[0], parts[1] +} + +func cleanPath(path string) string { + if path == "" { + return "/" + } + return filepath.ToSlash(stdpath.Clean("/" + strings.TrimPrefix(path, "/"))) +} + +func wrapObj(path string, src model.Obj, size int64) model.Obj { + return &model.Object{ + Path: path, + Name: src.GetName(), + Size: size, + Modified: src.ModTime(), + IsFolder: src.IsDir(), + HashInfo: src.GetHash(), + } +} + +var _ driver.Driver = (*Strm)(nil) diff --git a/drivers/strm/hook.go b/drivers/strm/hook.go new file mode 100644 index 00000000000..6fad6993c67 --- /dev/null +++ b/drivers/strm/hook.go @@ -0,0 +1,3 @@ +package strm + +// Local sync is triggered during STRM directory listing. diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go new file mode 100644 index 00000000000..ac757e9ebad --- /dev/null +++ b/drivers/strm/meta.go @@ -0,0 +1,42 @@ +package strm + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +const ( + SaveLocalInsertMode = "insert" + SaveLocalUpdateMode = "update" + SaveLocalSyncMode = "sync" +) + +type Addition struct { + Paths string `json:"paths" required:"true" type:"text"` + SiteUrl string `json:"siteUrl" type:"text" required:"false" help:"The prefix URL of generated strm file"` + PathPrefix string `json:"PathPrefix" type:"text" required:"false" default:"/d" help:"Path prefix in strm content"` + DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Extensions to download as local files"` + FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Extensions to expose as .strm"` + EncodePath bool `json:"encodePath" default:"true" required:"true" help:"Encode path in strm content"` + WithoutUrl bool `json:"withoutUrl" default:"false" help:"Generate path-only strm content"` + WithSign bool `json:"withSign" default:"false" help:"Append sign query to generated URL"` + SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"Save generated files to local disk"` + SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"Local path for generated files"` + SaveLocalMode string `json:"SaveLocalMode" type:"select" help:"Local save mode" options:"insert,update,sync" default:"insert"` + Version int +} + +var config = driver.Config{ + Name: "Strm", + LocalSort: true, + OnlyProxy: true, + NoCache: true, + NoUpload: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Strm{Addition: Addition{EncodePath: true}} + }) +} diff --git a/drivers/strm/util.go b/drivers/strm/util.go new file mode 100644 index 00000000000..4cba1769d27 --- /dev/null +++ b/drivers/strm/util.go @@ -0,0 +1,305 @@ +package strm + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + stdpath "path" + "path/filepath" + "sort" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + pkgerr "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +const ( + defaultMediaExt = "mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" + defaultDownloadExt = "ass,srt,vtt,sub,strm" +) + +func parseAliases(raw string) map[string][]string { + aliases := map[string][]string{} + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + name, target := parseAliasLine(line) + aliases[name] = append(aliases[name], cleanPath(target)) + } + return aliases +} + +func parseAliasLine(line string) (string, string) { + if strings.Contains(line, ":") { + parts := strings.SplitN(line, ":", 2) + if !strings.Contains(parts[0], "/") { + return parts[0], parts[1] + } + } + return stdpath.Base(line), line +} + +func parseExtSet(csv string) map[string]struct{} { + ret := map[string]struct{}{} + for _, part := range strings.Split(csv, ",") { + ext := normalizeExt(part) + if ext != "" { + ret[ext] = struct{}{} + } + } + return ret +} + +func mergeDefaultExtCSV(csv, defaults string) string { + base := parseExtSet(csv) + for ext := range parseExtSet(defaults) { + base[ext] = struct{}{} + } + keys := make([]string, 0, len(base)) + for k := range base { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ",") +} + +func normalizeExt(ext string) string { + ext = strings.ToLower(strings.TrimSpace(ext)) + ext = strings.TrimPrefix(ext, ".") + return ext +} + +func fileExt(name string) string { + return normalizeExt(stdpath.Ext(name)) +} + +func defaultIfEmpty(v, fallback string) string { + if strings.TrimSpace(v) == "" { + return fallback + } + return v +} + +func normalizeSaveMode(mode string) string { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "sync": + return SaveLocalSyncMode + case "update": + return SaveLocalUpdateMode + case "insert", "missing": + return SaveLocalInsertMode + default: + return SaveLocalInsertMode + } +} + +func normalizePrefix(prefix string) string { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + return "/d" + } + if !strings.HasPrefix(prefix, "/") { + prefix = "/" + prefix + } + return prefix +} + +func (d *Strm) buildStrmLine(ctx context.Context, realPath string) string { + pathPart := realPath + if d.EncodePath { + pathPart = utils.EncodePath(pathPart, true) + } + if d.WithSign { + sep := "?" + if strings.Contains(pathPart, "?") { + sep = "&" + } + pathPart += sep + "sign=" + sign.Sign(realPath) + } + joined := stdpath.Join(d.normalizedPrefix, pathPart) + if !strings.HasPrefix(joined, "/") { + joined = "/" + joined + } + if d.WithoutUrl { + return joined + } + baseURL := strings.TrimSpace(d.SiteUrl) + if baseURL == "" { + if c, ok := ctx.(*gin.Context); ok { + baseURL = common.GetApiUrl(c.Request) + } else { + baseURL = common.GetApiUrl(nil) + } + } + baseURL = strings.TrimSuffix(baseURL, "/") + return baseURL + joined +} + +func (d *Strm) linkRealFile(ctx context.Context, realPath string, args model.LinkArgs) (*model.Link, error) { + storage, actualPath, err := op.GetStorageAndActualPath(realPath) + if err != nil { + return nil, err + } + if !args.Redirect { + link, _, linkErr := op.Link(ctx, storage, actualPath, args) + return link, linkErr + } + obj, err := fs.Get(ctx, realPath, &fs.GetArgs{NoLog: true}) + if err != nil { + return nil, err + } + if common.ShouldProxy(storage, obj.GetName()) { + api := common.GetApiUrl(args.HttpReq) + if api == "" { + api = strings.TrimSuffix(strings.TrimSpace(d.SiteUrl), "/") + } + if api == "" { + api = common.GetApiUrl(nil) + } + return &model.Link{URL: fmt.Sprintf("%s/p%s?sign=%s", api, utils.EncodePath(realPath, true), sign.Sign(realPath))}, nil + } + link, _, linkErr := op.Link(ctx, storage, actualPath, args) + return link, linkErr +} + +func (d *Strm) syncLocalDir(ctx context.Context, virtualDir string, objs []model.Obj) { + if !d.SaveStrmToLocal || strings.TrimSpace(d.SaveStrmLocalPath) == "" { + return + } + baseDir := filepath.Clean(d.SaveStrmLocalPath) + localDir := baseDir + if virtualDir != "/" { + localDir = filepath.Join(baseDir, filepath.FromSlash(strings.TrimPrefix(virtualDir, "/"))) + } + if err := os.MkdirAll(localDir, 0o755); err != nil { + log.Warnf("strm: mkdir failed %s: %v", localDir, err) + return + } + + expected := map[string]bool{} + for _, obj := range objs { + name := obj.GetName() + expected[name] = obj.IsDir() + localPath := filepath.Join(localDir, name) + if obj.IsDir() { + _ = os.MkdirAll(localPath, 0o755) + continue + } + payload, err := d.localPayload(ctx, obj) + if err != nil { + log.Warnf("strm: build local payload failed %s: %v", localPath, err) + continue + } + if err = d.writeLocal(localPath, payload); err != nil { + log.Warnf("strm: write local failed %s: %v", localPath, err) + } + } + + if d.normalizedMode == SaveLocalSyncMode { + d.syncDeleteExtras(localDir, expected) + } +} + +func (d *Strm) localPayload(ctx context.Context, obj model.Obj) ([]byte, error) { + if obj.GetID() == "strm" { + return []byte(d.buildStrmLine(ctx, obj.GetPath()) + "\n"), nil + } + link, err := d.linkRealFile(ctx, obj.GetPath(), model.LinkArgs{Redirect: true}) + if err != nil { + return nil, err + } + return readLinkBytes(ctx, link) +} + +func readLinkBytes(ctx context.Context, link *model.Link) ([]byte, error) { + if link.MFile != nil { + defer link.MFile.Close() + return io.ReadAll(link.MFile) + } + if link.RangeReadCloser != nil { + rc, err := link.RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) + if err == nil && rc != nil { + defer rc.Close() + return io.ReadAll(rc) + } + } + if link.URL == "" { + return nil, fmt.Errorf("empty link") + } + url := link.URL + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + api := common.GetApiUrl(nil) + if api == "" { + return nil, fmt.Errorf("relative url without site url: %s", url) + } + url = strings.TrimSuffix(api, "/") + url + } + res, err := base.RestyClient.R().SetContext(ctx).SetDoNotParseResponse(true).Get(url) + if err != nil { + return nil, err + } + defer res.RawBody().Close() + if res.StatusCode() >= http.StatusBadRequest { + return nil, fmt.Errorf("read url failed: status=%d", res.StatusCode()) + } + return io.ReadAll(res.RawBody()) +} + +func (d *Strm) writeLocal(path string, payload []byte) error { + if d.normalizedMode == SaveLocalInsertMode && utils.Exists(path) { + return nil + } + if st, err := os.Stat(path); err == nil && st.IsDir() { + if d.normalizedMode != SaveLocalSyncMode { + return nil + } + if err = os.RemoveAll(path); err != nil { + return err + } + } + if d.normalizedMode != SaveLocalInsertMode { + if old, err := os.ReadFile(path); err == nil { + if bytes.Equal(old, payload) { + return nil + } + } + } + f, err := utils.CreateNestedFile(path) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(payload) + return err +} + +func (d *Strm) syncDeleteExtras(localDir string, expected map[string]bool) { + entries, err := os.ReadDir(localDir) + if err != nil { + if pkgerr.Cause(err) != os.ErrNotExist { + log.Warnf("strm: read local dir failed %s: %v", localDir, err) + } + return + } + for _, e := range entries { + expectDir, ok := expected[e.Name()] + full := filepath.Join(localDir, e.Name()) + if !ok || expectDir != e.IsDir() { + _ = os.RemoveAll(full) + } + } +} From 866d4f4c3551849c679b46bd13ac7d613519be34 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 10 Mar 2026 11:27:46 +0800 Subject: [PATCH 082/133] feat(strm): add driver-level sign expiry and rotate action - add SignExpireHours for STRM-specific sign duration - add RotateSignNow to trigger immediate local STRM rewrite - auto reset rotate flag after scheduling background rotation --- drivers/strm/driver.go | 46 ++++++++++++++++++++++++++++++++++++++++++ drivers/strm/meta.go | 2 ++ drivers/strm/util.go | 28 +++++++++++++++++-------- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go index c0f3a27d0ac..175a558dfcc 100644 --- a/drivers/strm/driver.go +++ b/drivers/strm/driver.go @@ -11,6 +11,8 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + log "github.com/sirupsen/logrus" ) type Strm struct { @@ -70,6 +72,20 @@ func (d *Strm) Init(ctx context.Context) error { if d.SaveLocalMode == "" { d.SaveLocalMode = SaveLocalInsertMode } + if d.SignExpireHours < 0 { + d.SignExpireHours = 0 + } + if d.RotateSignNow { + d.RotateSignNow = false + op.MustSaveDriverStorage(d) + if d.SaveStrmToLocal && strings.TrimSpace(d.SaveStrmLocalPath) != "" { + go func() { + log.Infof("strm: start rotating signs for [%s]", d.MountPath) + d.rotateAllLocal(context.Background()) + log.Infof("strm: finished rotating signs for [%s]", d.MountPath) + }() + } + } return nil } @@ -159,6 +175,36 @@ func (d *Strm) listVirtualRoots() []model.Obj { return objs } +func (d *Strm) rotateAllLocal(ctx context.Context) { + for alias, roots := range d.aliases { + virtualRoot := "/" + if !d.autoFlatten { + virtualRoot = "/" + alias + } + for _, realRoot := range roots { + d.walkAndSync(ctx, virtualRoot, realRoot) + } + } +} + +func (d *Strm) walkAndSync(ctx context.Context, virtualDir, realDir string) { + objs, err := fs.List(ctx, realDir, &fs.ListArgs{NoLog: true, Refresh: true}) + if err != nil { + log.Warnf("strm: rotate list failed %s: %v", realDir, err) + return + } + mapped := d.mapListedObjects(ctx, realDir, objs) + d.syncLocalDirWithMode(ctx, virtualDir, mapped, SaveLocalUpdateMode) + for _, obj := range objs { + if !obj.IsDir() { + continue + } + childVirtual := stdpath.Join(virtualDir, obj.GetName()) + childReal := stdpath.Join(realDir, obj.GetName()) + d.walkAndSync(ctx, childVirtual, childReal) + } +} + func (d *Strm) mapListedObjects(ctx context.Context, realDir string, listed []model.Obj) []model.Obj { ret := make([]model.Obj, 0, len(listed)) for _, obj := range listed { diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go index ac757e9ebad..803ed035762 100644 --- a/drivers/strm/meta.go +++ b/drivers/strm/meta.go @@ -20,6 +20,8 @@ type Addition struct { EncodePath bool `json:"encodePath" default:"true" required:"true" help:"Encode path in strm content"` WithoutUrl bool `json:"withoutUrl" default:"false" help:"Generate path-only strm content"` WithSign bool `json:"withSign" default:"false" help:"Append sign query to generated URL"` + SignExpireHours int `json:"SignExpireHours" type:"number" default:"0" help:"Driver-level sign expiration in hours. 0 uses global link_expiration"` + RotateSignNow bool `json:"RotateSignNow" type:"bool" default:"false" help:"Set true and save to rotate signs now (rewrite local STRM), then auto reset to false"` SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"Save generated files to local disk"` SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"Local path for generated files"` SaveLocalMode string `json:"SaveLocalMode" type:"select" help:"Local save mode" options:"insert,update,sync" default:"insert"` diff --git a/drivers/strm/util.go b/drivers/strm/util.go index 4cba1769d27..2d640bc32fb 100644 --- a/drivers/strm/util.go +++ b/drivers/strm/util.go @@ -11,6 +11,7 @@ import ( "path/filepath" "sort" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/fs" @@ -128,7 +129,7 @@ func (d *Strm) buildStrmLine(ctx context.Context, realPath string) string { if strings.Contains(pathPart, "?") { sep = "&" } - pathPart += sep + "sign=" + sign.Sign(realPath) + pathPart += sep + "sign=" + d.generateSign(realPath) } joined := stdpath.Join(d.normalizedPrefix, pathPart) if !strings.HasPrefix(joined, "/") { @@ -170,13 +171,17 @@ func (d *Strm) linkRealFile(ctx context.Context, realPath string, args model.Lin if api == "" { api = common.GetApiUrl(nil) } - return &model.Link{URL: fmt.Sprintf("%s/p%s?sign=%s", api, utils.EncodePath(realPath, true), sign.Sign(realPath))}, nil + return &model.Link{URL: fmt.Sprintf("%s/p%s?sign=%s", api, utils.EncodePath(realPath, true), d.generateSign(realPath))}, nil } link, _, linkErr := op.Link(ctx, storage, actualPath, args) return link, linkErr } func (d *Strm) syncLocalDir(ctx context.Context, virtualDir string, objs []model.Obj) { + d.syncLocalDirWithMode(ctx, virtualDir, objs, d.normalizedMode) +} + +func (d *Strm) syncLocalDirWithMode(ctx context.Context, virtualDir string, objs []model.Obj, mode string) { if !d.SaveStrmToLocal || strings.TrimSpace(d.SaveStrmLocalPath) == "" { return } @@ -204,12 +209,12 @@ func (d *Strm) syncLocalDir(ctx context.Context, virtualDir string, objs []model log.Warnf("strm: build local payload failed %s: %v", localPath, err) continue } - if err = d.writeLocal(localPath, payload); err != nil { + if err = d.writeLocal(localPath, payload, mode); err != nil { log.Warnf("strm: write local failed %s: %v", localPath, err) } } - if d.normalizedMode == SaveLocalSyncMode { + if mode == SaveLocalSyncMode { d.syncDeleteExtras(localDir, expected) } } @@ -259,19 +264,19 @@ func readLinkBytes(ctx context.Context, link *model.Link) ([]byte, error) { return io.ReadAll(res.RawBody()) } -func (d *Strm) writeLocal(path string, payload []byte) error { - if d.normalizedMode == SaveLocalInsertMode && utils.Exists(path) { +func (d *Strm) writeLocal(path string, payload []byte, mode string) error { + if mode == SaveLocalInsertMode && utils.Exists(path) { return nil } if st, err := os.Stat(path); err == nil && st.IsDir() { - if d.normalizedMode != SaveLocalSyncMode { + if mode != SaveLocalSyncMode { return nil } if err = os.RemoveAll(path); err != nil { return err } } - if d.normalizedMode != SaveLocalInsertMode { + if mode != SaveLocalInsertMode { if old, err := os.ReadFile(path); err == nil { if bytes.Equal(old, payload) { return nil @@ -303,3 +308,10 @@ func (d *Strm) syncDeleteExtras(localDir string, expected map[string]bool) { } } } + +func (d *Strm) generateSign(path string) string { + if d.SignExpireHours > 0 { + return sign.WithDuration(path, time.Duration(d.SignExpireHours)*time.Hour) + } + return sign.Sign(path) +} From faa596c8d486fddef17a1b72c4fa1fb920bc95de Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 10 Mar 2026 19:41:02 +0800 Subject: [PATCH 083/133] feat(quark): support 302 with transcoding and stable video preview --- drivers/quark_uc/driver.go | 50 ++++++++++------------ drivers/quark_uc/meta.go | 10 +++-- drivers/quark_uc/types.go | 74 ++++++++++++++++++++++++++++++-- drivers/quark_uc/util.go | 87 +++++++++++++++++++++++++++++++++++--- server/handles/down.go | 3 ++ server/handles/fsread.go | 3 +- 6 files changed, 184 insertions(+), 43 deletions(-) diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index 7f497494502..691874c321e 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -7,12 +7,14 @@ import ( "hash" "io" "net/http" + "strings" "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" streamPkg "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" @@ -36,6 +38,14 @@ func (d *QuarkOrUC) GetAddition() driver.Additional { func (d *QuarkOrUC) Init(ctx context.Context) error { _, err := d.request("/config", http.MethodGet, nil, nil) + if err == nil && d.AdditionVersion != 2 { + d.AdditionVersion = 2 + if !d.UseTransCodingAddress && len(d.DownProxyUrl) == 0 { + d.WebProxy = true + d.WebdavPolicy = "native_proxy" + } + op.MustSaveDriverStorage(d) + } return err } @@ -44,39 +54,23 @@ func (d *QuarkOrUC) Drop(ctx context.Context) error { } func (d *QuarkOrUC) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files, err := d.GetFiles(dir.GetID()) - if err != nil { - return nil, err - } - return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil - }) + return d.GetFiles(dir.GetID()) } func (d *QuarkOrUC) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - data := base.Json{ - "fids": []string{file.GetID()}, - } - var resp DownResp - ua := d.conf.ua - _, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) { - req.SetHeader("User-Agent", ua). - SetBody(data) - }, &resp) - if err != nil { + f := file.(*File) + if d.UseTransCodingAddress && d.config.Name == "Quark" && f.Category == 1 && f.Size > 0 { + link, err := d.getTranscodingLink(file) + if err == nil { + return link, nil + } + if strings.Contains(err.Error(), "plf_invalid") { + log.Warnf("quark transcoding link invalid for %s, fallback to download link: %v", file.GetName(), err) + return d.getDownloadLink(file) + } return nil, err } - - return &model.Link{ - URL: resp.Data[0].DownloadUrl, - Header: http.Header{ - "Cookie": []string{d.Cookie}, - "Referer": []string{d.conf.referer}, - "User-Agent": []string{ua}, - }, - Concurrency: 3, - PartSize: 10 * utils.MB, - }, nil + return d.getDownloadLink(file) } func (d *QuarkOrUC) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { diff --git a/drivers/quark_uc/meta.go b/drivers/quark_uc/meta.go index f3acfe88562..6940c44b9d6 100644 --- a/drivers/quark_uc/meta.go +++ b/drivers/quark_uc/meta.go @@ -8,8 +8,11 @@ import ( type Addition struct { Cookie string `json:"cookie" required:"true"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"none,file_type,file_name,updated_at" default:"none"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + OrderBy string `json:"order_by" type:"select" options:"none,file_type,file_name,updated_at" default:"none"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + UseTransCodingAddress bool `json:"use_transcoding_address" help:"You can watch the transcoded video and support 302 redirection" required:"true" default:"false"` + OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` + AdditionVersion int } type Conf struct { @@ -24,7 +27,6 @@ func init() { return &QuarkOrUC{ config: driver.Config{ Name: "Quark", - OnlyLocal: true, DefaultRoot: "0", NoOverwriteUpload: true, }, @@ -40,7 +42,7 @@ func init() { return &QuarkOrUC{ config: driver.Config{ Name: "UC", - OnlyLocal: true, + OnlyProxy: true, DefaultRoot: "0", NoOverwriteUpload: true, }, diff --git a/drivers/quark_uc/types.go b/drivers/quark_uc/types.go index afbdb3eff89..13bfac2f7d5 100644 --- a/drivers/quark_uc/types.go +++ b/drivers/quark_uc/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" ) type Resp struct { @@ -18,13 +19,13 @@ type File struct { Fid string `json:"fid"` FileName string `json:"file_name"` //PdirFid string `json:"pdir_fid"` - //Category int `json:"category"` + Category int `json:"category"` //FileType int `json:"file_type"` Size int64 `json:"size"` //FormatType string `json:"format_type"` //Status int `json:"status"` //Tags string `json:"tags,omitempty"` - //LCreatedAt int64 `json:"l_created_at"` + LCreatedAt int64 `json:"l_created_at"` LUpdatedAt int64 `json:"l_updated_at"` //NameSpace int `json:"name_space"` //IncludeItems int `json:"include_items,omitempty"` @@ -32,8 +33,8 @@ type File struct { //BackupSign int `json:"backup_sign"` //Duration int `json:"duration"` //FileSource string `json:"file_source"` - File bool `json:"file"` - //CreatedAt int64 `json:"created_at"` + File bool `json:"file"` + CreatedAt int64 `json:"created_at"` UpdatedAt int64 `json:"updated_at"` //PrivateExtra struct {} `json:"_private_extra"` //ObjCategory string `json:"obj_category,omitempty"` @@ -50,6 +51,38 @@ func fileToObj(f File) *model.Object { } } +func (f *File) GetSize() int64 { + return f.Size +} + +func (f *File) GetName() string { + return f.FileName +} + +func (f *File) ModTime() time.Time { + return time.UnixMilli(f.UpdatedAt) +} + +func (f *File) CreateTime() time.Time { + return time.UnixMilli(f.CreatedAt) +} + +func (f *File) IsDir() bool { + return !f.File +} + +func (f *File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f *File) GetID() string { + return f.Fid +} + +func (f *File) GetPath() string { + return "" +} + type SortResp struct { Resp Data struct { @@ -100,6 +133,39 @@ type DownResp struct { //} `json:"metadata"` } +type TranscodingResp struct { + Resp + Data struct { + DefaultResolution string `json:"default_resolution"` + OriginDefaultResolution string `json:"origin_default_resolution"` + VideoList []struct { + Resolution string `json:"resolution"` + VideoInfo struct { + Duration int `json:"duration"` + Size int64 `json:"size"` + Format string `json:"format"` + Width int `json:"width"` + Height int `json:"height"` + Bitrate float64 `json:"bitrate"` + Codec string `json:"codec"` + Fps float64 `json:"fps"` + Rotate int `json:"rotate"` + UpdateTime int64 `json:"update_time"` + URL string `json:"url"` + Resolution string `json:"resolution"` + HlsType string `json:"hls_type"` + Finish bool `json:"finish"` + Resoultion string `json:"resoultion"` + Success bool `json:"success"` + } `json:"video_info,omitempty"` + } `json:"video_list"` + FileName string `json:"file_name"` + NameSpace int `json:"name_space"` + Size int64 `json:"size"` + Thumbnail string `json:"thumbnail"` + } `json:"data"` +} + type UpPreResp struct { Resp Data struct { diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index c5845cc6823..2f99c308f33 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "errors" "fmt" + "html" "io" "net/http" "strconv" @@ -50,20 +51,29 @@ func (d *QuarkOrUC) request(pathname string, method string, callback base.ReqCal d.Cookie = cookie.SetStr(d.Cookie, "__puus", __puus.Value) op.MustSaveDriverStorage(d) } + if d.UseTransCodingAddress && d.config.Name == "Quark" { + __pus := cookie.GetCookie(res.Cookies(), "__pus") + if __pus != nil { + d.Cookie = cookie.SetStr(d.Cookie, "__pus", __pus.Value) + op.MustSaveDriverStorage(d) + } + } if e.Status >= 400 || e.Code != 0 { return nil, errors.New(e.Message) } return res.Body(), nil } -func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { - files := make([]File, 0) +func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { + files := make([]model.Obj, 0) page := 1 size := 100 query := map[string]string{ - "pdir_fid": parent, - "_size": strconv.Itoa(size), - "_fetch_total": "1", + "pdir_fid": parent, + "_size": strconv.Itoa(size), + "_fetch_total": "1", + "fetch_all_file": "1", + "fetch_risk_file_name": "1", } if d.OrderBy != "none" { query["_sort"] = "file_type:asc," + d.OrderBy + ":" + d.OrderDirection @@ -77,7 +87,16 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { if err != nil { return nil, err } - files = append(files, resp.Data.List...) + for _, file := range resp.Data.List { + file.FileName = html.UnescapeString(file.FileName) + if d.OnlyListVideoFile { + if file.IsDir() || file.Category == 1 { + files = append(files, &file) + } + } else { + files = append(files, &file) + } + } if page*size >= resp.Metadata.Total { break } @@ -86,6 +105,62 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { return files, nil } +func (d *QuarkOrUC) getDownloadLink(file model.Obj) (*model.Link, error) { + data := base.Json{ + "fids": []string{file.GetID()}, + } + var resp DownResp + ua := d.conf.ua + _, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", ua). + SetBody(data) + }, &resp) + if err != nil { + return nil, err + } + + return &model.Link{ + URL: resp.Data[0].DownloadUrl, + Header: http.Header{ + "Cookie": []string{d.Cookie}, + "Referer": []string{d.conf.referer}, + "User-Agent": []string{ua}, + }, + Concurrency: 3, + PartSize: 10 * utils.MB, + }, nil +} + +func (d *QuarkOrUC) getTranscodingLink(file model.Obj) (*model.Link, error) { + data := base.Json{ + "fid": file.GetID(), + "resolutions": "low,normal,high,super,2k,4k", + "supports": "fmp4_av,m3u8,dolby_vision", + } + var resp TranscodingResp + ua := d.conf.ua + + _, err := d.request("/file/v2/play/project", http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", ua). + SetBody(data) + }, &resp) + if err != nil { + return nil, err + } + + for _, info := range resp.Data.VideoList { + if info.VideoInfo.URL != "" { + return &model.Link{ + URL: info.VideoInfo.URL, + Concurrency: 3, + PartSize: 10 * utils.MB, + }, nil + } + } + + return nil, errors.New("no link found") +} + func (d *QuarkOrUC) upPre(file model.FileStreamer, parentId string) (UpPreResp, error) { now := time.Now() data := base.Json{ diff --git a/server/handles/down.go b/server/handles/down.go index e93ed1eb103..37439f00bb3 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -187,6 +187,9 @@ func canProxy(storage driver.Driver, filename string) bool { if storage.Config().MustProxy() || storage.GetStorage().WebProxy || storage.GetStorage().WebdavProxy() { return true } + if storage.GetStorage().Driver == "Quark" && utils.GetFileType(filename) == conf.VIDEO { + return true + } if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { return true } diff --git a/server/handles/fsread.go b/server/handles/fsread.go index eb984698b16..c370f631fc0 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -377,7 +377,8 @@ func FsGet(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + forceProxyRawURL := storage.GetStorage().Driver == "Quark" && utils.GetFileType(obj.GetName()) == conf.VIDEO + if storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL { query := "" if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { query = "?sign=" + sign.Sign(reqPath) From 82ab5768ceb8fe62a24f98c904927f4ac56a20b1 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Sat, 14 Mar 2026 14:19:47 +0800 Subject: [PATCH 084/133] fix(auth): prevent and heal dirty admin permissions --- internal/db/role.go | 3 +++ internal/op/role.go | 30 ++++++++++++++++++++++++++++++ internal/op/user.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/internal/db/role.go b/internal/db/role.go index 808a6f5f06b..d0b776b391d 100644 --- a/internal/db/role.go +++ b/internal/db/role.go @@ -79,6 +79,9 @@ func UpdateRolePermissionsPathPrefix(oldPath, newPath string) ([]uint, error) { } for _, role := range roles { + if role.Name == "admin" || role.Name == "guest" { + continue + } updated := false for i, entry := range role.PermissionScopes { entryPath := path.Clean(entry.Path) diff --git a/internal/op/role.go b/internal/op/role.go index 5c9aad06f41..bd874eeed19 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -21,6 +21,24 @@ func init() { model.FetchRole = GetRole } +func enforceAdminRoleDefaults(r *model.Role) error { + if r == nil || r.Name != "admin" { + return nil + } + if len(r.PermissionScopes) == 1 { + scopePath := utils.FixAndCleanPath(r.PermissionScopes[0].Path) + if scopePath == "/" && r.PermissionScopes[0].Permission == 0xFFFF { + r.PermissionScopes[0].Path = "/" + return nil + } + } + + r.PermissionScopes = []model.PermissionEntry{ + {Path: "/", Permission: 0xFFFF}, + } + return db.UpdateRole(r) +} + func GetRole(id uint) (*model.Role, error) { key := fmt.Sprint(id) if r, ok := roleCache.Get(key); ok { @@ -31,7 +49,11 @@ func GetRole(id uint) (*model.Role, error) { if err != nil { return nil, err } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(_r.Name, _r, cache.WithEx[*model.Role](time.Hour)) return _r, nil }) return r, err @@ -46,7 +68,11 @@ func GetRoleByName(name string) (*model.Role, error) { if err != nil { return nil, err } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } roleCache.Set(name, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(fmt.Sprint(_r.ID), _r, cache.WithEx[*model.Role](time.Hour)) return _r, nil }) return r, err @@ -89,7 +115,11 @@ func GetRolesByUserID(userID uint) ([]model.Role, error) { if err != nil { return nil, err } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(_r.Name, _r, cache.WithEx[*model.Role](time.Hour)) return _r, nil }) if err != nil { diff --git a/internal/op/user.go b/internal/op/user.go index 44b19db3508..b58a87ed338 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -16,6 +16,25 @@ var userG singleflight.Group[*model.User] var guestUser *model.User var adminUser *model.User +func enforceAdminUserDefaults(u *model.User) error { + if u == nil || !u.IsAdmin() { + return nil + } + changed := false + if utils.FixAndCleanPath(u.BasePath) != "/" { + u.BasePath = "/" + changed = true + } + if u.Permission != 0xFFFF { + u.Permission = 0xFFFF + changed = true + } + if !changed { + return nil + } + return db.UpdateUser(u) +} + func GetAdmin() (*model.User, error) { if adminUser == nil { role, err := GetRoleByName("admin") @@ -26,7 +45,12 @@ func GetAdmin() (*model.User, error) { if err != nil { return nil, err } + if err := enforceAdminUserDefaults(user); err != nil { + return nil, err + } adminUser = user + } else if err := enforceAdminUserDefaults(adminUser); err != nil { + return nil, err } return adminUser, nil } @@ -59,6 +83,9 @@ func GetUserByName(username string) (*model.User, error) { return nil, errs.EmptyUsername } if user, ok := userCache.Get(username); ok { + if err := enforceAdminUserDefaults(user); err != nil { + return nil, err + } return user, nil } user, err, _ := userG.Do(username, func() (*model.User, error) { @@ -66,6 +93,9 @@ func GetUserByName(username string) (*model.User, error) { if err != nil { return nil, err } + if err := enforceAdminUserDefaults(_user); err != nil { + return nil, err + } userCache.Set(username, _user, cache.WithEx[*model.User](time.Hour)) return _user, nil }) From 2280505f2a6c02e0123018e5ac1de389236cd0d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 16 Mar 2026 19:13:57 +0800 Subject: [PATCH 085/133] chore(deps): refresh indirect module checksums (#9446) --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2e202bc9de0..bc2475c548d 100644 --- a/go.mod +++ b/go.mod @@ -250,7 +250,7 @@ require ( github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/shirou/gopsutil/v3 v3.24.4 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/shoenig/go-m1cpu v0.2.0 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index af3a6b3f96c..e6b04779ede 100644 --- a/go.sum +++ b/go.sum @@ -589,8 +589,11 @@ github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRB github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/go-m1cpu v0.2.0 h1:t4GNqvPZ84Vjtpboo/kT3pIkbaK3vc+JIlD/Wz1zSFY= +github.com/shoenig/go-m1cpu v0.2.0/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= From 6bde813ef14c6032af39bd0820fbf8901f8a37a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 16 Mar 2026 19:15:06 +0800 Subject: [PATCH 086/133] feat(wukong): add WuKongNetdisk driver with read-write support (#9449) --- drivers/all.go | 3 +- drivers/wukong/driver.go | 1116 ++++++++++++++++++++++++++++++++++++++ drivers/wukong/meta.go | 34 ++ drivers/wukong/types.go | 113 ++++ 4 files changed, 1265 insertions(+), 1 deletion(-) create mode 100644 drivers/wukong/driver.go create mode 100644 drivers/wukong/meta.go create mode 100644 drivers/wukong/types.go diff --git a/drivers/all.go b/drivers/all.go index 58cdac85f03..a4fce9d0cac 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -66,8 +66,8 @@ import ( _ "github.com/alist-org/alist/v3/drivers/seafile" _ "github.com/alist-org/alist/v3/drivers/sftp" _ "github.com/alist-org/alist/v3/drivers/smb" - _ "github.com/alist-org/alist/v3/drivers/strm" _ "github.com/alist-org/alist/v3/drivers/streamtape" + _ "github.com/alist-org/alist/v3/drivers/strm" _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" @@ -81,6 +81,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/webdav" _ "github.com/alist-org/alist/v3/drivers/weiyun" _ "github.com/alist-org/alist/v3/drivers/wopan" + _ "github.com/alist-org/alist/v3/drivers/wukong" _ "github.com/alist-org/alist/v3/drivers/yandex_disk" ) diff --git a/drivers/wukong/driver.go b/drivers/wukong/driver.go new file mode 100644 index 00000000000..cb5dee1ae40 --- /dev/null +++ b/drivers/wukong/driver.go @@ -0,0 +1,1116 @@ +package wukong + +import ( + "context" + "crypto/hmac" + "crypto/md5" + crand "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash/crc32" + "io" + "net/http" + "net/url" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" +) + +const ( + wukongBaseURL = "https://api.wkbrowser.com" + webReferer = "https://pan.wkbrowser.com/" + vodBaseURL = "https://vod.bytedanceapi.com" + vodRegion = "cn-north-1" + vodService = "vod" + videoSpaceName = "wukong_netdisk_ugc" + minUploadSubmitSuccess = 2000 + multipartChunkSize = int64(5 * 1024 * 1024) +) + +type Wukong struct { + model.Storage + Addition + client *resty.Client +} + +func (d *Wukong) Config() driver.Config { + return config +} + +func (d *Wukong) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Wukong) Init(ctx context.Context) error { + d.client = base.NewRestyClient(). + SetBaseURL(wukongBaseURL). + SetHeader("accept", "application/json, text/plain, */*"). + SetHeader("content-type", "application/json"). + SetHeader("referer", webReferer). + SetHeader("origin", "https://pan.wkbrowser.com") + if d.Cookie != "" { + d.client.SetHeader("cookie", d.Cookie) + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + if strings.TrimSpace(d.Aid) == "" { + d.Aid = "590353" + } + if strings.TrimSpace(d.Language) == "" { + d.Language = "zh" + } + if d.PageSize <= 0 { + d.PageSize = 100 + } + return nil +} + +func (d *Wukong) Drop(ctx context.Context) error { + return nil +} + +func (d *Wukong) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + fatherID := dir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + offset := 0 + limit := d.PageSize + objs := make([]model.Obj, 0) + for { + var resp filterFileResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "offset": strconv.Itoa(offset), + "limit": strconv.Itoa(limit), + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "father_id": asIDValue(fatherID), + "filter_type": 2, + "is_desc": 1, + "file_type": 0, + }). + SetResult(&resp). + Post("/netdisk/user_file/filter_file") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong list failed: code=%d message=%s", resp.Code, resp.Message) + } + + for _, item := range resp.Data.FileList { + objs = append(objs, &model.Object{ + ID: strconv.FormatInt(item.FileID, 10), + Path: strconv.FormatInt(item.FatherID, 10), + Name: item.FileName, + Size: item.Size, + Modified: parseUnix(item.UpdatedAt), + Ctime: parseUnix(item.CreatedAt), + IsFolder: item.IsDirectory == 1, + }) + } + + if !hasMore(resp.Data.HasMore) || len(resp.Data.FileList) == 0 { + break + } + offset += len(resp.Data.FileList) + } + return objs, nil +} + +func (d *Wukong) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + fileID := file.GetID() + if fileID == "" { + return nil, errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(fileID)}, + }). + SetResult(&resp). + Post("/netdisk/user_file/detail") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong detail failed: code=%d message=%s", resp.Code, resp.Message) + } + + url := extractDetailMainURL(resp.Data) + if url == "" { + url = extractURL(resp.Data) + } + if url == "" { + return nil, errs.NotImplement + } + + return &model.Link{ + URL: url, + Header: http.Header{ + "Referer": []string{webReferer}, + "Cookie": []string{d.Cookie}, + }, + }, nil +} + +func (d *Wukong) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + fatherID := parentDir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "father_id": asIDValue(fatherID), + "file_name": dirName, + }). + SetResult(&resp). + Post("/netdisk/user_file/create_directory") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong create directory failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + srcID := srcObj.GetID() + if srcID == "" { + return errors.New("missing source file id") + } + + dstID := dstDir.GetID() + if dstID == "" { + dstID = d.RootFolderID + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(srcID)}, + "new_father_id": asIDValue(dstID), + }). + SetResult(&resp). + Post("/netdisk/user_file/move_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong move failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + srcID := srcObj.GetID() + if srcID == "" { + return errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id": asIDValue(srcID), + "new_name": newName, + }). + SetResult(&resp). + Post("/netdisk/user_file/rename_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong rename failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Remove(ctx context.Context, obj model.Obj) error { + fileID := obj.GetID() + if fileID == "" { + return errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(fileID)}, + }). + SetResult(&resp). + Post("/netdisk/user_file/delete_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong delete failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + fatherID := dstDir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return err + } + defer tempFile.Close() + if _, err = tempFile.Seek(0, io.SeekStart); err != nil { + return err + } + + md5Hex, crc32Hex, err := calcFileMD5AndCRC32(tempFile) + if err != nil { + return err + } + ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(file.GetName())), ".") + fileType := detectWukongFileType(file.GetMimetype(), file.GetName()) + size := file.GetSize() + up(5) + + uploadType := detectUploadType(file.GetMimetype(), file.GetName()) + authToken, err := d.getUploadAuthToken(ctx, uploadType) + if err != nil { + return err + } + up(10) + + candidates, err := d.getUploadCandidates(ctx, authToken) + if err != nil { + return err + } + bestHosts := collectCandidateHosts(candidates) + if len(bestHosts) == 0 { + return errors.New("wukong upload candidates is empty") + } + up(20) + + applyResp, err := d.applyUploadInner(ctx, authToken, uploadType, size, strings.Join(bestHosts, ",")) + if err != nil { + return err + } + if len(applyResp.Result.InnerUploadAddress.UploadNodes) == 0 || + len(applyResp.Result.InnerUploadAddress.UploadNodes[0].StoreInfos) == 0 { + return errors.New("wukong apply upload inner returns empty upload node") + } + node := applyResp.Result.InnerUploadAddress.UploadNodes[0] + store := node.StoreInfos[0] + up(30) + + if size > multipartChunkSize { + if err = d.uploadToTOSMultipart(ctx, node.UploadHost, store.StoreURI, store.Auth, getStorageUserID(store.StorageHeader), tempFile, size, up); err != nil { + return err + } + } else { + if _, err = tempFile.Seek(0, io.SeekStart); err != nil { + return err + } + reader := &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{Reader: tempFile, Size: size}, + UpdateProgress: func(percent float64) { + up(30 + percent*0.5) + }, + } + if err = d.uploadToTOS(ctx, node.UploadHost, store.StoreURI, store.Auth, getStorageUserID(store.StorageHeader), crc32Hex, file.GetName(), reader, size); err != nil { + return err + } + } + up(85) + + videoVid, err := d.commitUploadInner(ctx, authToken, chooseCommitSpace(uploadType, authToken.SpaceName), node.SessionKey) + if err != nil { + return err + } + up(92) + + if fileType == 3000 && videoVid == "" { + return errors.New("wukong video upload missing vid in commit response") + } + if err = d.uploadSubmit(ctx, fatherID, file.GetName(), ext, fileType, size, md5Hex, store.StoreURI, videoVid); err != nil { + return err + } + up(100) + return nil +} + +func (d *Wukong) getUploadAuthToken(ctx context.Context, uploadType string) (*uploadAuthTokenResp, error) { + var resp uploadAuthTokenResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "upload_source": uploadSourceByType(uploadType), + "type": uploadType, + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetResult(&resp). + Get("/toutiao/upload/auth_token/v1/") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong get upload auth token failed: code=%d message=%s", resp.Code, resp.Message) + } + return &resp, nil +} + +func (d *Wukong) getUploadCandidates(ctx context.Context, auth *uploadAuthTokenResp) (*getUploadCandidatesResp, error) { + q := map[string]string{ + "Action": "GetUploadCandidates", + "Version": "2020-11-19", + "SpaceName": videoSpaceName, + } + var resp getUploadCandidatesResp + if err := d.vodRequest(ctx, http.MethodGet, q, nil, auth, &resp); err != nil { + return nil, err + } + if resp.ResponseMetadata.Error.Code != "" { + return nil, fmt.Errorf("wukong get upload candidates failed: %s", resp.ResponseMetadata.Error.Message) + } + return &resp, nil +} + +func (d *Wukong) applyUploadInner(ctx context.Context, auth *uploadAuthTokenResp, uploadType string, fileSize int64, bestHosts string) (*applyUploadInnerResp, error) { + spaceName := auth.SpaceName + if uploadType == "video" { + spaceName = videoSpaceName + } + q := map[string]string{ + "Action": "ApplyUploadInner", + "Version": "2020-11-19", + "SpaceName": spaceName, + "FileType": uploadType, + "IsInner": "1", + "ClientBestHosts": bestHosts, + "NeedFallback": "true", + "FileSize": strconv.FormatInt(fileSize, 10), + "s": randomString(8), + } + var resp applyUploadInnerResp + if err := d.vodRequest(ctx, http.MethodGet, q, nil, auth, &resp); err != nil { + return nil, err + } + if resp.ResponseMetadata.Error.Code != "" { + return nil, fmt.Errorf("wukong apply upload inner failed: %s", resp.ResponseMetadata.Error.Message) + } + return &resp, nil +} + +func (d *Wukong) commitUploadInner(ctx context.Context, auth *uploadAuthTokenResp, spaceName, sessionKey string) (string, error) { + q := map[string]string{ + "Action": "CommitUploadInner", + "Version": "2020-11-19", + "SpaceName": spaceName, + } + body, _ := json.Marshal(map[string]any{ + "SessionKey": sessionKey, + "Functions": []any{}, + }) + var resp commitUploadInnerResp + if err := d.vodRequest(ctx, http.MethodPost, q, body, auth, &resp); err != nil { + return "", err + } + if resp.ResponseMetadata.Error.Code != "" { + return "", fmt.Errorf("wukong commit upload inner failed: %s", resp.ResponseMetadata.Error.Message) + } + if len(resp.Result.Results) > 0 { + status := resp.Result.Results[0].URIStatus + if status != 0 && status != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong commit upload inner failed: uri_status=%d", status) + } + } + return extractVideoVid(&resp), nil +} + +func (d *Wukong) uploadSubmit(ctx context.Context, fatherID, fileName, ext string, fileType int, size int64, md5Hex, storeURI, videoVid string) error { + var resp uploadSubmitResp + body := map[string]any{ + "base_info": map[string]any{ + "father_id": asIDValue(fatherID), + "file_type": fileType, + "size": size, + "extension": ext, + "file_name": fileName, + "is_directory": 0, + "md5": md5Hex, + "slice_md5": md5Hex, + }, + } + switch fileType { + case 3000: + if videoVid != "" { + body["video_info"] = map[string]any{"vid": videoVid} + } + case 2000: + body["image_info"] = map[string]any{"uri": storeURI} + default: + body["general_info"] = map[string]any{"key": storeURI} + } + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(body). + SetResult(&resp). + Post("/netdisk/upload_submit/") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong upload submit failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func extractVideoVid(resp *commitUploadInnerResp) string { + for _, item := range resp.Result.Results { + if item.Vid != "" { + return item.Vid + } + } + if resp.Result.PluginResult != nil { + if vid := findStringByKey(resp.Result.PluginResult, "Vid"); vid != "" { + return vid + } + if vid := findStringByKey(resp.Result.PluginResult, "vid"); vid != "" { + return vid + } + } + return "" +} + +func findStringByKey(v any, key string) string { + switch cur := v.(type) { + case map[string]any: + if val, ok := cur[key]; ok { + if s, ok := val.(string); ok && s != "" { + return s + } + } + for _, child := range cur { + if s := findStringByKey(child, key); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findStringByKey(child, key); s != "" { + return s + } + } + } + return "" +} + +func (d *Wukong) uploadToTOS(ctx context.Context, host, storeURI, auth, storageUser, crc32Hex, fileName string, body io.Reader, size int64) error { + var resp tosUploadResp + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Crc32", crc32Hex). + SetHeader("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, url.QueryEscape(fileName))). + SetHeader("Content-Length", strconv.FormatInt(size, 10)). + SetBody(body). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + _, err := req.Post(uploadURL) + if err != nil { + return err + } + if resp.Code != minUploadSubmitSuccess { + return fmt.Errorf("wukong upload to tos failed: code=%d message=%s", resp.Code, resp.Message) + } + if resp.Data.Crc32 != "" && !strings.EqualFold(resp.Data.Crc32, crc32Hex) { + return fmt.Errorf("wukong upload to tos crc32 mismatch: local=%s remote=%s", crc32Hex, resp.Data.Crc32) + } + return nil +} + +func (d *Wukong) uploadToTOSMultipart(ctx context.Context, host, storeURI, auth, storageUser string, tempFile model.File, size int64, up driver.UpdateProgress) error { + uploadID, err := d.initMultipartUpload(ctx, host, storeURI, auth, storageUser) + if err != nil { + return err + } + + totalParts := int((size + multipartChunkSize - 1) / multipartChunkSize) + if totalParts <= 0 { + return errors.New("invalid multipart parts") + } + parts := make([]string, 0, totalParts) + for i := 0; i < totalParts; i++ { + partNumber := i + 1 + offset := int64(i) * multipartChunkSize + partSize := multipartChunkSize + if remain := size - offset; remain < partSize { + partSize = remain + } + buf := make([]byte, partSize) + n, readErr := tempFile.ReadAt(buf, offset) + if readErr != nil && readErr != io.EOF { + return readErr + } + buf = buf[:n] + crc32Hex := fmt.Sprintf("%08x", crc32.ChecksumIEEE(buf)) + remoteCRC32, err := d.uploadMultipartPart(ctx, host, storeURI, auth, storageUser, uploadID, partNumber, buf, crc32Hex) + if err != nil { + return err + } + if remoteCRC32 != "" && !strings.EqualFold(remoteCRC32, crc32Hex) { + return fmt.Errorf("multipart part crc32 mismatch: part=%d local=%s remote=%s", partNumber, crc32Hex, remoteCRC32) + } + parts = append(parts, fmt.Sprintf("%d:%s", partNumber, crc32Hex)) + up(30 + float64(partNumber)/float64(totalParts)*50) + } + + return d.finishMultipartUpload(ctx, host, storeURI, auth, storageUser, uploadID, strings.Join(parts, ",")) +} + +func (d *Wukong) initMultipartUpload(ctx context.Context, host, storeURI, auth, storageUser string) (string, error) { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetQueryParams(map[string]string{ + "uploadmode": "part", + "phase": "init", + }). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return "", err + } + if resp.Code != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong init multipart upload failed: code=%d message=%s", resp.Code, resp.Message) + } + if resp.Data.UploadID == "" { + return "", errors.New("wukong init multipart upload returns empty uploadid") + } + return resp.Data.UploadID, nil +} + +func (d *Wukong) uploadMultipartPart(ctx context.Context, host, storeURI, auth, storageUser, uploadID string, partNumber int, data []byte, crc32Hex string) (string, error) { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Crc32", crc32Hex). + SetHeader("Content-Length", strconv.Itoa(len(data))). + SetQueryParams(map[string]string{ + "uploadid": uploadID, + "part_number": strconv.Itoa(partNumber), + "phase": "transfer", + }). + SetBody(data). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return "", err + } + if resp.Code != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong multipart transfer failed: code=%d message=%s part=%d", resp.Code, resp.Message, partNumber) + } + return resp.Data.Crc32, nil +} + +func (d *Wukong) finishMultipartUpload(ctx context.Context, host, storeURI, auth, storageUser, uploadID, body string) error { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetQueryParams(map[string]string{ + "uploadid": uploadID, + "phase": "finish", + "uploadmode": "part", + }). + SetBody(body). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return err + } + if resp.Code != minUploadSubmitSuccess && resp.Code != 4024 { + return fmt.Errorf("wukong multipart finish failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) vodRequest(ctx context.Context, method string, query map[string]string, body []byte, auth *uploadAuthTokenResp, resp any) error { + reqURL := vodBaseURL + "/" + amzDate := time.Now().UTC().Format("20060102T150405Z") + dateStamp := amzDate[:8] + headers := map[string]string{ + "x-amz-date": amzDate, + "x-amz-security-token": auth.SessionToken, + } + if method == http.MethodPost { + headers["x-amz-content-sha256"] = hashSHA256Bytes(body) + } + authorization := buildVodAuthorization(method, "/", query, headers, body, auth, dateStamp) + + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Authorization", authorization). + SetHeader("x-amz-date", amzDate). + SetHeader("x-amz-security-token", auth.SessionToken). + SetQueryParams(query). + SetResult(resp) + if method == http.MethodPost { + req.SetHeader("x-amz-content-sha256", headers["x-amz-content-sha256"]) + req.SetHeader("Content-Type", "text/plain;charset=UTF-8") + req.SetBody(body) + } + _, err := req.Execute(method, reqURL) + return err +} + +func buildVodAuthorization(method, canonicalURI string, query map[string]string, headers map[string]string, body []byte, auth *uploadAuthTokenResp, dateStamp string) string { + canonicalQueryString := getCanonicalQueryStringFromMap(query) + canonicalHeaders, signedHeaders := getCanonicalHeaders(headers) + payloadHash := hashSHA256Bytes(body) + canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQueryString + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash + credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", dateStamp, vodRegion, vodService) + stringToSign := "AWS4-HMAC-SHA256\n" + headers["x-amz-date"] + "\n" + credentialScope + "\n" + hashSHA256String(canonicalRequest) + signingKey := getSigningKey(auth.SecretAccessKey, dateStamp, vodRegion, vodService) + signature := hmacSHA256Hex(signingKey, stringToSign) + return fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", auth.AccessKeyID, credentialScope, signedHeaders, signature) +} + +func getCanonicalQueryStringFromMap(query map[string]string) string { + if len(query) == 0 { + return "" + } + keys := make([]string, 0, len(query)) + for k := range query { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, awsURLEncode(k)+"="+awsURLEncode(query[k])) + } + return strings.Join(parts, "&") +} + +func getCanonicalHeaders(headers map[string]string) (string, string) { + keys := make([]string, 0, len(headers)) + for k := range headers { + keys = append(keys, strings.ToLower(k)) + } + sort.Strings(keys) + var h strings.Builder + for _, k := range keys { + h.WriteString(k) + h.WriteString(":") + h.WriteString(strings.TrimSpace(headers[k])) + h.WriteString("\n") + } + return h.String(), strings.Join(keys, ";") +} + +func awsURLEncode(s string) string { + s = url.QueryEscape(s) + return strings.ReplaceAll(s, "+", "%20") +} + +func hashSHA256Bytes(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func hashSHA256String(s string) string { + return hashSHA256Bytes([]byte(s)) +} + +func hmacSHA256(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + _, _ = h.Write([]byte(data)) + return h.Sum(nil) +} + +func hmacSHA256Hex(key []byte, data string) string { + return hex.EncodeToString(hmacSHA256(key, data)) +} + +func getSigningKey(secret, dateStamp, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secret), dateStamp) + kRegion := hmacSHA256(kDate, region) + kService := hmacSHA256(kRegion, service) + return hmacSHA256(kService, "aws4_request") +} + +func collectCandidateHosts(resp *getUploadCandidatesResp) []string { + seen := map[string]struct{}{} + hosts := make([]string, 0, len(resp.Result.Domains)) + add := func(domain vodDomain) { + if domain.Name == "" { + return + } + if _, ok := seen[domain.Name]; ok { + return + } + seen[domain.Name] = struct{}{} + hosts = append(hosts, domain.Name) + } + for _, candidate := range resp.Result.Candidates { + for _, domain := range candidate.Domains { + add(domain) + } + } + for _, domain := range resp.Result.Domains { + add(domain) + } + return hosts +} + +func getStorageUserID(header map[string]any) string { + if header == nil { + return "" + } + if s, ok := header["USER_ID"].(string); ok { + return s + } + if f, ok := header["USER_ID"].(float64); ok { + return strconv.FormatInt(int64(f), 10) + } + return "" +} + +func calcFileMD5AndCRC32(f model.File) (string, string, error) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", "", err + } + md5Hasher := md5.New() + crc := crc32.NewIEEE() + _, err := io.Copy(io.MultiWriter(md5Hasher, crc), f) + if err != nil { + return "", "", err + } + if _, err = f.Seek(0, io.SeekStart); err != nil { + return "", "", err + } + return hex.EncodeToString(md5Hasher.Sum(nil)), fmt.Sprintf("%08x", crc.Sum32()), nil +} + +func detectWukongFileType(mimetype, fileName string) int { + lowerName := strings.ToLower(fileName) + switch { + case strings.HasPrefix(mimetype, "image/"): + return 2000 + case strings.HasPrefix(mimetype, "video/"), strings.HasSuffix(lowerName, ".flv"), strings.HasSuffix(lowerName, ".mkv"): + return 3000 + case strings.HasPrefix(mimetype, "audio/"), strings.HasSuffix(lowerName, ".mp3"), strings.HasSuffix(lowerName, ".m4a"), strings.HasSuffix(lowerName, ".wav"): + return 4000 + case strings.HasSuffix(lowerName, ".zip"), strings.HasSuffix(lowerName, ".rar"), strings.HasSuffix(lowerName, ".7z"), strings.HasSuffix(lowerName, ".tar"), strings.HasSuffix(lowerName, ".gz"), strings.HasSuffix(lowerName, ".tgz"): + return 6000 + default: + return 5000 + } +} + +func randomString(n int) string { + const letters = "abcdefghijklmnopqrstuvwxyz0123456789" + if n <= 0 { + return "" + } + buf := make([]byte, n) + if _, err := crand.Read(buf); err == nil { + for i := range buf { + buf[i] = letters[int(buf[i])%len(letters)] + } + return string(buf) + } + + now := uint64(time.Now().UnixNano()) + b := make([]byte, n) + for i := range b { + now = now*6364136223846793005 + 1 + b[i] = letters[int(now%uint64(len(letters)))] + } + return string(b) +} + +func uploadSourceByType(uploadType string) string { + switch uploadType { + case "video": + return "10150001" + case "image": + return "20150001" + default: + return "50150001" + } +} + +func detectUploadType(mimetype, fileName string) string { + lowerName := strings.ToLower(fileName) + if strings.HasPrefix(mimetype, "video/") || strings.HasPrefix(mimetype, "audio/") || + strings.HasSuffix(lowerName, ".flv") || strings.HasSuffix(lowerName, ".mkv") || + strings.HasSuffix(lowerName, ".mp3") || strings.HasSuffix(lowerName, ".m4a") || strings.HasSuffix(lowerName, ".wav") { + return "video" + } + if strings.HasPrefix(mimetype, "image/") { + return "image" + } + return "object" +} + +func chooseCommitSpace(uploadType, authSpace string) string { + if uploadType == "video" { + return videoSpaceName + } + return authSpace +} + +func asIDValue(id string) any { + if n, err := strconv.ParseInt(id, 10, 64); err == nil { + return n + } + return id +} + +func parseUnix(ts int64) time.Time { + if ts <= 0 { + return time.Time{} + } + if ts > 1e12 { + return time.UnixMilli(ts) + } + return time.Unix(ts, 0) +} + +func hasMore(v any) bool { + switch val := v.(type) { + case bool: + return val + case float64: + return val != 0 + case int: + return val != 0 + case int64: + return val != 0 + case string: + return val == "1" || strings.EqualFold(val, "true") + default: + return false + } +} + +func extractURL(data map[string]any) string { + priority := []string{ + "download_url", + "main_url", + "MainUrl", + "MainHTTPUrl", + "url", + "source_url", + "play_url", + "backup_url", + "BackupUrl", + "BackupHTTPUrl", + } + for _, key := range priority { + if url := findURLByKey(data, key); url != "" { + return url + } + } + return findAnyHTTPURL(data) +} + +func extractDetailMainURL(data map[string]any) string { + rawList, ok := data["list"] + if !ok { + return "" + } + list, ok := rawList.([]any) + if !ok || len(list) == 0 { + return "" + } + first, ok := list[0].(map[string]any) + if !ok { + return "" + } + generalInfo, ok := first["general_info"].(map[string]any) + if !ok { + return "" + } + mainURL, ok := generalInfo["main_url"].(string) + if !ok || !isHTTPURL(mainURL) { + return "" + } + return mainURL +} + +func findURLByKey(v any, key string) string { + switch cur := v.(type) { + case map[string]any: + if val, ok := cur[key]; ok { + if s, ok := val.(string); ok && isHTTPURL(s) { + return s + } + if decoded := tryDecodeJSONAny(val); decoded != nil { + if s := findURLByKey(decoded, key); s != "" { + return s + } + } + } + for _, child := range cur { + if s := findURLByKey(child, key); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findURLByKey(child, key); s != "" { + return s + } + } + case string: + if decoded := tryDecodeJSONString(cur); decoded != nil { + return findURLByKey(decoded, key) + } + } + return "" +} + +func findAnyHTTPURL(v any) string { + switch cur := v.(type) { + case string: + if isHTTPURL(cur) { + return cur + } + if decoded := tryDecodeJSONString(cur); decoded != nil { + return findAnyHTTPURL(decoded) + } + case map[string]any: + for _, child := range cur { + if s := findAnyHTTPURL(child); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findAnyHTTPURL(child); s != "" { + return s + } + } + } + return "" +} + +func isHTTPURL(v string) bool { + return strings.HasPrefix(v, "http://") || strings.HasPrefix(v, "https://") +} + +func tryDecodeJSONAny(v any) any { + s, ok := v.(string) + if !ok { + return nil + } + return tryDecodeJSONString(s) +} + +func tryDecodeJSONString(s string) any { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + if !(strings.HasPrefix(s, "{") || strings.HasPrefix(s, "[")) { + return nil + } + var out any + if err := json.Unmarshal([]byte(s), &out); err != nil { + return nil + } + return out +} + +var _ driver.Driver = (*Wukong)(nil) diff --git a/drivers/wukong/meta.go b/drivers/wukong/meta.go new file mode 100644 index 00000000000..451fd7942b4 --- /dev/null +++ b/drivers/wukong/meta.go @@ -0,0 +1,34 @@ +package wukong + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Cookie string `json:"cookie" type:"text" required:"true" help:"Cookie from https://pan.wkbrowser.com/"` + Aid string `json:"aid" default:"590353" help:"aid query param used by web requests"` + Language string `json:"language" default:"zh"` + PageSize int `json:"page_size" type:"number" default:"100"` +} + +var config = driver.Config{ + Name: "WuKongNetdisk", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Wukong{} + }) +} diff --git a/drivers/wukong/types.go b/drivers/wukong/types.go new file mode 100644 index 00000000000..86ada25e06d --- /dev/null +++ b/drivers/wukong/types.go @@ -0,0 +1,113 @@ +package wukong + +type filterFileResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + FileList []wukongFile `json:"file_list"` + HasMore any `json:"has_more"` + } `json:"data"` +} + +type wukongFile struct { + FileID int64 `json:"file_id"` + FatherID int64 `json:"father_id"` + IsDirectory int `json:"is_directory"` + FileType int `json:"file_type"` + Size int64 `json:"size"` + FileName string `json:"file_name"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +type rawResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data map[string]any `json:"data"` +} + +type uploadAuthTokenResp struct { + Code int `json:"code"` + Message string `json:"message"` + CurrentTime int64 `json:"current_time"` + ExpireTime int64 `json:"expire_time"` + SpaceName string `json:"space_name"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + SessionToken string `json:"session_token"` +} + +type vodResponseMetadata struct { + RequestID string `json:"RequestId"` + Action string `json:"Action"` + Version string `json:"Version"` + Service string `json:"Service"` + Region string `json:"Region"` + Error struct { + CodeN int `json:"CodeN,omitempty"` + Code string `json:"Code,omitempty"` + Message string `json:"Message,omitempty"` + } `json:"Error,omitempty"` +} + +type vodDomain struct { + Name string `json:"Name"` + Sign string `json:"Sign"` + StoreID string `json:"StoreID"` +} + +type getUploadCandidatesResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + Candidates []struct { + Domains []vodDomain `json:"Domains"` + } `json:"Candidates"` + Domains []vodDomain `json:"Domains"` + } `json:"Result"` +} + +type applyUploadInnerResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + InnerUploadAddress struct { + UploadNodes []struct { + StoreInfos []struct { + StoreURI string `json:"StoreUri"` + Auth string `json:"Auth"` + UploadID string `json:"UploadID"` + StorageHeader map[string]any `json:"StorageHeader"` + } `json:"StoreInfos"` + UploadHost string `json:"UploadHost"` + SessionKey string `json:"SessionKey"` + } `json:"UploadNodes"` + } `json:"InnerUploadAddress"` + } `json:"Result"` +} + +type tosUploadResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Crc32 string `json:"crc32"` + UploadID string `json:"uploadid"` + PartNumber string `json:"part_number"` + Etag string `json:"etag"` + } `json:"data"` +} + +type commitUploadInnerResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + Results []struct { + URI string `json:"Uri"` + URIStatus int `json:"UriStatus"` + Vid string `json:"Vid"` + } `json:"Results"` + PluginResult any `json:"PluginResult"` + } `json:"Result"` +} + +type uploadSubmitResp struct { + Code int `json:"code"` + Message string `json:"message"` +} From a673503fe2afd5af6a8bef512d86a8c30bcfa490 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 18 Mar 2026 16:28:06 +0800 Subject: [PATCH 087/133] feat(s3): add placeholder toggle and fix empty-folder delete behavior --- drivers/s3/driver.go | 47 ++++++++++++++++++++++++++++++++++---------- drivers/s3/meta.go | 1 + drivers/s3/util.go | 23 +++++++++++++++++++--- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index 7825ca6f935..896f69b3028 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -69,6 +69,9 @@ func (d *S3) GetAddition() driver.Additional { } func (d *S3) Init(ctx context.Context) error { + if !strings.Contains(d.Storage.Addition, `"use_placeholder"`) { + d.UsePlaceholder = true + } if d.Region == "" { d.Region = "alist" } @@ -151,16 +154,20 @@ func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*mo } func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - return d.Put(ctx, &model.Object{ - Path: stdpath.Join(parentDir.GetPath(), dirName), - }, &stream.FileStream{ - Obj: &model.Object{ - Name: getPlaceholderName(d.Placeholder), - Modified: time.Now(), - }, - Reader: io.NopCloser(bytes.NewReader([]byte{})), - Mimetype: "application/octet-stream", - }, func(float64) {}) + dirPath := stdpath.Join(parentDir.GetPath(), dirName) + if d.UsePlaceholder { + return d.Put(ctx, &model.Object{ + Path: dirPath, + }, &stream.FileStream{ + Obj: &model.Object{ + Name: getPlaceholderName(d.Placeholder), + Modified: time.Now(), + }, + Reader: io.NopCloser(bytes.NewReader([]byte{})), + Mimetype: "application/octet-stream", + }, func(float64) {}) + } + return d.createDirMarker(ctx, dirPath) } func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -214,6 +221,26 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up return err } +func (d *S3) putEmptyObject(ctx context.Context, key string) error { + uploader := s3manager.NewUploader(d.Session) + contentType := "application/octet-stream" + input := &s3manager.UploadInput{ + Bucket: &d.Bucket, + Key: &key, + Body: driver.NewLimitedUploadStream(ctx, bytes.NewReader([]byte{})), + ContentType: &contentType, + } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } + _, err := uploader.UploadWithContext(ctx, input) + return err +} + +func (d *S3) createDirMarker(ctx context.Context, dirPath string) error { + return d.putEmptyObject(ctx, getKey(dirPath, true)) +} + var ( _ driver.Driver = (*S3)(nil) _ driver.Other = (*S3)(nil) diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 89d723b60b5..0c675e85646 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -19,6 +19,7 @@ type Addition struct { Placeholder string `json:"placeholder"` ForcePathStyle bool `json:"force_path_style"` ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` + UsePlaceholder bool `json:"use_placeholder" default:"true" help:"Create hidden placeholder file (for example .alist) to keep empty folders."` RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` StorageClass string `json:"storage_class" type:"select" options:",standard,standard_ia,onezone_ia,intelligent_tiering,glacier,glacier_ir,deep_archive,archive" help:"Storage class for new objects. AWS and Tencent COS support different subsets (COS uses ARCHIVE/DEEP_ARCHIVE)."` diff --git a/drivers/s3/util.go b/drivers/s3/util.go index 9d2b285ce1d..863b88bcab8 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -8,6 +8,7 @@ import ( "path" "strings" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" @@ -105,6 +106,9 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { files = append(files, &file) } for _, object := range listObjectsResult.Contents { + if strings.HasSuffix(*object.Key, "/") { + continue + } name := path.Base(*object.Key) if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue @@ -210,10 +214,13 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error { } func (d *S3) copyDir(ctx context.Context, src string, dst string) error { - objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) + objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true, Refresh: true}) if err != nil { return err } + if len(objs) == 0 && !d.UsePlaceholder { + return d.createDirMarker(ctx, dst) + } for _, obj := range objs { cSrc := path.Join(src, obj.GetName()) cDst := path.Join(dst, obj.GetName()) @@ -230,8 +237,13 @@ func (d *S3) copyDir(ctx context.Context, src string, dst string) error { } func (d *S3) removeDir(ctx context.Context, src string) error { - objs, err := op.List(ctx, d, src, model.ListArgs{}) + d.cleanupDirArtifacts(src) + + objs, err := op.List(ctx, d, src, model.ListArgs{Refresh: true}) if err != nil { + if errs.IsObjectNotFound(err) { + return nil + } return err } for _, obj := range objs { @@ -245,9 +257,14 @@ func (d *S3) removeDir(ctx context.Context, src string) error { return err } } + d.cleanupDirArtifacts(src) + return nil +} + +func (d *S3) cleanupDirArtifacts(src string) { _ = d.removeFile(path.Join(src, getPlaceholderName(d.Placeholder))) _ = d.removeFile(path.Join(src, d.Placeholder)) - return nil + _ = d.removeFile(getKey(src, true)) } func (d *S3) removeFile(src string) error { From a8ff7c99950573092bb24a26b6b9b4601d4af08f Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 19 Mar 2026 11:37:25 +0800 Subject: [PATCH 088/133] feat: add public share support --- internal/db/db.go | 2 +- internal/db/share.go | 66 ++++++ internal/model/share.go | 31 +++ internal/share/access.go | 31 +++ server/handles/share.go | 368 +++++++++++++++++++++++++++++++++ server/handles/share_page.go | 16 ++ server/handles/share_public.go | 259 +++++++++++++++++++++++ server/router.go | 18 ++ 8 files changed, 790 insertions(+), 1 deletion(-) create mode 100644 internal/db/share.go create mode 100644 internal/model/share.go create mode 100644 internal/share/access.go create mode 100644 server/handles/share.go create mode 100644 server/handles/share_page.go create mode 100644 server/handles/share_public.go diff --git a/internal/db/db.go b/internal/db/db.go index 4577059d4f8..f33927be0ff 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile), new(model.Session)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile), new(model.Session), new(model.Share)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/share.go b/internal/db/share.go new file mode 100644 index 00000000000..54eb3d5f2f9 --- /dev/null +++ b/internal/db/share.go @@ -0,0 +1,66 @@ +package db + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" + "gorm.io/gorm" +) + +func GetShareByShareID(shareID string) (*model.Share, error) { + var share model.Share + if err := db.Where("share_id = ?", shareID).Take(&share).Error; err != nil { + return nil, err + } + return &share, nil +} + +func ShareIDExists(shareID string) bool { + var count int64 + if err := db.Model(&model.Share{}).Where("share_id = ?", shareID).Count(&count).Error; err != nil { + return false + } + return count > 0 +} + +func CreateShare(share *model.Share) error { + return db.Create(share).Error +} + +func UpdateShare(share *model.Share) error { + return db.Save(share).Error +} + +func GetSharesByCreator(creatorID uint, pageIndex, pageSize int) (shares []model.Share, count int64, err error) { + tx := db.Model(&model.Share{}).Where("creator_id = ?", creatorID) + err = tx.Count(&count).Error + if err != nil { + return nil, 0, err + } + err = tx.Order("created_at desc").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&shares).Error + return +} + +func DeleteShareByShareID(creatorID uint, shareID string) error { + return db.Where("creator_id = ? AND share_id = ?", creatorID, shareID).Delete(&model.Share{}).Error +} + +func TouchShareView(shareID string) error { + now := time.Now() + return db.Model(&model.Share{}). + Where("share_id = ?", shareID). + UpdateColumns(map[string]interface{}{ + "last_access_at": now, + "view_count": gorm.Expr("view_count + ?", 1), + }).Error +} + +func TouchShareDownload(shareID string) error { + now := time.Now() + return db.Model(&model.Share{}). + Where("share_id = ?", shareID). + UpdateColumns(map[string]interface{}{ + "last_access_at": now, + "download_count": gorm.Expr("download_count + ?", 1), + }).Error +} diff --git a/internal/model/share.go b/internal/model/share.go new file mode 100644 index 00000000000..a3fc90af008 --- /dev/null +++ b/internal/model/share.go @@ -0,0 +1,31 @@ +package model + +import "time" + +type Share struct { + ID uint `json:"id" gorm:"primaryKey"` + ShareID string `json:"share_id" gorm:"uniqueIndex;size:32;not null"` + CreatorID uint `json:"creator_id" gorm:"index;not null"` + Name string `json:"name" gorm:"size:255;not null"` + RootPath string `json:"root_path" gorm:"size:4096;not null"` + IsDir bool `json:"is_dir"` + PasswordHash string `json:"-" gorm:"size:64"` + PasswordSalt string `json:"-" gorm:"size:32"` + AllowPreview bool `json:"allow_preview" gorm:"default:true"` + AllowDownload bool `json:"allow_download" gorm:"default:true"` + Enabled bool `json:"enabled" gorm:"default:true;index"` + ViewCount int64 `json:"view_count"` + DownloadCount int64 `json:"download_count"` + LastAccessAt *time.Time `json:"last_access_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (s Share) HasPassword() bool { + return s.PasswordHash != "" +} + +func (s Share) IsExpired(now time.Time) bool { + return s.ExpiresAt != nil && s.ExpiresAt.Before(now) +} diff --git a/internal/share/access.go b/internal/share/access.go new file mode 100644 index 00000000000..7baa34e3ef9 --- /dev/null +++ b/internal/share/access.go @@ -0,0 +1,31 @@ +package share + +import ( + "fmt" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + signPkg "github.com/alist-org/alist/v3/pkg/sign" +) + +func tokenPayload(share *model.Share) string { + updatedAt := int64(0) + if !share.UpdatedAt.IsZero() { + updatedAt = share.UpdatedAt.Unix() + } + return fmt.Sprintf("%s:%s:%d", share.ShareID, share.PasswordHash, updatedAt) +} + +func signer() signPkg.Sign { + return signPkg.NewHMACSign([]byte(setting.GetStr(conf.Token) + "-share-access")) +} + +func SignAccess(share *model.Share, d time.Duration) string { + return signer().Sign(tokenPayload(share), time.Now().Add(d).Unix()) +} + +func VerifyAccess(share *model.Share, token string) error { + return signer().Verify(tokenPayload(share), token) +} diff --git a/server/handles/share.go b/server/handles/share.go new file mode 100644 index 00000000000..4626da8f0a2 --- /dev/null +++ b/server/handles/share.go @@ -0,0 +1,368 @@ +package handles + +import ( + "crypto/subtle" + "fmt" + "net/url" + stdpath "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/db" + shareauth "github.com/alist-org/alist/v3/internal/share" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +const shareAccessTokenLifetime = 24 * time.Hour + +type CreateShareReq struct { + Path string `json:"path" binding:"required"` + Name string `json:"name"` + Password string `json:"password"` + ExpireHours int64 `json:"expire_hours"` + AllowPreview *bool `json:"allow_preview"` + AllowDownload *bool `json:"allow_download"` +} + +type ShareDeleteReq struct { + ShareID string `json:"share_id" binding:"required"` +} + +type ShareAuthReq struct { + ShareID string `json:"share_id" binding:"required"` + Password string `json:"password"` +} + +type PublicShareReq struct { + ShareID string `json:"share_id" binding:"required"` + Path string `json:"path"` + Token string `json:"token"` +} + +type PublicShareListReq struct { + model.PageReq + ShareID string `json:"share_id" binding:"required"` + Path string `json:"path"` + Token string `json:"token"` +} + +type ShareResp struct { + ID uint `json:"id"` + ShareID string `json:"share_id"` + Name string `json:"name"` + RootPath string `json:"root_path"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Enabled bool `json:"enabled"` + ViewCount int64 `json:"view_count"` + DownloadCount int64 `json:"download_count"` + LastAccessAt *time.Time `json:"last_access_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + URL string `json:"url"` +} + +type PublicShareInfoResp struct { + ShareID string `json:"share_id"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Authed bool `json:"authed"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` +} + +type PublicShareObjResp struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Thumb string `json:"thumb"` + Type int `json:"type"` + Path string `json:"path"` + StorageClass string `json:"storage_class,omitempty"` + DownloadURL string `json:"download_url,omitempty"` + PreviewURL string `json:"preview_url,omitempty"` +} + +type PublicShareListResp struct { + Content []PublicShareObjResp `json:"content"` + Total int64 `json:"total"` + Page int `json:"page"` + PerPage int `json:"per_page"` + HasMore bool `json:"has_more"` + PagesTotal int `json:"pages_total"` +} + +type PublicShareGetResp struct { + Item PublicShareObjResp `json:"item"` + Provider string `json:"provider"` +} + +func shareURL(c *gin.Context, shareID string) string { + return fmt.Sprintf("%s/s/%s", common.GetApiUrl(c.Request), shareID) +} + +func toShareResp(c *gin.Context, share *model.Share) ShareResp { + return ShareResp{ + ID: share.ID, + ShareID: share.ShareID, + Name: share.Name, + RootPath: share.RootPath, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Enabled: share.Enabled, + ViewCount: share.ViewCount, + DownloadCount: share.DownloadCount, + LastAccessAt: share.LastAccessAt, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, + UpdatedAt: share.UpdatedAt, + URL: shareURL(c, share.ShareID), + } +} + +func normalizeShareName(obj model.Obj, name string) string { + if strings.TrimSpace(name) != "" { + return strings.TrimSpace(name) + } + return obj.GetName() +} + +func generateShareID() (string, error) { + for range 10 { + shareID := random.String(8) + if !db.ShareIDExists(shareID) { + return shareID, nil + } + } + return "", fmt.Errorf("failed to generate unique share id") +} + +func sharePasswordHash(password, salt string) string { + return model.HashPwd(model.StaticHash(password), salt) +} + +func sharePasswordMatched(share *model.Share, password string) bool { + if !share.HasPassword() { + return true + } + hash := sharePasswordHash(password, share.PasswordSalt) + return subtle.ConstantTimeCompare([]byte(hash), []byte(share.PasswordHash)) == 1 +} + +func getShareAccessToken(c *gin.Context, fallback string) string { + if fallback != "" { + return fallback + } + if token := c.Query("auth"); token != "" { + return token + } + return c.GetHeader("X-Share-Token") +} + +func ensureShareAvailable(c *gin.Context, share *model.Share) bool { + now := time.Now() + if !share.Enabled { + common.ErrorStrResp(c, "share is disabled", 404) + return false + } + if share.IsExpired(now) { + common.ErrorStrResp(c, "share is expired", 410) + return false + } + return true +} + +func ensureShareAccess(c *gin.Context, share *model.Share, token string) bool { + if !share.HasPassword() { + return true + } + if token == "" { + common.ErrorStrResp(c, "share password required", 401) + return false + } + if err := shareauth.VerifyAccess(share, token); err != nil { + common.ErrorResp(c, err, 401) + return false + } + return true +} + +func resolveShareTarget(share *model.Share, rawRelPath string) (string, string, error) { + cleanRelPath := utils.FixAndCleanPath(rawRelPath) + if !share.IsDir && cleanRelPath != "/" { + return "", "", fmt.Errorf("file share does not support nested path") + } + if cleanRelPath == "/" { + return share.RootPath, "/", nil + } + target := utils.FixAndCleanPath(stdpath.Join(share.RootPath, cleanRelPath)) + if !utils.IsSubPath(share.RootPath, target) { + return "", "", fmt.Errorf("share path out of range") + } + return target, cleanRelPath, nil +} + +func buildPublicShareAssetURL(c *gin.Context, prefix, shareID, relPath, token string, preview bool) string { + base := common.GetApiUrl(c.Request) + prefix + shareID + cleanPath := utils.FixAndCleanPath(relPath) + if cleanPath != "/" { + base += utils.EncodePath(cleanPath, true) + } + query := url.Values{} + if token != "" { + query.Set("auth", token) + } + if preview { + query.Set("type", "preview") + } + if encoded := query.Encode(); encoded != "" { + base += "?" + encoded + } + return base +} + +func buildPublicSharePreviewURL(c *gin.Context, obj model.Obj, targetPath, shareID, relPath, token string) string { + prefix := "/sd/" + storage, err := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if err == nil && canProxy(storage, obj.GetName()) { + prefix = "/sp/" + } + return buildPublicShareAssetURL(c, prefix, shareID, relPath, token, true) +} + +func toPublicShareObjResp(c *gin.Context, share *model.Share, obj model.Obj, targetPath, relPath, token string) PublicShareObjResp { + thumb, _ := model.GetThumb(obj) + storageClass, _ := model.GetStorageClass(obj) + resp := PublicShareObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + Thumb: thumb, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + Path: relPath, + StorageClass: storageClass, + } + if !obj.IsDir() && share.AllowDownload { + resp.DownloadURL = buildPublicShareAssetURL(c, "/sd/", share.ShareID, relPath, token, false) + } + if !obj.IsDir() && share.AllowPreview { + resp.PreviewURL = buildPublicSharePreviewURL(c, obj, targetPath, share.ShareID, relPath, token) + } + return resp +} + +func CreateShare(c *gin.Context) { + var req CreateShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CanReadPathByRole(user, reqPath) { + common.ErrorStrResp(c, "you have no permission", 403) + return + } + obj, err := fs.Get(c, reqPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + shareID, err := generateShareID() + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + allowPreview := true + if req.AllowPreview != nil { + allowPreview = *req.AllowPreview + } + allowDownload := true + if req.AllowDownload != nil { + allowDownload = *req.AllowDownload + } + var expiresAt *time.Time + if req.ExpireHours > 0 { + expires := time.Now().Add(time.Duration(req.ExpireHours) * time.Hour) + expiresAt = &expires + } + share := &model.Share{ + ShareID: shareID, + CreatorID: user.ID, + Name: normalizeShareName(obj, req.Name), + RootPath: reqPath, + IsDir: obj.IsDir(), + AllowPreview: allowPreview, + AllowDownload: allowDownload, + Enabled: true, + ExpiresAt: expiresAt, + } + if req.Password != "" { + share.PasswordSalt = random.String(16) + share.PasswordHash = sharePasswordHash(req.Password, share.PasswordSalt) + } + if err := db.CreateShare(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, toShareResp(c, share)) +} + +func ListShares(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + user := c.MustGet("user").(*model.User) + shares, total, err := db.GetSharesByCreator(user.ID, req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + resp := make([]ShareResp, 0, len(shares)) + for i := range shares { + resp = append(resp, toShareResp(c, &shares[i])) + } + common.SuccessResp(c, common.PageResp{ + Content: resp, + Total: total, + }) +} + +func DeleteShare(c *gin.Context) { + var req ShareDeleteReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if err := db.DeleteShareByShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/share_page.go b/server/handles/share_page.go new file mode 100644 index 00000000000..98cc62d8daf --- /dev/null +++ b/server/handles/share_page.go @@ -0,0 +1,16 @@ +package handles + +import ( + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/gin-gonic/gin" +) + +func GetSharePage(c *gin.Context) { + c.Header("Content-Type", "text/html") + c.Status(200) + _, _ = c.Writer.WriteString(strings.Replace(conf.IndexHtml, "", "", 1)) + c.Writer.Flush() + c.Writer.WriteHeaderNow() +} diff --git a/server/handles/share_public.go b/server/handles/share_public.go new file mode 100644 index 00000000000..36476bae466 --- /dev/null +++ b/server/handles/share_public.go @@ -0,0 +1,259 @@ +package handles + +import ( + stdpath "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/db" + shareauth "github.com/alist-org/alist/v3/internal/share" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +func GetPublicShareInfo(c *gin.Context) { + shareID := c.Query("share_id") + if shareID == "" { + common.ErrorStrResp(c, "share_id is required", 400) + return + } + share, err := db.GetShareByShareID(shareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, "") + authed := !share.HasPassword() + if share.HasPassword() && token != "" { + authed = shareauth.VerifyAccess(share, token) == nil + } + if authed { + _ = db.TouchShareView(share.ShareID) + } + common.SuccessResp(c, PublicShareInfoResp{ + ShareID: share.ShareID, + Name: share.Name, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Authed: authed, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, + }) +} + +func AuthPublicShare(c *gin.Context) { + var req ShareAuthReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !sharePasswordMatched(share, req.Password) { + common.ErrorStrResp(c, "password is incorrect", 403) + return + } + token := "" + if share.HasPassword() { + ttl := shareAccessTokenLifetime + if share.ExpiresAt != nil { + remaining := time.Until(*share.ExpiresAt) + if remaining <= 0 { + common.ErrorStrResp(c, "share is expired", 410) + return + } + if remaining < ttl { + ttl = remaining + } + } + token = shareauth.SignAccess(share, ttl) + } + _ = db.TouchShareView(share.ShareID) + common.SuccessResp(c, gin.H{"token": token}) +} + +func ListPublicShare(c *gin.Context) { + var req PublicShareListReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Page, req.PerPage = normalizeListPage(req.Page, req.PerPage) + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, req.Token) + if !ensureShareAccess(c, share, token) { + return + } + targetPath, relPath, err := resolveShareTarget(share, req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !obj.IsDir() { + common.ErrorStrResp(c, "path is not a directory", 400) + return + } + objs, err := fs.List(c, targetPath, &fs.ListArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + total, pageObjs := pagination(objs, &req.PageReq) + content := make([]PublicShareObjResp, 0, len(pageObjs)) + for _, item := range pageObjs { + itemRelPath := relPath + if itemRelPath == "/" { + itemRelPath = stdpath.Join("/", item.GetName()) + } else { + itemRelPath = stdpath.Join(relPath, item.GetName()) + } + itemTargetPath, _, err := resolveShareTarget(share, itemRelPath) + if err != nil { + continue + } + content = append(content, toPublicShareObjResp(c, share, item, itemTargetPath, itemRelPath, token)) + } + common.SuccessResp(c, PublicShareListResp{ + Content: content, + Total: int64(total), + Page: req.Page, + PerPage: req.PerPage, + HasMore: req.Page*req.PerPage < total, + PagesTotal: calcPagesTotal(total, req.PerPage), + }) +} + +func GetPublicShare(c *gin.Context) { + var req PublicShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, req.Token) + if !ensureShareAccess(c, share, token) { + return + } + targetPath, relPath, err := resolveShareTarget(share, req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + provider := "unknown" + storage, storageErr := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if storageErr == nil { + provider = storage.GetStorage().Driver + } + common.SuccessResp(c, PublicShareGetResp{ + Item: toPublicShareObjResp(c, share, obj, targetPath, relPath, token), + Provider: provider, + }) +} + +func ShareDown(c *gin.Context) { + share, err := db.GetShareByShareID(c.Param("share_id")) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !share.AllowDownload { + common.ErrorStrResp(c, "download is not allowed", 403) + return + } + token := getShareAccessToken(c, "") + if !ensureShareAccess(c, share, token) { + return + } + targetPath, _, err := resolveShareTarget(share, strings.TrimPrefix(c.Param("path"), "/")) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if obj.IsDir() { + common.ErrorStrResp(c, "directory download is not supported", 400) + return + } + _ = db.TouchShareDownload(share.ShareID) + c.Set("path", targetPath) + Down(c) +} + +func ShareProxy(c *gin.Context) { + share, err := db.GetShareByShareID(c.Param("share_id")) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !share.AllowPreview { + common.ErrorStrResp(c, "preview is not allowed", 403) + return + } + token := getShareAccessToken(c, "") + if !ensureShareAccess(c, share, token) { + return + } + targetPath, _, err := resolveShareTarget(share, strings.TrimPrefix(c.Param("path"), "/")) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if obj.IsDir() { + common.ErrorStrResp(c, "directory preview is not supported", 400) + return + } + c.Set("path", targetPath) + Proxy(c) +} diff --git a/server/router.go b/server/router.go index 63503838af7..e63b667297e 100644 --- a/server/router.go +++ b/server/router.go @@ -47,6 +47,16 @@ func Init(e *gin.Engine) { g.GET("/p/*path", signCheck, downloadLimiter, handles.Proxy) g.HEAD("/d/*path", signCheck, handles.Down) g.HEAD("/p/*path", signCheck, handles.Proxy) + g.GET("/s/:share_id", handles.GetSharePage) + g.GET("/s/:share_id/*path", handles.GetSharePage) + g.GET("/sd/:share_id", downloadLimiter, handles.ShareDown) + g.GET("/sd/:share_id/*path", downloadLimiter, handles.ShareDown) + g.HEAD("/sd/:share_id", handles.ShareDown) + g.HEAD("/sd/:share_id/*path", handles.ShareDown) + g.GET("/sp/:share_id", downloadLimiter, handles.ShareProxy) + g.GET("/sp/:share_id/*path", downloadLimiter, handles.ShareProxy) + g.HEAD("/sp/:share_id", handles.ShareProxy) + g.HEAD("/sp/:share_id/*path", handles.ShareProxy) archiveSignCheck := middlewares.Down(sign.VerifyArchive) g.GET("/ad/*path", archiveSignCheck, downloadLimiter, handles.ArchiveDown) g.GET("/ap/*path", archiveSignCheck, downloadLimiter, handles.ArchiveProxy) @@ -93,8 +103,16 @@ func Init(e *gin.Engine) { public.Any("/settings", handles.PublicSettings) public.Any("/offline_download_tools", handles.OfflineDownloadTools) public.Any("/archive_extensions", handles.ArchiveExtensions) + public.GET("/share/info", handles.GetPublicShareInfo) + public.POST("/share/auth", handles.AuthPublicShare) + public.POST("/share/list", handles.ListPublicShare) + public.POST("/share/get", handles.GetPublicShare) _fs(auth.Group("/fs")) + share := auth.Group("/share", middlewares.AuthNotGuest) + share.POST("/create", handles.CreateShare) + share.GET("/list", handles.ListShares) + share.POST("/delete", handles.DeleteShare) _task(auth.Group("/task", middlewares.AuthNotGuest)) _label(auth.Group("/label")) _labelFileBinding(auth.Group("/label_file_binding")) From 35832d0431aa7bab16768abf863373c846956f58 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 24 Mar 2026 18:15:25 +0800 Subject: [PATCH 089/133] fix(139): recover personal host for personal_new requests --- drivers/139/driver.go | 24 ++++-------------------- drivers/139/util.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index a57609bc550..c182dd24e93 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -47,28 +47,12 @@ func (d *Yun139) Init(ctx context.Context) error { if err != nil { return err } - - // Query Route Policy - var resp QueryRoutePolicyResp - _, err = d.requestRoute(base.Json{ - "userInfo": base.Json{ - "userType": 1, - "accountType": 1, - "accountName": d.Account}, - "modAddrType": 1, - }, &resp) - if err != nil { - return err - } - for _, policyItem := range resp.Data.RoutePolicyList { - if policyItem.ModName == "personal" { - d.PersonalCloudHost = policyItem.HttpsUrl - break + if d.Addition.Type == MetaPersonalNew { + err = d.ensurePersonalCloudHost() + if err != nil { + return err } } - if len(d.PersonalCloudHost) == 0 { - return fmt.Errorf("PersonalCloudHost is empty") - } d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { diff --git a/drivers/139/util.go b/drivers/139/util.go index 5adc39b4116..79c78842470 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -215,6 +215,46 @@ func (d *Yun139) requestRoute(data interface{}, resp interface{}) ([]byte, error return res.Body(), nil } +func (d *Yun139) ensurePersonalCloudHost() error { + if d.ref != nil { + return d.ref.ensurePersonalCloudHost() + } + if d.PersonalCloudHost != "" { + return nil + } + if len(d.Authorization) == 0 { + return fmt.Errorf("authorization is empty") + } + if d.Account == "" { + if err := d.refreshToken(); err != nil { + return err + } + } + + var resp QueryRoutePolicyResp + _, err := d.requestRoute(base.Json{ + "userInfo": base.Json{ + "userType": 1, + "accountType": 1, + "accountName": d.Account, + }, + "modAddrType": 1, + }, &resp) + if err != nil { + return err + } + for _, policyItem := range resp.Data.RoutePolicyList { + if policyItem.ModName == "personal" && policyItem.HttpsUrl != "" { + d.PersonalCloudHost = strings.TrimRight(policyItem.HttpsUrl, "/") + break + } + } + if d.PersonalCloudHost == "" { + return fmt.Errorf("personal cloud host is empty") + } + return nil +} + func (d *Yun139) post(pathname string, data interface{}, resp interface{}) ([]byte, error) { return d.request(pathname, http.MethodPost, func(req *resty.Request) { req.SetBody(data) @@ -449,6 +489,9 @@ func unicode(str string) string { } func (d *Yun139) personalRequest(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if err := d.ensurePersonalCloudHost(); err != nil { + return nil, err + } url := d.getPersonalCloudHost() + pathname req := base.RestyClient.R() randStr := random.String(16) From 37a6b266be9b74697841db630ad41406753dfc64 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 25 Mar 2026 16:24:10 +0800 Subject: [PATCH 090/133] fix: handle aliyundrive open rate limits --- drivers/aliyundrive_open/driver.go | 47 ++++----- drivers/aliyundrive_open/limiter.go | 97 ++++++++++++++++++ drivers/aliyundrive_open/upload.go | 16 +-- drivers/aliyundrive_open/util.go | 150 +++++++++++++++++++++++++--- 4 files changed, 260 insertions(+), 50 deletions(-) create mode 100644 drivers/aliyundrive_open/limiter.go diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index 394eadb1b8c..5ef814184e3 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -3,12 +3,10 @@ package aliyundrive_open import ( "context" "errors" - "fmt" "net/http" "path/filepath" "time" - "github.com/Xhofe/rateg" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -24,9 +22,8 @@ type AliyundriveOpen struct { DriveId string - limitList func(ctx context.Context, data base.Json) (*Files, error) - limitLink func(ctx context.Context, file model.Obj) (*model.Link, error) - ref *AliyundriveOpen + limiter *limiter + ref *AliyundriveOpen } func (d *AliyundriveOpen) Config() driver.Config { @@ -38,25 +35,23 @@ func (d *AliyundriveOpen) GetAddition() driver.Additional { } func (d *AliyundriveOpen) Init(ctx context.Context) error { + d.limiter = getLimiterForUser(globalLimiterUserID) if d.LIVPDownloadFormat == "" { d.LIVPDownloadFormat = "jpeg" } if d.DriveType == "" { d.DriveType = "default" } - res, err := d.request("/adrive/v1.0/user/getDriveInfo", http.MethodPost, nil) + res, err := d.request(ctx, limiterOther, "/adrive/v1.0/user/getDriveInfo", http.MethodPost, nil) if err != nil { + d.limiter.free() + d.limiter = nil return err } d.DriveId = utils.Json.Get(res, d.DriveType+"_drive_id").ToString() - d.limitList = rateg.LimitFnCtx(d.list, rateg.LimitFnOption{ - Limit: 4, - Bucket: 1, - }) - d.limitLink = rateg.LimitFnCtx(d.link, rateg.LimitFnOption{ - Limit: 1, - Bucket: 1, - }) + userID := utils.Json.Get(res, "user_id").ToString() + d.limiter.free() + d.limiter = getLimiterForUser(userID) return nil } @@ -70,6 +65,8 @@ func (d *AliyundriveOpen) InitReference(storage driver.Driver) error { } func (d *AliyundriveOpen) Drop(ctx context.Context) error { + d.limiter.free() + d.limiter = nil d.ref = nil return nil } @@ -87,9 +84,6 @@ func (d *AliyundriveOpen) GetRoot(ctx context.Context) (model.Obj, error) { } func (d *AliyundriveOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - if d.limitList == nil { - return nil, fmt.Errorf("driver not init") - } files, err := d.getFiles(ctx, dir.GetID()) if err != nil { return nil, err @@ -108,7 +102,7 @@ func (d *AliyundriveOpen) List(ctx context.Context, dir model.Obj, args model.Li } func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link, error) { - res, err := d.request("/adrive/v1.0/openFile/getDownloadUrl", http.MethodPost, func(req *resty.Request) { + res, err := d.request(ctx, limiterLink, "/adrive/v1.0/openFile/getDownloadUrl", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": file.GetID(), @@ -133,16 +127,13 @@ func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link } func (d *AliyundriveOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - if d.limitLink == nil { - return nil, fmt.Errorf("driver not init") - } - return d.limitLink(ctx, file) + return d.link(ctx, file) } func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { nowTime, _ := getNowTime() newDir := File{CreatedAt: nowTime, UpdatedAt: nowTime} - _, err := d.request("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "parent_file_id": parentDir.GetID(), @@ -168,7 +159,7 @@ func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirN func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { var resp MoveOrCopyResp - _, err := d.request("/adrive/v1.0/openFile/move", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/move", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), @@ -198,7 +189,7 @@ func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (m func (d *AliyundriveOpen) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { var newFile File - _, err := d.request("/adrive/v1.0/openFile/update", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/update", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), @@ -230,7 +221,7 @@ func (d *AliyundriveOpen) Rename(ctx context.Context, srcObj model.Obj, newName func (d *AliyundriveOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { var resp MoveOrCopyResp - _, err := d.request("/adrive/v1.0/openFile/copy", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/copy", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), @@ -256,7 +247,7 @@ func (d *AliyundriveOpen) Remove(ctx context.Context, obj model.Obj) error { if d.RemoveWay == "delete" { uri = "/adrive/v1.0/openFile/delete" } - _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, uri, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": obj.GetID(), @@ -295,7 +286,7 @@ func (d *AliyundriveOpen) Other(ctx context.Context, args model.OtherArgs) (inte default: return nil, errs.NotSupport } - _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, uri, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { diff --git a/drivers/aliyundrive_open/limiter.go b/drivers/aliyundrive_open/limiter.go new file mode 100644 index 00000000000..5138fbe2e50 --- /dev/null +++ b/drivers/aliyundrive_open/limiter.go @@ -0,0 +1,97 @@ +package aliyundrive_open + +import ( + "context" + "fmt" + "sync" + + "golang.org/x/time/rate" +) + +// Aliyun Open API rate limits are per user per app, so requests for the same +// user should share one limiter across all storage instances. +type limiterType int + +const ( + limiterList limiterType = iota + limiterLink + limiterOther +) + +const ( + listRateLimit = 3.9 + linkRateLimit = 0.9 + otherRateLimit = 14.9 + globalLimiterUserID = "" +) + +type limiter struct { + usedBy int + list *rate.Limiter + link *rate.Limiter + other *rate.Limiter +} + +var ( + limiters = make(map[string]*limiter) + limitersLock sync.Mutex +) + +func getLimiterForUser(userID string) *limiter { + limitersLock.Lock() + defer limitersLock.Unlock() + defer func() { + for id, lim := range limiters { + if lim.usedBy <= 0 && id != globalLimiterUserID { + delete(limiters, id) + } + } + }() + if lim, ok := limiters[userID]; ok { + lim.usedBy++ + return lim + } + lim := &limiter{ + usedBy: 1, + list: rate.NewLimiter(rate.Limit(listRateLimit), 1), + link: rate.NewLimiter(rate.Limit(linkRateLimit), 1), + other: rate.NewLimiter(rate.Limit(otherRateLimit), 1), + } + limiters[userID] = lim + return lim +} + +func (l *limiter) wait(ctx context.Context, typ limiterType) error { + if l == nil { + return fmt.Errorf("driver not init") + } + switch typ { + case limiterList: + return l.list.Wait(ctx) + case limiterLink: + return l.link.Wait(ctx) + case limiterOther: + return l.other.Wait(ctx) + default: + return fmt.Errorf("unknown limiter type") + } +} + +func (l *limiter) free() { + if l == nil { + return + } + limitersLock.Lock() + defer limitersLock.Unlock() + l.usedBy-- +} + +func (d *AliyundriveOpen) wait(ctx context.Context, typ limiterType) error { + if d == nil { + return fmt.Errorf("driver not init") + } + if d.ref != nil { + return d.ref.wait(ctx, typ) + } + return d.limiter.wait(ctx, typ) +} diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index 4114c195182..19a3c3d0f05 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -50,10 +50,10 @@ func calPartSize(fileSize int64) int64 { return partSize } -func (d *AliyundriveOpen) getUploadUrl(count int, fileId, uploadId string) ([]PartInfo, error) { +func (d *AliyundriveOpen) getUploadUrl(ctx context.Context, count int, fileId, uploadId string) ([]PartInfo, error) { partInfoList := makePartInfos(count) var resp CreateResp - _, err := d.request("/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": fileId, @@ -84,10 +84,10 @@ func (d *AliyundriveOpen) uploadPart(ctx context.Context, r io.Reader, partInfo return nil } -func (d *AliyundriveOpen) completeUpload(fileId, uploadId string) (model.Obj, error) { +func (d *AliyundriveOpen) completeUpload(ctx context.Context, fileId, uploadId string) (model.Obj, error) { // 3. complete var newFile File - _, err := d.request("/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": fileId, @@ -183,7 +183,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m createData["pre_hash"] = hash } var createResp CreateResp - _, err, e := d.requestReturnErrResp("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err, e := d.requestReturnErrResp(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) if err != nil { @@ -208,7 +208,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m if err != nil { return nil, fmt.Errorf("cal proof code error: %s", err.Error()) } - _, err = d.request("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err = d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) if err != nil { @@ -229,7 +229,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m } // refresh upload url if 50 minutes passed if time.Since(preTime) > 50*time.Minute { - createResp.PartInfoList, err = d.getUploadUrl(count, createResp.FileId, createResp.UploadId) + createResp.PartInfoList, err = d.getUploadUrl(ctx, count, createResp.FileId, createResp.UploadId) if err != nil { return nil, err } @@ -266,5 +266,5 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m log.Debugf("[aliyundrive_open] create file success, resp: %+v", createResp) // 3. complete - return d.completeUpload(createResp.FileId, createResp.UploadId) + return d.completeUpload(ctx, createResp.FileId, createResp.UploadId) } diff --git a/drivers/aliyundrive_open/util.go b/drivers/aliyundrive_open/util.go index c3cda10aa88..544c8bdd10b 100644 --- a/drivers/aliyundrive_open/util.go +++ b/drivers/aliyundrive_open/util.go @@ -19,13 +19,94 @@ import ( // do others that not defined in Driver interface -func (d *AliyundriveOpen) _refreshToken() (string, string, error) { - url := API_URL + "/oauth/access_token" +const legacyOauthTokenURL = "https://api.alistgo.com/alist/ali_open/token" + +type refreshRateLimitError struct { + message string + retryAfter time.Duration +} + +func (e *refreshRateLimitError) Error() string { + if e.retryAfter > 0 { + return fmt.Sprintf("%s, retry after %s", e.message, e.retryAfter.Round(time.Second)) + } + return e.message +} + +func (d *AliyundriveOpen) _refreshToken(ctx context.Context) (string, string, error) { if d.OauthTokenURL != "" && d.ClientID == "" { - url = d.OauthTokenURL + return d.refreshTokenWithOnlineAPI(ctx) + } + + if d.ClientID == "" || d.ClientSecret == "" { + return "", "", fmt.Errorf("empty ClientID or ClientSecret") + } + return d.refreshTokenWithPost(ctx, API_URL+"/oauth/access_token") +} + +func (d *AliyundriveOpen) refreshTokenWithOnlineAPI(ctx context.Context) (string, string, error) { + // New hosted renew endpoint uses a GET API and returns {"refresh_token","access_token","text"}. + // Older hosted endpoints still expect the legacy POST payload, so we fall back when we detect that shape. + var resp struct { + RefreshToken string `json:"refresh_token"` + AccessToken string `json:"access_token"` + ErrorMessage string `json:"text"` + } + var e ErrResp + if err := d.wait(ctx, limiterOther); err != nil { + return "", "", err + } + res, err := base.RestyClient.R(). + SetResult(&resp). + SetError(&e). + SetQueryParams(map[string]string{ + "refresh_ui": d.RefreshToken, + "server_use": "true", + "driver_txt": "alicloud_qr", + }). + Get(d.OauthTokenURL) + if err != nil { + return "", "", err + } + if resp.RefreshToken != "" && resp.AccessToken != "" { + return resp.RefreshToken, resp.AccessToken, nil + } + if resp.ErrorMessage != "" { + if res != nil && res.StatusCode() == http.StatusTooManyRequests { + return d.refreshTokenWithLegacyFallback(ctx, resp.ErrorMessage, retryAfterFromResponse(res)) + } + return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) + } + if res != nil && res.StatusCode() == http.StatusTooManyRequests { + return d.refreshTokenWithLegacyFallback(ctx, http.StatusText(http.StatusTooManyRequests), retryAfterFromResponse(res)) } + if e.Code != "" || e.Message != "" { + return d.refreshTokenWithPost(ctx, d.OauthTokenURL) + } + return "", "", fmt.Errorf("empty token returned from online API") +} + +func (d *AliyundriveOpen) refreshTokenWithLegacyFallback(ctx context.Context, message string, retryAfter time.Duration) (string, string, error) { + if d.OauthTokenURL != legacyOauthTokenURL { + log.Warnf("[ali_open] online refresh API is rate-limited, trying legacy fallback: %s", legacyOauthTokenURL) + if refresh, access, err := d.refreshTokenWithPost(ctx, legacyOauthTokenURL); err == nil { + return refresh, access, nil + } else if _, ok := err.(*refreshRateLimitError); !ok { + return "", "", err + } + } + return "", "", &refreshRateLimitError{ + message: fmt.Sprintf("failed to refresh token: %s", message), + retryAfter: retryAfter, + } +} + +func (d *AliyundriveOpen) refreshTokenWithPost(ctx context.Context, url string) (string, string, error) { //var resp base.TokenResp var e ErrResp + if err := d.wait(ctx, limiterOther); err != nil { + return "", "", err + } res, err := base.RestyClient.R(). //ForceContentType("application/json"). SetBody(base.Json{ @@ -41,6 +122,16 @@ func (d *AliyundriveOpen) _refreshToken() (string, string, error) { return "", "", err } log.Debugf("[ali_open] refresh token response: %s", res.String()) + if res.StatusCode() == http.StatusTooManyRequests { + message := e.Message + if message == "" { + message = http.StatusText(http.StatusTooManyRequests) + } + return "", "", &refreshRateLimitError{ + message: fmt.Sprintf("failed to refresh token: %s", message), + retryAfter: retryAfterFromResponse(res), + } + } if e.Code != "" { return "", "", fmt.Errorf("failed to refresh token: %s", e.Message) } @@ -74,18 +165,29 @@ func getSub(token string) (string, error) { return utils.Json.Get(bs, "sub").ToString(), nil } -func (d *AliyundriveOpen) refreshToken() error { +func (d *AliyundriveOpen) refreshToken(ctx context.Context) error { if d.ref != nil { - return d.ref.refreshToken() + return d.ref.refreshToken(ctx) } - refresh, access, err := d._refreshToken() + refresh, access, err := d._refreshToken(ctx) for i := 0; i < 3; i++ { if err == nil { break + } + if rateLimitErr, ok := err.(*refreshRateLimitError); ok { + wait := rateLimitErr.retryAfter + if wait <= 0 { + wait = time.Duration(i+1) * 2 * time.Second + } + if wait > 15*time.Second { + wait = 15 * time.Second + } + log.Warnf("[ali_open] %s", rateLimitErr.Error()) + time.Sleep(wait) } else { log.Errorf("[ali_open] failed to refresh token: %s", err) } - refresh, access, err = d._refreshToken() + refresh, access, err = d._refreshToken(ctx) } if err != nil { return err @@ -96,12 +198,29 @@ func (d *AliyundriveOpen) refreshToken() error { return nil } -func (d *AliyundriveOpen) request(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { - b, err, _ := d.requestReturnErrResp(uri, method, callback, retry...) +func retryAfterFromResponse(res *resty.Response) time.Duration { + if res == nil { + return 0 + } + retryAfter := strings.TrimSpace(res.Header().Get("Retry-After")) + if retryAfter == "" { + return 0 + } + if seconds, err := time.ParseDuration(retryAfter + "s"); err == nil { + return seconds + } + if t, err := http.ParseTime(retryAfter); err == nil { + return time.Until(t) + } + return 0 +} + +func (d *AliyundriveOpen) request(ctx context.Context, limitTy limiterType, uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { + b, err, _ := d.requestReturnErrResp(ctx, limitTy, uri, method, callback, retry...) return b, err } -func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error, *ErrResp) { +func (d *AliyundriveOpen) requestReturnErrResp(ctx context.Context, limitTy limiterType, uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error, *ErrResp) { req := base.RestyClient.R() // TODO check whether access_token is expired req.SetHeader("Authorization", "Bearer "+d.getAccessToken()) @@ -113,6 +232,9 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base } var e ErrResp req.SetError(&e) + if err := d.wait(ctx, limitTy); err != nil { + return nil, err, nil + } res, err := req.Execute(method, API_URL+uri) if err != nil { if res != nil { @@ -123,11 +245,11 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base isRetry := len(retry) > 0 && retry[0] if e.Code != "" { if !isRetry && (utils.SliceContains([]string{"AccessTokenInvalid", "AccessTokenExpired", "I400JD"}, e.Code) || d.getAccessToken() == "") { - err = d.refreshToken() + err = d.refreshToken(ctx) if err != nil { return nil, err, nil } - return d.requestReturnErrResp(uri, method, callback, true) + return d.requestReturnErrResp(ctx, limitTy, uri, method, callback, true) } return nil, fmt.Errorf("%s:%s", e.Code, e.Message), &e } @@ -136,7 +258,7 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base func (d *AliyundriveOpen) list(ctx context.Context, data base.Json) (*Files, error) { var resp Files - _, err := d.request("/adrive/v1.0/openFile/list", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterList, "/adrive/v1.0/openFile/list", http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { @@ -165,7 +287,7 @@ func (d *AliyundriveOpen) getFiles(ctx context.Context, fileId string) ([]File, //"video_thumbnail_width": 480, //"image_thumbnail_width": 480, } - resp, err := d.limitList(ctx, data) + resp, err := d.list(ctx, data) if err != nil { return nil, err } From c07cd1bf5b99d762a8ec4f4e5f68eec2cf14056f Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 25 Mar 2026 20:48:09 +0800 Subject: [PATCH 091/133] feat(driver): add baidu youth driver --- drivers/all.go | 1 + drivers/baidu_youth/driver.go | 425 ++++++++++++++++++++++++ drivers/baidu_youth/meta.go | 40 +++ drivers/baidu_youth/types.go | 130 ++++++++ drivers/baidu_youth/util.go | 606 ++++++++++++++++++++++++++++++++++ internal/driver/proxy.go | 7 + server/common/check.go | 6 +- server/handles/fsread.go | 19 +- 8 files changed, 1228 insertions(+), 6 deletions(-) create mode 100644 drivers/baidu_youth/driver.go create mode 100644 drivers/baidu_youth/meta.go create mode 100644 drivers/baidu_youth/types.go create mode 100644 drivers/baidu_youth/util.go create mode 100644 internal/driver/proxy.go diff --git a/drivers/all.go b/drivers/all.go index a4fce9d0cac..1f096a86224 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -21,6 +21,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_netdisk" _ "github.com/alist-org/alist/v3/drivers/baidu_photo" _ "github.com/alist-org/alist/v3/drivers/baidu_share" + _ "github.com/alist-org/alist/v3/drivers/baidu_youth" _ "github.com/alist-org/alist/v3/drivers/bitqiu" _ "github.com/alist-org/alist/v3/drivers/chaoxing" _ "github.com/alist-org/alist/v3/drivers/cloudreve" diff --git a/drivers/baidu_youth/driver.go b/drivers/baidu_youth/driver.go new file mode 100644 index 00000000000..a0b6a4853ec --- /dev/null +++ b/drivers/baidu_youth/driver.go @@ -0,0 +1,425 @@ +package baidu_youth + +import ( + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "os" + stdpath "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/errgroup" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" +) + +type BaiduYouth struct { + model.Storage + Addition + + uk int64 + bdstoken string + sk string + uploadThread int + upClient *resty.Client +} + +var ErrUploadIDExpired = errors.New("uploadid expired") + +func (d *BaiduYouth) Config() driver.Config { + return config +} + +func (d *BaiduYouth) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *BaiduYouth) Init(ctx context.Context) error { + d.Cookie = strings.TrimSpace(d.Cookie) + if d.Storage.Addition != "" { + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(d.Storage.Addition), &raw); err == nil { + if _, ok := raw["force_proxy"]; !ok { + d.ForceProxy = true + } + } + } + d.upClient = base.NewRestyClient(). + SetTimeout(UPLOAD_TIMEOUT). + SetRetryCount(UPLOAD_RETRY_COUNT). + SetRetryWaitTime(UPLOAD_RETRY_WAIT_TIME). + SetRetryMaxWaitTime(UPLOAD_RETRY_MAX_WAIT_TIME) + + d.uploadThread, _ = strconv.Atoi(d.UploadThread) + if d.uploadThread < 1 { + d.uploadThread, d.UploadThread = 1, "1" + } else if d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 32, "32" + } + + u, err := url.Parse(d.UploadAPI) + if d.UploadAPI == "" || err != nil || u.Scheme == "" || u.Host == "" { + d.UploadAPI = UPLOAD_FALLBACK_API + } else { + d.UploadAPI = strings.TrimRight(d.UploadAPI, "/") + } + + uk, bdstoken, sk, err := d.getUserSession(ctx) + if err != nil { + return err + } + d.uk = uk + d.bdstoken = bdstoken + d.sk = sk + return nil +} + +func (d *BaiduYouth) ShouldProxyDownloads() bool { + return d.ForceProxy +} + +func (d *BaiduYouth) Drop(ctx context.Context) error { + d.uk = 0 + d.bdstoken = "" + d.sk = "" + return nil +} + +func (d *BaiduYouth) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(ctx, dir.GetPath()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *BaiduYouth) Get(ctx context.Context, path string) (model.Obj, error) { + return d.getByPath(ctx, path) +} + +func (d *BaiduYouth) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if d.DownloadAPI == "crack" { + return d.linkCrack(ctx, file) + } + return d.linkOfficial(ctx, file) +} + +func (d *BaiduYouth) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + path := stdpath.Join(parentDir.GetPath(), dirName) + var resp CreateResp + _, err := d.postForm(ctx, "/youth/api/create", map[string]string{ + "a": "commit", + "bdstoken": d.bdstoken, + }, map[string]string{ + "block_list": "[]", + "isdir": "1", + "path": path, + }, &resp) + if err != nil { + return nil, err + } + if result := resp.ResultFile(); result.Path != "" || result.FsId != 0 { + return fileToObj(result), nil + } + return d.getByPath(ctx, path) +} + +func (d *BaiduYouth) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + _, err := d.manage(ctx, "move", []base.Json{ + { + "dest": dstDir.GetPath(), + "newname": srcObj.GetName(), + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + newPath := stdpath.Join(dstDir.GetPath(), srcObj.GetName()) + if obj, ok := srcObj.(*model.ObjThumb); ok { + obj.SetPath(newPath) + obj.Modified = time.Now() + return obj, nil + } + return d.getByPath(ctx, newPath) +} + +func (d *BaiduYouth) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + _, err := d.manage(ctx, "rename", []base.Json{ + { + "id": srcObj.GetID(), + "newname": newName, + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + newPath := stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName) + if obj, ok := srcObj.(*model.ObjThumb); ok { + obj.SetPath(newPath) + obj.Name = newName + obj.Modified = time.Now() + return obj, nil + } + return d.getByPath(ctx, newPath) +} + +func (d *BaiduYouth) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + newPath := stdpath.Join(dstDir.GetPath(), srcObj.GetName()) + _, err := d.manage(ctx, "copy", []base.Json{ + { + "dest": dstDir.GetPath(), + "newname": srcObj.GetName(), + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + if obj, ok := srcObj.(*model.ObjThumb); ok { + copied := *obj + copied.SetPath(newPath) + copied.Modified = time.Now() + return &copied, nil + } + // Youth copy returns success before /api/filemetas can resolve the new path. + // Avoid turning a successful copy into a false failure because the immediate + // post-copy lookup is temporarily unavailable. + return &model.Object{ + ID: newPath, + Path: newPath, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + HashInfo: srcObj.GetHash(), + }, nil +} + +func (d *BaiduYouth) Remove(ctx context.Context, obj model.Obj) error { + _, err := d.manage(ctx, "delete", []string{obj.GetPath()}) + return err +} + +func (d *BaiduYouth) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if stream.GetSize() < 1 { + return nil, ErrBaiduYouthEmptyFilesNotAllowed + } + + var ( + cache = stream.GetFile() + tmpF *os.File + err error + ) + if _, ok := cache.(io.ReaderAt); !ok { + tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + defer func() { + _ = tmpF.Close() + _ = os.Remove(tmpF.Name()) + }() + cache = tmpF + } + + streamSize := stream.GetSize() + sliceSize := DefaultSliceSize + count := int(streamSize / sliceSize) + lastBlockSize := streamSize % sliceSize + if lastBlockSize > 0 { + count++ + } else { + lastBlockSize = sliceSize + } + + const sliceMD5Size int64 = 256 * utils.KB + blockList := make([]string, 0, count) + byteSize := sliceSize + fileMd5H := md5.New() + sliceMd5H := md5.New() + sliceMd5H2 := md5.New() + sliceMd5H2Writer := utils.LimitWriter(sliceMd5H2, sliceMD5Size) + writers := []io.Writer{fileMd5H, sliceMd5H, sliceMd5H2Writer} + if tmpF != nil { + writers = append(writers, tmpF) + } + + written := int64(0) + for i := 1; i <= count; i++ { + if utils.IsCanceled(ctx) { + return nil, ctx.Err() + } + if i == count { + byteSize = lastBlockSize + } + n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) + written += n + if err != nil && err != io.EOF { + return nil, err + } + blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) + sliceMd5H.Reset() + } + + if tmpF != nil { + if written != streamSize { + return nil, errs.NewErr(errs.StreamIncomplete, "temp file size mismatch: %d != %d", written, streamSize) + } + if _, err = tmpF.Seek(0, io.SeekStart); err != nil { + return nil, err + } + } + + contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) + sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) + blockListStr, err := utils.Json.MarshalToString(blockList) + if err != nil { + return nil, err + } + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) + mtime := stream.ModTime().Unix() + ctime := stream.CreateTime().Unix() + + progressKey := d.uploadProgressKey() + precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, progressKey, contentMd5) + if !ok { + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) + if err != nil { + return nil, err + } + } + + if precreateResp.ReturnType >= 2 { + result := precreateResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil + } + + cacheReaderAt, ok := cache.(io.ReaderAt) + if !ok { + return nil, fmt.Errorf("cache object must implement io.ReaderAt") + } + +uploadLoop: + for attempt := 0; attempt < 2; attempt++ { + completed := count - len(precreateResp.BlockList) + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) || partseq < 0 { + continue + } + + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize + } + + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "partseq": strconv.Itoa(partseq), + "path": path, + "type": "tmpfile", + "uploadid": precreateResp.Uploadid, + } + if precreateResp.Uploadsign != "" { + params["uploadsign"] = precreateResp.Uploadsign + } + section := io.NewSectionReader(cacheReaderAt, offset, size) + if err := d.uploadSlice(ctx, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section)); err != nil { + return err + } + precreateResp.BlockList[i] = -1 + success := completed + int(threadG.Success()) + 1 + up(float64(success) * 100 / float64(count)) + return nil + }) + } + + err = threadG.Wait() + if err == nil { + break uploadLoop + } + + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(part int) bool { + return part >= 0 + }) + base.SaveUploadProgress(d, precreateResp, progressKey, contentMd5) + + if errors.Is(err, context.Canceled) { + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) + if err != nil { + return nil, err + } + if precreateResp.ReturnType >= 2 { + result := precreateResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil + } + base.SaveUploadProgress(d, precreateResp, progressKey, contentMd5) + continue uploadLoop + } + return nil, err + } + + var createResp CreateResp + _, err = d.createFile(ctx, path, stdpath.Dir(path), streamSize, precreateResp.Uploadid, precreateResp.Uploadsign, blockListStr, &createResp, mtime, ctime) + if err != nil { + return nil, err + } + + base.SaveUploadProgress(d, nil, progressKey, contentMd5) + result := createResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil +} + +var _ driver.Driver = (*BaiduYouth)(nil) +var _ driver.Getter = (*BaiduYouth)(nil) +var _ driver.MkdirResult = (*BaiduYouth)(nil) +var _ driver.MoveResult = (*BaiduYouth)(nil) +var _ driver.RenameResult = (*BaiduYouth)(nil) +var _ driver.CopyResult = (*BaiduYouth)(nil) +var _ driver.Remove = (*BaiduYouth)(nil) +var _ driver.PutResult = (*BaiduYouth)(nil) diff --git a/drivers/baidu_youth/meta.go b/drivers/baidu_youth/meta.go new file mode 100644 index 00000000000..7d5cb6a8698 --- /dev/null +++ b/drivers/baidu_youth/meta.go @@ -0,0 +1,40 @@ +package baidu_youth + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Cookie string `json:"cookie" required:"true"` + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + ForceProxy bool `json:"force_proxy" type:"bool" default:"true" help:"Proxy downloads through AList. Disable to redirect the browser to a fresh Baidu direct link."` + DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` + UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` +} + +const ( + UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" + UPLOAD_TIMEOUT = time.Minute * 30 + UPLOAD_RETRY_COUNT = 3 + UPLOAD_RETRY_WAIT_TIME = time.Second + UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 + DefaultSliceSize int64 = 4 * 1024 * 1024 +) + +var config = driver.Config{ + Name: "BaiduYouth", + DefaultRoot: "/", + OnlyProxy: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &BaiduYouth{} + }) +} diff --git a/drivers/baidu_youth/types.go b/drivers/baidu_youth/types.go new file mode 100644 index 00000000000..82024009403 --- /dev/null +++ b/drivers/baidu_youth/types.go @@ -0,0 +1,130 @@ +package baidu_youth + +import ( + "errors" + "path" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +var ( + ErrBaiduYouthEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu youth") +) + +type File struct { + Category int `json:"category"` + FsId int64 `json:"fs_id"` + Thumbs struct { + Url3 string `json:"url3"` + } `json:"thumbs"` + Size int64 `json:"size"` + Path string `json:"path"` + ServerFilename string `json:"server_filename"` + Md5 string `json:"md5"` + Isdir int `json:"isdir"` + ServerCtime int64 `json:"server_ctime"` + ServerMtime int64 `json:"server_mtime"` + LocalMtime int64 `json:"local_mtime"` + LocalCtime int64 `json:"local_ctime"` + Ctime int64 `json:"ctime"` + Mtime int64 `json:"mtime"` + Dlink string `json:"dlink"` +} + +func fileToObj(f File) *model.ObjThumb { + if f.ServerFilename == "" { + f.ServerFilename = path.Base(f.Path) + } + if f.ServerCtime == 0 { + if f.LocalCtime != 0 { + f.ServerCtime = f.LocalCtime + } else { + f.ServerCtime = f.Ctime + } + } + if f.ServerMtime == 0 { + if f.LocalMtime != 0 { + f.ServerMtime = f.LocalMtime + } else { + f.ServerMtime = f.Mtime + } + } + return &model.ObjThumb{ + Object: model.Object{ + ID: strconv.FormatInt(f.FsId, 10), + Path: f.Path, + Name: f.ServerFilename, + Size: f.Size, + Modified: time.Unix(f.ServerMtime, 0), + Ctime: time.Unix(f.ServerCtime, 0), + IsFolder: f.Isdir == 1, + HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, + } +} + +type ListResp struct { + Errno int `json:"errno"` + List []File `json:"list"` +} + +type FileMetaResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Info []File `json:"info"` + List []File `json:"list"` +} + +type LocateDownloadResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Path string `json:"path"` + URL string `json:"url"` +} + +type MediaInfoResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Info File `json:"info"` +} + +type CreateResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Info File `json:"info"` + File +} + +func (r CreateResp) ResultFile() File { + if r.Info.Path != "" || r.Info.FsId != 0 { + return r.Info + } + return r.File +} + +type PrecreateResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + ReturnType int `json:"return_type"` + Path string `json:"path"` + Uploadid string `json:"uploadid"` + Uploadsign string `json:"uploadsign"` + BlockList []int `json:"block_list"` + Info File `json:"info"` + File +} + +func (r PrecreateResp) ResultFile() File { + if r.Info.Path != "" || r.Info.FsId != 0 { + return r.Info + } + return r.File +} diff --git a/drivers/baidu_youth/util.go b/drivers/baidu_youth/util.go new file mode 100644 index 00000000000..b83518fe350 --- /dev/null +++ b/drivers/baidu_youth/util.go @@ -0,0 +1,606 @@ +package baidu_youth + +import ( + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/http" + stdpath "path" + "strconv" + "strings" + "time" + "unicode" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + panBaseURL = "https://pan.baidu.com" + panReferer = panBaseURL + "/" + panUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36" + youthReferer = panBaseURL + "/youth/pan/main#/index?category=all" + youthAppID = "250528" + uploadAppID = "25179614" + videoAPIChannel = "android_15_25010PN30C_bd-netdisk_1523a" + videoAPIDevUID = "0%1" +) + +func (d *BaiduYouth) normalizeURL(furl string) string { + if strings.HasPrefix(furl, "http://") || strings.HasPrefix(furl, "https://") { + return furl + } + return panBaseURL + furl +} + +func (d *BaiduYouth) commonHeaders() map[string]string { + return map[string]string{ + "Accept": "application/json, text/plain, */*", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Cache-Control": "no-cache", + "Cookie": d.Cookie, + "Origin": panBaseURL, + "Pragma": "no-cache", + "Referer": youthReferer, + "User-Agent": panUserAgent, + "X-Requested-With": "XMLHttpRequest", + } +} + +func youthQueryParams() map[string]string { + return map[string]string{ + "app_id": youthAppID, + "clienttype": "0", + "web": "1", + } +} + +func youthUploadQueryParams() map[string]string { + return map[string]string{ + "app_id": uploadAppID, + "channel": "chunlei", + "clienttype": "0", + "web": "1", + } +} + +func extractBaiduMessage(body []byte) string { + for _, path := range [][]interface{}{ + {"show_msg"}, + {"errmsg"}, + {"error_msg"}, + {"error_description"}, + {"message"}, + } { + msg := utils.Json.Get(body, path...).ToString() + if msg != "" { + return msg + } + } + return "" +} + +func (d *BaiduYouth) doRequest(furl string, method string, defaultQuery map[string]string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R().SetHeaders(d.commonHeaders()) + if defaultQuery != nil { + req.SetQueryParams(defaultQuery) + } + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, d.normalizeURL(furl)) + if err != nil { + return nil, err + } + body := res.Body() + if errno := utils.Json.Get(body, "errno").ToInt(); errno != 0 { + msg := extractBaiduMessage(body) + if errno == -6 && msg == "" { + msg = "cookie expired or invalid" + } + if msg == "" { + msg = "request failed" + } + return nil, fmt.Errorf("[baidu_youth] %s (errno=%d)", msg, errno) + } + return body, nil +} + +func (d *BaiduYouth) get(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, youthQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) postForm(ctx context.Context, pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodPost, youthQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + req.SetFormData(form) + }, resp) +} + +func (d *BaiduYouth) getUpload(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, youthUploadQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) postUploadForm(ctx context.Context, pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodPost, youthUploadQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + req.SetFormData(form) + }, resp) +} + +func (d *BaiduYouth) getBare(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, nil, func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) getUserSK(ctx context.Context) (string, error) { + body, err := d.get(ctx, "/youth/api/report/user", map[string]string{ + "action": "sapi_auth", + "timestamp": strconv.FormatInt(time.Now().UnixMilli(), 10), + }, nil) + if err != nil { + return "", err + } + return utils.Json.Get(body, "uinfo").ToString(), nil +} + +func (d *BaiduYouth) getUserSession(ctx context.Context) (int64, string, string, error) { + body, err := d.get(ctx, "/youth/api/user/getinfo", map[string]string{ + "need_selfinfo": "1", + }, nil) + if err != nil { + return 0, "", "", err + } + uk := int64(utils.Json.Get(body, "records", 0, "uk").ToInt()) + bdstoken := utils.Json.Get(body, "records", 0, "bdstoken").ToString() + sk := utils.Json.Get(body, "records", 0, "sk").ToString() + if bdstoken == "" || uk == 0 || sk == "" { + body, err = d.getBare(ctx, "/api/gettemplatevariable", map[string]string{ + "fields": `["bdstoken","uk","sk"]`, + }, nil) + if err != nil { + return 0, "", "", err + } + if uk == 0 { + uk = int64(utils.Json.Get(body, "result", "uk").ToInt()) + } + if bdstoken == "" { + bdstoken = utils.Json.Get(body, "result", "bdstoken").ToString() + } + if sk == "" { + sk = utils.Json.Get(body, "result", "sk").ToString() + } + } + if sk == "" { + sk, _ = d.getUserSK(ctx) + } + if bdstoken == "" { + return 0, "", "", fmt.Errorf("failed to get bdstoken from baidu youth cookie") + } + if uk == 0 { + return 0, "", "", fmt.Errorf("failed to get uk from baidu youth cookie") + } + return uk, bdstoken, sk, nil +} + +func (d *BaiduYouth) getFiles(ctx context.Context, dir string) ([]File, error) { + page := 1 + num := 1000 + params := map[string]string{ + "dir": dir, + } + if d.OrderBy != "" { + params["order"] = d.OrderBy + if d.OrderDirection == "desc" { + params["desc"] = "1" + } else { + params["desc"] = "0" + } + } + files := make([]File, 0) + for { + params["page"] = strconv.Itoa(page) + params["num"] = strconv.Itoa(num) + var resp ListResp + _, err := d.get(ctx, "/youth/api/list", params, &resp) + if err != nil { + return nil, err + } + if len(resp.List) == 0 { + return files, nil + } + files = append(files, resp.List...) + if len(resp.List) < num { + return files, nil + } + page++ + } +} + +func (d *BaiduYouth) getFileByPath(ctx context.Context, path string) (File, error) { + if path == "/" { + return File{ + Path: "/", + ServerFilename: "/", + Isdir: 1, + }, nil + } + target, err := utils.Json.MarshalToString([]string{path}) + if err != nil { + return File{}, err + } + var resp FileMetaResp + _, err = d.get(ctx, "/api/filemetas", map[string]string{ + "target": target, + }, &resp) + if err != nil { + return File{}, err + } + if len(resp.Info) > 0 { + return resp.Info[0], nil + } + if len(resp.List) > 0 { + return resp.List[0], nil + } + return File{}, errs.NewErr(errs.ObjectNotFound, "baidu youth path not found: %s", path) +} + +func (d *BaiduYouth) getByPath(ctx context.Context, path string) (model.Obj, error) { + if path == "/" { + return &model.Object{ + Path: "/", + Name: "/", + IsFolder: true, + }, nil + } + file, err := d.getFileByPath(ctx, path) + if err != nil { + return nil, err + } + return fileToObj(file), nil +} + +func (d *BaiduYouth) linkOfficial(ctx context.Context, file model.Obj) (*model.Link, error) { + return d.buildDownloadLink(ctx, file) +} + +func (d *BaiduYouth) linkCrack(ctx context.Context, file model.Obj) (*model.Link, error) { + return d.buildDownloadLink(ctx, file) +} + +func (d *BaiduYouth) downloadHeaders() http.Header { + return http.Header{ + "Accept": []string{"*/*"}, + "Accept-Language": []string{"zh-CN,zh;q=0.9,en;q=0.8"}, + "Cache-Control": []string{"no-cache"}, + "Pragma": []string{"no-cache"}, + "Referer": []string{panReferer}, + "User-Agent": []string{panUserAgent}, + } +} + +func nextDPLogID() string { + return strconv.FormatInt(time.Now().UnixNano(), 10) +} + +func (d *BaiduYouth) getCurrentUserSK(ctx context.Context) (string, error) { + sk, err := d.getUserSK(ctx) + if err == nil && sk != "" { + return sk, nil + } + if d.sk != "" { + return d.sk, nil + } + if err != nil { + return "", err + } + return "", fmt.Errorf("baidu youth sk is empty") +} + +func (d *BaiduYouth) locatedownloadRand(sk string, nowMilli int64) string { + sum := sha1.Sum([]byte(strconv.FormatInt(d.uk, 10) + sk + strconv.FormatInt(nowMilli, 10) + "0")) + return hex.EncodeToString(sum[:]) +} + +func (d *BaiduYouth) locatedownloadSign(fileMD5 string, fileID string, nowMilli int64) string { + sum := md5.Sum([]byte(fileMD5 + "_" + strconv.FormatInt(d.uk, 10) + "_" + fileID + "_" + strconv.FormatInt(nowMilli, 10))) + return hex.EncodeToString(sum[:]) +} + +func (d *BaiduYouth) resolveDownloadMeta(ctx context.Context, file model.Obj) (string, string, string, error) { + parentDir := stdpath.Dir(file.GetPath()) + if parentDir == "." { + parentDir = "/" + } + + files, err := d.getFiles(ctx, parentDir) + if err != nil { + return "", "", "", err + } + for _, listedFile := range files { + if listedFile.Path != file.GetPath() { + continue + } + fileID := strconv.FormatInt(listedFile.FsId, 10) + if listedFile.Path != "" && fileID != "" && listedFile.Md5 != "" { + return listedFile.Path, fileID, listedFile.Md5, nil + } + return "", "", "", fmt.Errorf("baidu youth list metadata incomplete for %s", file.GetPath()) + } + return "", "", "", errs.NewErr(errs.ObjectNotFound, "baidu youth list metadata not found: %s", file.GetPath()) +} + +func normalizeLocatedownloadURL(rawURL string) (string, error) { + if rawURL == "" { + return "", fmt.Errorf("baidu youth locatedownload url is empty") + } + if !strings.Contains(rawURL, "response-cache-control=") { + sep := "&" + if !strings.Contains(rawURL, "?") { + sep = "?" + } + rawURL += sep + "response-cache-control=private" + } + return rawURL, nil +} + +func (d *BaiduYouth) getMediaInfoDLink(ctx context.Context, file model.Obj) (string, error) { + path, fileID, _, err := d.resolveDownloadMeta(ctx, file) + if err != nil { + return "", err + } + + var resp MediaInfoResp + _, err = d.doRequest("/youth/api/mediainfo", http.MethodGet, nil, func(req *resty.Request) { + req.SetContext(ctx) + req.SetQueryParams(map[string]string{ + "channel": videoAPIChannel, + "clienttype": "1", + "devuid": videoAPIDevUID, + "dlink": "1", + "fs_id": fileID, + "media": "1", + "nom3u8": "1", + "origin": "dlna", + "path": path, + "type": "VideoURL", + }) + }, &resp) + if err != nil { + return "", err + } + if resp.Info.Dlink == "" { + return "", fmt.Errorf("baidu youth video dlink not found for %s", path) + } + return resp.Info.Dlink, nil +} + +func (d *BaiduYouth) buildVideoLink(ctx context.Context, file model.Obj) (*model.Link, error) { + dlink, err := d.getMediaInfoDLink(ctx, file) + if err != nil { + return nil, err + } + return &model.Link{ + URL: dlink, + Header: http.Header{ + "Referer": []string{panReferer}, + }, + }, nil +} + +func (d *BaiduYouth) requestLocateDownloadURL(ctx context.Context, path string, fileID string, fileMD5 string, sk string) (string, error) { + nowMilli := time.Now().UnixMilli() + + var resp LocateDownloadResp + _, err := d.get(ctx, "/youth/api/locatedownload", map[string]string{ + "devuid": "0", + "dp-logid": nextDPLogID(), + "path": path, + "rand": d.locatedownloadRand(sk, nowMilli), + "sign": d.locatedownloadSign(fileMD5, fileID, nowMilli), + "time": strconv.FormatInt(nowMilli, 10), + }, &resp) + if err != nil { + return "", err + } + if resp.URL == "" { + return "", fmt.Errorf("baidu youth locatedownload url not found for %s", path) + } + return normalizeLocatedownloadURL(resp.URL) +} + +func (d *BaiduYouth) getLocateDownloadURL(ctx context.Context, file model.Obj) (string, error) { + path, fileID, fileMD5, err := d.resolveDownloadMeta(ctx, file) + if err != nil { + return "", err + } + + sk, err := d.getCurrentUserSK(ctx) + if err != nil { + return "", err + } + downloadURL, err := d.requestLocateDownloadURL(ctx, path, fileID, fileMD5, sk) + if err == nil { + return downloadURL, nil + } + + if !strings.Contains(err.Error(), "errno=-30006") { + return "", err + } + sk, refreshErr := d.getUserSK(ctx) + if refreshErr != nil { + return "", err + } + if sk == "" { + return "", err + } + return d.requestLocateDownloadURL(ctx, path, fileID, fileMD5, sk) +} + +func (d *BaiduYouth) buildDownloadLink(ctx context.Context, file model.Obj) (*model.Link, error) { + downloadURL, err := d.getLocateDownloadURL(ctx, file) + if err != nil { + return nil, err + } + return &model.Link{ + URL: downloadURL, + Header: d.downloadHeaders(), + }, nil +} + +func (d *BaiduYouth) manage(ctx context.Context, opera string, filelist any) ([]byte, error) { + marshal, err := utils.Json.MarshalToString(filelist) + if err != nil { + return nil, err + } + return d.postForm(ctx, "/youth/api/filemanager", map[string]string{ + "async": "0", + "bdstoken": d.bdstoken, + "onnest": "fail", + "opera": opera, + }, map[string]string{ + "filelist": marshal, + "ondup": "fail", + }, nil) +} + +func (d *BaiduYouth) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + form := map[string]string{ + "autoinit": "1", + "block_list": blockListStr, + "isdir": "0", + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "target_path": stdpath.Dir(path), + } + if contentMd5 != "" { + form["content-md5"] = contentMd5 + } + if sliceMd5 != "" { + form["slice-md5"] = sliceMd5 + } + joinTime(form, ctime, mtime) + var resp PrecreateResp + _, err := d.postUploadForm(ctx, "/youth/api/precreate", map[string]string{ + "bdstoken": d.bdstoken, + }, form, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *BaiduYouth) createFile(ctx context.Context, path, targetPath string, size int64, uploadID, uploadSign, blockList string, resp interface{}, mtime, ctime int64) ([]byte, error) { + form := map[string]string{ + "block_list": blockList, + "path": path, + "size": strconv.FormatInt(size, 10), + "target_path": targetPath, + "uploadid": uploadID, + } + if uploadSign != "" { + form["uploadsign"] = uploadSign + } + joinTime(form, ctime, mtime) + return d.postUploadForm(ctx, "/youth/api/create", map[string]string{ + "bdstoken": d.bdstoken, + "isdir": "0", + }, form, resp) +} + +func joinTime(form map[string]string, ctime, mtime int64) { + if ctime != 0 { + form["local_ctime"] = strconv.FormatInt(ctime, 10) + } + if mtime != 0 { + form["local_mtime"] = strconv.FormatInt(mtime, 10) + } +} + +func (d *BaiduYouth) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { + res, err := d.upClient.R(). + SetContext(ctx). + SetHeaders(d.commonHeaders()). + SetQueryParams(youthUploadQueryParams()). + SetQueryParams(params). + SetFileReader("file", fileName, file). + Post(d.UploadAPI + "/rest/2.0/pcs/superfile2") + if err != nil { + return err + } + body := res.Body() + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() + lower := strings.ToLower(string(body)) + if strings.Contains(lower, "uploadid") && (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + if errCode != 0 || errNo != 0 { + msg := extractBaiduMessage(body) + if msg == "" { + msg = "error uploading to baidu youth" + } + return errs.NewErr(errs.StreamIncomplete, "%s: %s", msg, string(body)) + } + return nil +} + +func (d *BaiduYouth) uploadProgressKey() string { + if d.uk != 0 { + return strconv.FormatInt(d.uk, 10) + } + sum := md5.Sum([]byte(d.Cookie)) + return hex.EncodeToString(sum[:]) +} + +func DecryptMd5(encryptMd5 string) string { + if encryptMd5 == "" { + return "" + } + if _, err := hex.DecodeString(encryptMd5); err == nil { + return encryptMd5 + } + + var out strings.Builder + out.Grow(len(encryptMd5)) + for i, n := 0, int64(0); i < len(encryptMd5); i++ { + if i == 9 { + n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') + } else { + n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) + } + out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) + } + + encryptMd5 = out.String() + return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] +} diff --git a/internal/driver/proxy.go b/internal/driver/proxy.go new file mode 100644 index 00000000000..f94273c9f92 --- /dev/null +++ b/internal/driver/proxy.go @@ -0,0 +1,7 @@ +package driver + +// ProxyDriver lets a driver override the default "must proxy" download behavior +// on a per-storage basis. +type ProxyDriver interface { + ShouldProxyDownloads() bool +} diff --git a/server/common/check.go b/server/common/check.go index 34aaa41d93a..010d3131745 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -41,7 +41,11 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri // 2. storage.WebProxy // 3. proxy_types func ShouldProxy(storage driver.Driver, filename string) bool { - if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + if proxyDriver, ok := storage.(driver.ProxyDriver); ok { + if proxyDriver.ShouldProxyDownloads() || storage.GetStorage().WebProxy { + return true + } + } else if storage.Config().MustProxy() || storage.GetStorage().WebProxy { return true } if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { diff --git a/server/handles/fsread.go b/server/handles/fsread.go index c370f631fc0..67f84e5d70a 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -377,12 +377,21 @@ func FsGet(c *gin.Context) { common.ErrorResp(c, err, 500) return } + query := "" + if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { + query = "?sign=" + sign.Sign(reqPath) + } + forceRedirectRawURL := storage.GetStorage().Driver == "BaiduYouth" forceProxyRawURL := storage.GetStorage().Driver == "Quark" && utils.GetFileType(obj.GetName()) == conf.VIDEO - if storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL { - query := "" - if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { - query = "?sign=" + sign.Sign(reqPath) - } + if forceRedirectRawURL { + // Baidu Youth direct links are minted per request and are not stable enough + // to expose as fs/get raw_url. Return the local /d endpoint so the frontend + // obtains a fresh link on each download click. + rawURL = fmt.Sprintf("%s/d%s%s", + common.GetApiUrl(c.Request), + utils.EncodePath(reqPath, true), + query) + } else if storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL { if storage.GetStorage().DownProxyUrl != "" { rawURL = common.BuildDownProxyURL( storage.GetStorage().DownProxyUrl, From 094765ebcf2e9d55637db0ce5c75826eb27b1431 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 26 Mar 2026 10:59:50 +0800 Subject: [PATCH 092/133] feat(chunker): add storage driver --- drivers/all.go | 1 + drivers/chunker/driver.go | 381 +++++++++++++++++++++++ drivers/chunker/meta.go | 43 +++ drivers/chunker/types.go | 72 +++++ drivers/chunker/util.go | 572 +++++++++++++++++++++++++++++++++++ drivers/chunker/util_test.go | 115 +++++++ drivers/crypt/meta.go | 6 +- 7 files changed, 1187 insertions(+), 3 deletions(-) create mode 100644 drivers/chunker/driver.go create mode 100644 drivers/chunker/meta.go create mode 100644 drivers/chunker/types.go create mode 100644 drivers/chunker/util.go create mode 100644 drivers/chunker/util_test.go diff --git a/drivers/all.go b/drivers/all.go index a4fce9d0cac..7f6597561c1 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -23,6 +23,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_share" _ "github.com/alist-org/alist/v3/drivers/bitqiu" _ "github.com/alist-org/alist/v3/drivers/chaoxing" + _ "github.com/alist-org/alist/v3/drivers/chunker" _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" _ "github.com/alist-org/alist/v3/drivers/crypt" diff --git a/drivers/chunker/driver.go b/drivers/chunker/driver.go new file mode 100644 index 00000000000..004fde1c476 --- /dev/null +++ b/drivers/chunker/driver.go @@ -0,0 +1,381 @@ +package chunker + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "hash" + "io" + "path" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Chunker) Config() driver.Config { + return config +} + +func (d *Chunker) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Chunker) Init(ctx context.Context) error { + if d.ChunkSize == 0 { + d.ChunkSize = defaultChunkSize + } + if d.StartFrom == 0 { + d.StartFrom = defaultStartFrom + } + d.NameFormat = utils.GetNoneEmpty(d.NameFormat, defaultChunkNameFmt) + d.MetaFormat = utils.GetNoneEmpty(d.MetaFormat, defaultMetaFormat) + d.HashType = utils.GetNoneEmpty(d.HashType, defaultHashType) + + if err := d.setChunkNameFormat(d.NameFormat); err != nil { + return fmt.Errorf("invalid name_format: %w", err) + } + if err := d.validateOptions(); err != nil { + return err + } + + storage, err := fs.GetStorage(d.RemotePath, &fs.GetStoragesArgs{}) + if err != nil { + return fmt.Errorf("can't find remote storage: %w", err) + } + d.remoteStorage = storage + return nil +} + +func (d *Chunker) Drop(ctx context.Context) error { + d.remoteStorage = nil + return nil +} + +func (d *Chunker) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.listDirObjects(ctx, dir.GetPath(), args.Refresh) +} + +func (d *Chunker) Get(ctx context.Context, pathStr string) (model.Obj, error) { + if utils.PathEqual(pathStr, "/") { + return &model.Object{ + Name: "Root", + Path: "/", + IsFolder: true, + }, nil + } + parent, name := path.Split(utils.FixAndCleanPath(pathStr)) + if parent == "" { + parent = "/" + } + objs, err := d.listDirObjects(ctx, parent, false) + if err != nil { + return nil, err + } + for _, obj := range objs { + if obj.GetName() == name { + return obj, nil + } + } + return nil, errs.ObjectNotFound +} + +func (d *Chunker) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj := d.linkedObject(file) + if obj == nil || !obj.Chunked { + actualPath, err := d.getActualPathForRemote(file.GetPath()) + if err != nil { + return nil, fmt.Errorf("failed to convert path to remote path: %w", err) + } + link, _, err := op.Link(ctx, d.remoteStorage, actualPath, args) + return link, err + } + + linkedParts := make([]linkedPart, 0, len(obj.Parts)) + baseClosers := utils.EmptyClosers() + for _, part := range obj.Parts { + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID) + if err != nil { + return nil, fmt.Errorf("failed to convert chunk path: %w", err) + } + link, _, err := op.Link(ctx, d.remoteStorage, actualPath, args) + if err != nil { + return nil, err + } + if link.MFile != nil { + baseClosers.Add(link.MFile) + } + if link.RangeReadCloser != nil { + baseClosers.Add(link.RangeReadCloser) + } + linkedParts = append(linkedParts, linkedPart{ + part: part, + link: link, + }) + } + + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{ + RangeReader: func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return d.openChunkReader(ctx, linkedParts, obj.GetSize(), httpRange) + }, + Closers: baseClosers, + }, + }, nil +} + +func (d *Chunker) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + dstDirActualPath, err := d.getActualPathForRemote(parentDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.MakeDir(ctx, d.remoteStorage, path.Join(dstDirActualPath, dirName)) +} + +func (d *Chunker) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + obj := d.linkedObject(srcObj) + if srcObj.IsDir() || obj == nil || !obj.Chunked { + srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Move(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + } + + dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + for _, logicalPath := range d.chunkPathsForObject(obj) { + actualPath, err := d.getActualPathForRemote(logicalPath) + if err != nil { + return err + } + if err := op.Move(ctx, d.remoteStorage, actualPath, dstRemoteActualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + obj := d.linkedObject(srcObj) + if srcObj.IsDir() || obj == nil || !obj.Chunked { + remoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Rename(ctx, d.remoteStorage, remoteActualPath, newName) + } + + for _, part := range obj.Parts { + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID) + if err != nil { + return err + } + newChunkName := d.chunkPartBaseName(path.Join(path.Dir(obj.GetPath()), newName), part.No, part.XactID) + if err := op.Rename(ctx, d.remoteStorage, actualPath, newChunkName); err != nil { + return err + } + } + if obj.UsesMeta { + actualPath, err := d.getActualPathForRemote(obj.GetPath()) + if err != nil { + return err + } + if err := op.Rename(ctx, d.remoteStorage, actualPath, newName); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + obj := d.linkedObject(srcObj) + if srcObj.IsDir() || obj == nil || !obj.Chunked { + srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Copy(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + } + + dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + for _, logicalPath := range d.chunkPathsForObject(obj) { + actualPath, err := d.getActualPathForRemote(logicalPath) + if err != nil { + return err + } + if err := op.Copy(ctx, d.remoteStorage, actualPath, dstRemoteActualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Remove(ctx context.Context, obj model.Obj) error { + chunkedObj := d.linkedObject(obj) + if obj.IsDir() || chunkedObj == nil || !chunkedObj.Chunked { + remoteActualPath, err := d.getActualPathForRemote(obj.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Remove(ctx, d.remoteStorage, remoteActualPath) + } + + for _, logicalPath := range d.chunkPathsForObject(chunkedObj) { + actualPath, err := d.getActualPathForRemote(logicalPath) + if err != nil { + return err + } + if err := op.Remove(ctx, d.remoteStorage, actualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.FileStreamer, up driver.UpdateProgress) error { + dstDirActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + + existing := d.linkedObject(streamer.GetExist()) + logicalPath := path.Join(dstDir.GetPath(), streamer.GetName()) + if streamer.GetSize() <= d.ChunkSize { + if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, streamer, up, false); err != nil { + return err + } + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(logicalPath)) + } + + if up == nil { + up = func(float64) {} + } + + var ( + md5Hasher hash.Hash + sha1Hasher hash.Hash + writers []io.Writer + ) + switch d.HashType { + case "md5": + md5Hasher = md5.New() + writers = append(writers, md5Hasher) + case "sha1": + sha1Hasher = sha1.New() + writers = append(writers, sha1Hasher) + } + writers = append(writers, driver.NewProgress(streamer.GetSize(), up)) + + baseReader := io.TeeReader(streamer, io.MultiWriter(writers...)) + xactID := strconv.FormatInt(time.Now().UnixNano(), 36) + if len(xactID) > 9 { + xactID = xactID[len(xactID)-9:] + } + if len(xactID) < 4 { + xactID = fmt.Sprintf("%04s", xactID) + } + + chunkCount := 0 + remaining := streamer.GetSize() + keepPaths := []string{logicalPath} + for remaining > 0 { + chunkLen := utils.Min(remaining, d.ChunkSize) + chunkName := d.chunkPartBaseName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) + chunkPath := d.makeChunkName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) + partReader := driver.NewLimitedUploadStream(ctx, &driver.ReaderWithCtx{ + Reader: io.LimitReader(baseReader, chunkLen), + Ctx: ctx, + }) + partStream := &stream.FileStream{ + Obj: &model.Object{ + Name: chunkName, + Size: chunkLen, + Modified: streamer.ModTime(), + Ctime: streamer.CreateTime(), + IsFolder: false, + }, + Reader: partReader, + Mimetype: "application/octet-stream", + WebPutAsTask: streamer.NeedStore(), + ForceStreamUpload: true, + } + if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, partStream, nil, false); err != nil { + return err + } + keepPaths = append(keepPaths, chunkPath) + remaining -= chunkLen + chunkCount++ + } + + if d.MetaFormat == "simplejson" { + md5Value := "" + if md5Hasher != nil { + md5Value = hex.EncodeToString(md5Hasher.Sum(nil)) + } + sha1Value := "" + if sha1Hasher != nil { + sha1Value = hex.EncodeToString(sha1Hasher.Sum(nil)) + } + txn := xactID + metaData, err := marshalMetadata(streamer.GetSize(), chunkCount, md5Value, sha1Value, txn) + if err != nil { + return err + } + metaStream := &stream.FileStream{ + Obj: &model.Object{ + Name: streamer.GetName(), + Size: int64(len(metaData)), + Modified: streamer.ModTime(), + Ctime: streamer.CreateTime(), + IsFolder: false, + }, + Reader: bytes.NewReader(metaData), + Mimetype: "application/json", + WebPutAsTask: false, + ForceStreamUpload: true, + } + if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, metaStream, nil, false); err != nil { + return err + } + } else { + actualPath, err := d.getActualPathForRemote(logicalPath) + if err == nil { + _ = op.Remove(ctx, d.remoteStorage, actualPath) + } + } + + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(keepPaths...)) +} + +func xactIDIfNeeded(metaFormat, xactID string) string { + if metaFormat == "simplejson" { + return xactID + } + return "" +} + +var _ driver.Driver = (*Chunker)(nil) diff --git a/drivers/chunker/meta.go b/drivers/chunker/meta.go new file mode 100644 index 00000000000..fabfbc67545 --- /dev/null +++ b/drivers/chunker/meta.go @@ -0,0 +1,43 @@ +package chunker + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +const ( + defaultChunkSize int64 = 2147483648 + defaultChunkNameFmt = "*.rclone_chunk.###" + defaultMetaFormat = "simplejson" + defaultHashType = "md5" + defaultStartFrom = 1 +) + +type Addition struct { + RemotePath string `json:"remote_path" required:"true" help:"AList mounted folder path used to store chunked data, e.g. /my-storage/chunks"` + ChunkSize int64 `json:"chunk_size" type:"number" required:"true" default:"2147483648" help:"Files larger than this will be split into chunks"` + NameFormat string `json:"name_format" required:"true" default:"*.rclone_chunk.###" help:"Compatible with rclone chunker naming"` + StartFrom int `json:"start_from" type:"number" required:"true" default:"1" help:"Chunk number base, usually 0 or 1"` + MetaFormat string `json:"meta_format" type:"select" required:"true" options:"simplejson,none" default:"simplejson" help:"simplejson is compatible with rclone chunker metadata"` + HashType string `json:"hash_type" type:"select" required:"true" options:"none,md5,sha1" default:"md5" help:"Hash stored in metadata for chunked files"` +} + +var config = driver.Config{ + Name: "Chunker", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: true, + NoCache: true, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Chunker{} + }) +} diff --git a/drivers/chunker/types.go b/drivers/chunker/types.go new file mode 100644 index 00000000000..e89fa45c5cd --- /dev/null +++ b/drivers/chunker/types.go @@ -0,0 +1,72 @@ +package chunker + +import ( + "regexp" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" +) + +const ( + ctrlTypeRegStr = `[a-z][a-z0-9]{2,6}` + tempSuffixFormat = `_%04s` + tempSuffixRegStr = `_([0-9a-z]{4,9})` + tempSuffixRegOld = `\.\.tmp_([0-9]{10,13})` + maxMetadataSizeRead = 1023 + maxMetadataSizeWrite = 255 + maxSafeChunkNumber = 10000000 + chunkerMetadataVerion = 2 +) + +var ctrlTypeRegexp = regexp.MustCompile(`^` + ctrlTypeRegStr + `$`) + +type Chunker struct { + model.Storage + Addition + remoteStorage driver.Driver + dataNameFmt string + nameRegexp *regexp.Regexp +} + +type metadataJSON struct { + Version *int `json:"ver"` + Size *int64 `json:"size"` + ChunkNum *int `json:"nchunks"` + MD5 string `json:"md5,omitempty"` + SHA1 string `json:"sha1,omitempty"` + XactID string `json:"txn,omitempty"` +} + +type chunkMetadata struct { + Version int + Size int64 + NChunks int + MD5 string + SHA1 string + XactID string +} + +type chunkPart struct { + No int + Size int64 + XactID string +} + +type groupInfo struct { + base model.Obj + partsByXact map[string]map[int]chunkPart +} + +type Object struct { + model.Object + Main model.Obj + Parts []chunkPart + Meta *chunkMetadata + Chunked bool + UsesMeta bool +} + +type linkedPart struct { + part chunkPart + link *model.Link +} diff --git a/drivers/chunker/util.go b/drivers/chunker/util.go new file mode 100644 index 00000000000..5711b2126f9 --- /dev/null +++ b/drivers/chunker/util.go @@ -0,0 +1,572 @@ +package chunker + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Chunker) validateOptions() error { + if d.RemotePath == "" { + return errors.New("remote_path is required") + } + if d.ChunkSize <= 0 { + return errors.New("chunk_size must be positive") + } + switch d.MetaFormat { + case "simplejson", "none": + default: + return fmt.Errorf("unsupported meta_format: %s", d.MetaFormat) + } + switch d.HashType { + case "none", "md5", "sha1": + default: + return fmt.Errorf("unsupported hash_type: %s", d.HashType) + } + if d.MetaFormat == "none" && d.HashType != "none" { + return fmt.Errorf("hash_type %q requires meta_format=simplejson", d.HashType) + } + return nil +} + +func (d *Chunker) setChunkNameFormat(pattern string) error { + if strings.Count(pattern, "*") != 1 { + return errors.New("pattern must have exactly one asterisk (*)") + } + hashCount := strings.Count(pattern, "#") + if hashCount < 1 { + return errors.New("pattern must contain a hash character (#)") + } + if strings.Index(pattern, "*") > strings.Index(pattern, "#") { + return errors.New("asterisk (*) in pattern must come before hashes (#)") + } + if ok, _ := regexp.MatchString("^[^#]*[#]+[^#]*$", pattern); !ok { + return errors.New("hashes (#) in pattern must be consecutive") + } + if dir, _ := path.Split(pattern); dir != "" { + return errors.New("directory separator prohibited") + } + if pattern[0] != '*' { + return errors.New("pattern must start with asterisk") + } + + reHashes := regexp.MustCompile("[#]+") + reDigits := "[0-9]+" + if hashCount > 1 { + reDigits = fmt.Sprintf("[0-9]{%d,}", hashCount) + } + reDataOrCtrl := fmt.Sprintf("(?:(%s)|_(%s))", reDigits, ctrlTypeRegStr) + + strRegex := regexp.QuoteMeta(pattern) + strRegex = reHashes.ReplaceAllLiteralString(strRegex, reDataOrCtrl) + strRegex = strings.Replace(strRegex, "\\*", "(.+?)", 1) + strRegex = fmt.Sprintf("^%s(?:%s|%s)?$", strRegex, tempSuffixRegStr, tempSuffixRegOld) + d.nameRegexp = regexp.MustCompile(strRegex) + + fmtDigits := "%d" + if hashCount > 1 { + fmtDigits = fmt.Sprintf("%%0%dd", hashCount) + } + strFmt := strings.ReplaceAll(pattern, "%", "%%") + strFmt = strings.Replace(strFmt, "*", "%s", 1) + d.dataNameFmt = reHashes.ReplaceAllLiteralString(strFmt, fmtDigits) + return nil +} + +func (d *Chunker) makeChunkName(filePath string, chunkNo int, xactID string) string { + dir, baseName := path.Split(filePath) + name := fmt.Sprintf(d.dataNameFmt, baseName, chunkNo+d.StartFrom) + if xactID != "" { + name += fmt.Sprintf(tempSuffixFormat, xactID) + } + return dir + name +} + +func (d *Chunker) parseChunkName(filePath string) (parentPath string, chunkNo int, ctrlType, xactID string) { + dir, name := path.Split(filePath) + match := d.nameRegexp.FindStringSubmatch(name) + if match == nil || match[1] == "" { + return "", -1, "", "" + } + + chunkNo = -1 + if match[2] != "" { + n, err := strconv.Atoi(match[2]) + if err != nil { + return "", -1, "", "" + } + chunkNo = n - d.StartFrom + if chunkNo < 0 { + return "", -1, "", "" + } + } + + if match[4] != "" { + xactID = match[4] + } + if match[5] != "" { + oldNum, err := strconv.ParseInt(match[5], 10, 64) + if err != nil || oldNum < 0 { + return "", -1, "", "" + } + xactID = fmt.Sprintf(tempSuffixFormat, strconv.FormatInt(oldNum, 36))[1:] + } + + return dir + match[1], chunkNo, match[3], xactID +} + +func marshalMetadata(size int64, nChunks int, md5Value, sha1Value, xactID string) ([]byte, error) { + version := chunkerMetadataVerion + if xactID == "" && version == 2 { + version = 1 + } + meta := metadataJSON{ + Version: &version, + Size: &size, + ChunkNum: &nChunks, + MD5: md5Value, + SHA1: sha1Value, + XactID: xactID, + } + data, err := json.Marshal(&meta) + if err == nil && len(data) >= maxMetadataSizeWrite { + return nil, errors.New("metadata can't be this big") + } + return data, err +} + +func unmarshalMetadata(data []byte) (*chunkMetadata, error) { + if len(data) > maxMetadataSizeWrite { + return nil, errors.New("metadata is too large") + } + if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' { + return nil, errors.New("invalid json") + } + var meta metadataJSON + if err := json.Unmarshal(data, &meta); err != nil { + return nil, err + } + if meta.Version == nil || meta.Size == nil || meta.ChunkNum == nil { + return nil, errors.New("missing required field") + } + if *meta.Version < 1 { + return nil, errors.New("wrong version") + } + if *meta.Size < 0 { + return nil, errors.New("negative file size") + } + if *meta.ChunkNum < 1 || *meta.ChunkNum > maxSafeChunkNumber { + return nil, errors.New("wrong number of chunks") + } + if meta.MD5 != "" { + if _, err := hex.DecodeString(meta.MD5); err != nil || len(meta.MD5) != 32 { + return nil, errors.New("wrong md5 hash") + } + } + if meta.SHA1 != "" { + if _, err := hex.DecodeString(meta.SHA1); err != nil || len(meta.SHA1) != 40 { + return nil, errors.New("wrong sha1 hash") + } + } + if *meta.Version > chunkerMetadataVerion { + return nil, errors.New("unknown metadata version") + } + return &chunkMetadata{ + Version: *meta.Version, + Size: *meta.Size, + NChunks: *meta.ChunkNum, + MD5: meta.MD5, + SHA1: meta.SHA1, + XactID: meta.XactID, + }, nil +} + +func (d *Chunker) joinRemotePath(logicalPath string) string { + logicalPath = utils.FixAndCleanPath(logicalPath) + if utils.PathEqual(logicalPath, "/") { + return d.RemotePath + } + return path.Join(d.RemotePath, logicalPath) +} + +func (d *Chunker) getActualPathForRemote(logicalPath string) (string, error) { + _, actualPath, err := op.GetStorageAndActualPath(d.joinRemotePath(logicalPath)) + return actualPath, err +} + +func (d *Chunker) getActualChunkPath(filePath string, chunkNo int, xactID string) (string, error) { + return d.getActualPathForRemote(d.makeChunkName(filePath, chunkNo, xactID)) +} + +func (d *Chunker) listDirObjects(ctx context.Context, dirPath string, refresh bool) ([]model.Obj, error) { + remotePath := d.joinRemotePath(dirPath) + entries, err := fsList(ctx, remotePath, refresh) + if err != nil { + return nil, err + } + + groups := map[string]*groupInfo{} + var dirs []model.Obj + + for _, entry := range entries { + if entry.IsDir() { + dirs = append(dirs, &model.Object{ + Name: entry.GetName(), + Path: path.Join(dirPath, entry.GetName()), + Size: 0, + Modified: entry.ModTime(), + Ctime: entry.CreateTime(), + IsFolder: true, + HashInfo: entry.GetHash(), + }) + continue + } + + mainName, chunkNo, ctrlType, xactID := d.parseChunkName(entry.GetName()) + if mainName == "" { + g := groups[entry.GetName()] + if g == nil { + g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} + groups[entry.GetName()] = g + } + g.base = entry + continue + } + if chunkNo < 0 || ctrlType != "" { + continue + } + g := groups[mainName] + if g == nil { + g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} + groups[mainName] = g + } + if g.partsByXact[xactID] == nil { + g.partsByXact[xactID] = map[int]chunkPart{} + } + g.partsByXact[xactID][chunkNo] = chunkPart{ + No: chunkNo, + Size: entry.GetSize(), + XactID: xactID, + } + } + + result := make([]model.Obj, 0, len(dirs)+len(groups)) + result = append(result, dirs...) + for name, group := range groups { + obj, ok, err := d.buildListedObject(ctx, dirPath, name, group) + if err != nil { + return nil, err + } + if ok { + result = append(result, obj) + } + } + return result, nil +} + +func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, group *groupInfo) (model.Obj, bool, error) { + var meta *chunkMetadata + var err error + if group.base != nil && group.base.GetSize() <= maxMetadataSizeRead && len(group.partsByXact) > 0 { + meta, err = d.readMetadata(ctx, path.Join(dirPath, name), group.base.GetSize()) + if err != nil { + meta = nil + } + } + + if meta == nil && group.base != nil && len(group.partsByXact) == 0 { + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: group.base.GetSize(), + Modified: group.base.ModTime(), + Ctime: group.base.CreateTime(), + IsFolder: false, + HashInfo: group.base.GetHash(), + }, + Main: group.base, + }, true, nil + } + + selected := map[int]chunkPart{} + switch { + case meta != nil: + selected = group.partsByXact[meta.XactID] + case group.base == nil: + selected = group.partsByXact[""] + default: + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: group.base.GetSize(), + Modified: group.base.ModTime(), + Ctime: group.base.CreateTime(), + IsFolder: false, + HashInfo: group.base.GetHash(), + }, + Main: group.base, + }, true, nil + } + + parts := sortChunkParts(selected) + if len(parts) == 0 { + if meta != nil { + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: meta.Size, + Modified: group.base.ModTime(), + Ctime: group.base.CreateTime(), + IsFolder: false, + HashInfo: buildHashInfo(meta), + }, + Main: group.base, + Meta: meta, + Chunked: true, + UsesMeta: true, + }, true, nil + } + return nil, false, nil + } + + size := int64(0) + for _, part := range parts { + size += part.Size + } + modified := time.Time{} + ctime := time.Time{} + if group.base != nil { + modified = group.base.ModTime() + ctime = group.base.CreateTime() + } + if meta != nil { + size = meta.Size + } + + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: size, + Modified: modified, + Ctime: ctime, + IsFolder: false, + HashInfo: buildHashInfo(meta), + }, + Main: group.base, + Parts: parts, + Meta: meta, + Chunked: true, + UsesMeta: meta != nil, + }, true, nil +} + +func (d *Chunker) readMetadata(ctx context.Context, logicalPath string, size int64) (*chunkMetadata, error) { + actualPath, err := d.getActualPathForRemote(logicalPath) + if err != nil { + return nil, err + } + link, obj, err := op.Link(ctx, d.remoteStorage, actualPath, model.LinkArgs{}) + if err != nil { + return nil, err + } + ss, err := stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: obj}, link) + if err != nil { + return nil, err + } + defer ss.Close() + + reader, err := ss.RangeRead(http_range.Range{Start: 0, Length: size}) + if err != nil { + return nil, err + } + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } + data, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return unmarshalMetadata(data) +} + +func sortChunkParts(parts map[int]chunkPart) []chunkPart { + result := make([]chunkPart, 0, len(parts)) + for _, part := range parts { + result = append(result, part) + } + sort.Slice(result, func(i, j int) bool { + return result[i].No < result[j].No + }) + return result +} + +func buildHashInfo(meta *chunkMetadata) utils.HashInfo { + if meta == nil { + return utils.HashInfo{} + } + hashes := map[*utils.HashType]string{} + if meta.MD5 != "" { + hashes[utils.MD5] = meta.MD5 + } + if meta.SHA1 != "" { + hashes[utils.SHA1] = meta.SHA1 + } + return utils.NewHashInfoByMap(hashes) +} + +func (d *Chunker) linkedObject(obj model.Obj) *Object { + if linked, ok := obj.(*Object); ok { + return linked + } + return nil +} + +func (d *Chunker) chunkPathsForObject(obj *Object) []string { + if obj == nil { + return nil + } + paths := make([]string, 0, len(obj.Parts)+1) + if obj.Chunked && obj.UsesMeta { + paths = append(paths, obj.GetPath()) + } + if !obj.Chunked { + paths = append(paths, obj.GetPath()) + return paths + } + for _, part := range obj.Parts { + paths = append(paths, d.makeChunkName(obj.GetPath(), part.No, part.XactID)) + } + return paths +} + +func (d *Chunker) cleanupReplacedObject(ctx context.Context, obj *Object, keep map[string]struct{}) error { + if obj == nil { + return nil + } + var errs []error + for _, logicalPath := range d.chunkPathsForObject(obj) { + if _, ok := keep[logicalPath]; ok { + continue + } + actualPath, err := d.getActualPathForRemote(logicalPath) + if err != nil { + errs = append(errs, err) + continue + } + if err := op.Remove(ctx, d.remoteStorage, actualPath); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (d *Chunker) buildKeepSet(paths ...string) map[string]struct{} { + keep := make(map[string]struct{}, len(paths)) + for _, p := range paths { + if p == "" { + continue + } + keep[utils.FixAndCleanPath(p)] = struct{}{} + } + return keep +} + +func (d *Chunker) chunkPartBaseName(filePath string, chunkNo int, xactID string) string { + return path.Base(d.makeChunkName(filePath, chunkNo, xactID)) +} + +func (d *Chunker) openChunkReader(ctx context.Context, parts []linkedPart, totalSize int64, req http_range.Range) (io.ReadCloser, error) { + if req.Start < 0 || req.Start > totalSize { + return nil, fmt.Errorf("range start out of bound") + } + if req.Length < 0 || req.Start+req.Length > totalSize { + req.Length = totalSize - req.Start + } + if req.Length == 0 { + return io.NopCloser(strings.NewReader("")), nil + } + + var ( + readers []io.Reader + closers = utils.EmptyClosers() + offset int64 + remaining = req.Length + ) + for _, part := range parts { + partStart := offset + partEnd := offset + part.part.Size + offset = partEnd + if req.Start >= partEnd || remaining <= 0 { + continue + } + localStart := int64(0) + if req.Start > partStart { + localStart = req.Start - partStart + } + localLength := utils.Min(part.part.Size-localStart, remaining) + rc, err := d.openPartRange(ctx, part.link, part.part.Size, localStart, localLength) + if err != nil { + _ = closers.Close() + return nil, err + } + readers = append(readers, rc) + closers.Add(rc) + remaining -= localLength + } + if remaining > 0 { + _ = closers.Close() + return nil, errors.New("missing chunk data") + } + return utils.NewReadCloser(io.MultiReader(readers...), func() error { + return closers.Close() + }), nil +} + +func (d *Chunker) openPartRange(ctx context.Context, link *model.Link, size, offset, length int64) (io.ReadCloser, error) { + httpRange := http_range.Range{Start: offset, Length: length} + switch { + case link.MFile != nil: + return io.NopCloser(io.NewSectionReader(link.MFile, offset, length)), nil + case link.RangeReadCloser != nil: + return link.RangeReadCloser.RangeRead(ctx, httpRange) + case link.URL != "": + rrc, err := stream.GetRangeReadCloserFromLink(size, link) + if err != nil { + return nil, err + } + rc, err := rrc.RangeRead(ctx, httpRange) + if err != nil { + _ = rrc.Close() + return nil, err + } + return utils.NewReadCloser(rc, func() error { + return rrc.Close() + }), nil + default: + return nil, errors.New("chunk part has no readable link") + } +} + +func fsList(ctx context.Context, remotePath string, refresh bool) ([]model.Obj, error) { + return fs.List(ctx, remotePath, &fs.ListArgs{NoLog: true, Refresh: refresh}) +} diff --git a/drivers/chunker/util_test.go b/drivers/chunker/util_test.go new file mode 100644 index 00000000000..96f06051b75 --- /dev/null +++ b/drivers/chunker/util_test.go @@ -0,0 +1,115 @@ +package chunker + +import ( + "context" + "testing" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +func newTestChunker(t *testing.T) *Chunker { + t.Helper() + d := &Chunker{ + Addition: Addition{ + NameFormat: defaultChunkNameFmt, + StartFrom: defaultStartFrom, + }, + } + if err := d.setChunkNameFormat(d.NameFormat); err != nil { + t.Fatalf("setChunkNameFormat: %v", err) + } + return d +} + +func TestParseChunkName(t *testing.T) { + d := newTestChunker(t) + + mainName, chunkNo, ctrlType, xactID := d.parseChunkName("movie.mkv.rclone_chunk.001") + if mainName != "movie.mkv" || chunkNo != 0 || ctrlType != "" || xactID != "" { + t.Fatalf("unexpected parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) + } + + mainName, chunkNo, ctrlType, xactID = d.parseChunkName("movie.mkv.rclone_chunk.003_abcd") + if mainName != "movie.mkv" || chunkNo != 2 || ctrlType != "" || xactID != "abcd" { + t.Fatalf("unexpected temp parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) + } + + mainName, chunkNo, ctrlType, xactID = d.parseChunkName("movie.mkv.rclone_chunk._meta") + if mainName != "movie.mkv" || chunkNo != -1 || ctrlType != "meta" || xactID != "" { + t.Fatalf("unexpected control parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) + } +} + +func TestMarshalAndUnmarshalMetadata(t *testing.T) { + data, err := marshalMetadata(123, 2, "5d41402abc4b2a76b9719d911017c592", "", "") + if err != nil { + t.Fatalf("marshalMetadata: %v", err) + } + meta, err := unmarshalMetadata(data) + if err != nil { + t.Fatalf("unmarshalMetadata: %v", err) + } + if meta.Version != 1 || meta.Size != 123 || meta.NChunks != 2 || meta.MD5 == "" || meta.XactID != "" { + t.Fatalf("unexpected metadata: %+v", meta) + } + + data, err = marshalMetadata(456, 3, "", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "txn1") + if err != nil { + t.Fatalf("marshalMetadata with txn: %v", err) + } + meta, err = unmarshalMetadata(data) + if err != nil { + t.Fatalf("unmarshalMetadata with txn: %v", err) + } + if meta.Version != 2 || meta.Size != 456 || meta.NChunks != 3 || meta.SHA1 == "" || meta.XactID != "txn1" { + t.Fatalf("unexpected txn metadata: %+v", meta) + } +} + +func TestBuildListedObjectWithoutMetadata(t *testing.T) { + d := newTestChunker(t) + now := time.Now() + + obj, ok, err := d.buildListedObject(context.Background(), "/", "archive.bin", &groupInfo{ + partsByXact: map[string]map[int]chunkPart{ + "": { + 0: {No: 0, Size: 5}, + 1: {No: 1, Size: 7}, + }, + }, + }) + if err != nil { + t.Fatalf("buildListedObject: %v", err) + } + if !ok { + t.Fatal("expected grouped object") + } + grouped, ok := obj.(*Object) + if !ok { + t.Fatalf("expected *Object, got %T", obj) + } + if !grouped.Chunked || grouped.GetSize() != 12 || len(grouped.Parts) != 2 { + t.Fatalf("unexpected grouped object: %+v", grouped) + } + + raw, ok, err := d.buildListedObject(context.Background(), "/", "raw.txt", &groupInfo{ + base: &model.Object{ + Name: "raw.txt", + Size: 9, + Modified: now, + Ctime: now, + }, + partsByXact: map[string]map[int]chunkPart{}, + }) + if err != nil { + t.Fatalf("build raw listed object: %v", err) + } + if !ok { + t.Fatal("expected raw object") + } + rawObj := raw.(*Object) + if rawObj.Chunked || rawObj.GetSize() != 9 { + t.Fatalf("unexpected raw object: %+v", rawObj) + } +} diff --git a/drivers/crypt/meta.go b/drivers/crypt/meta.go index 180773a3f48..0878f63869f 100644 --- a/drivers/crypt/meta.go +++ b/drivers/crypt/meta.go @@ -13,16 +13,16 @@ type Addition struct { FileNameEnc string `json:"filename_encryption" type:"select" required:"true" options:"off,standard,obfuscate" default:"off"` DirNameEnc string `json:"directory_name_encryption" type:"select" required:"true" options:"false,true" default:"false"` - RemotePath string `json:"remote_path" required:"true" help:"This is where the encrypted data stores"` + RemotePath string `json:"remote_path" required:"true" help:"AList mounted folder path used to store encrypted data, e.g. /my-storage/secret"` Password string `json:"password" required:"true" confidential:"true" help:"the main password"` Salt string `json:"salt" confidential:"true" help:"If you don't know what is salt, treat it as a second password. Optional but recommended"` EncryptedSuffix string `json:"encrypted_suffix" required:"true" default:".bin" help:"for advanced user only! encrypted files will have this suffix"` FileNameEncoding string `json:"filename_encoding" type:"select" required:"true" options:"base64,base32,base32768" default:"base64" help:"for advanced user only!"` - Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` + Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` - ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` + ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` } var config = driver.Config{ From ce6a192d62a9c2d2b95d4eba5985aef9415f0d75 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 26 Mar 2026 15:39:56 +0800 Subject: [PATCH 093/133] feat(chunker): support multi-target chunk storage --- drivers/chunker/driver.go | 196 ++++++++---- drivers/chunker/meta.go | 16 +- drivers/chunker/types.go | 37 ++- drivers/chunker/util.go | 570 +++++++++++++++++++++++++++-------- drivers/chunker/util_test.go | 106 ++++++- 5 files changed, 726 insertions(+), 199 deletions(-) diff --git a/drivers/chunker/driver.go b/drivers/chunker/driver.go index 004fde1c476..f05d4a9f89a 100644 --- a/drivers/chunker/driver.go +++ b/drivers/chunker/driver.go @@ -49,16 +49,28 @@ func (d *Chunker) Init(ctx context.Context) error { return err } - storage, err := fs.GetStorage(d.RemotePath, &fs.GetStoragesArgs{}) - if err != nil { - return fmt.Errorf("can't find remote storage: %w", err) + targetPaths := d.configuredRemotePaths() + d.remoteTargets = make([]remoteTarget, 0, len(targetPaths)) + for _, targetPath := range targetPaths { + storage, err := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if err != nil { + return fmt.Errorf("can't find remote storage %q: %w", targetPath, err) + } + d.remoteTargets = append(d.remoteTargets, remoteTarget{ + MountPath: targetPath, + Storage: storage, + }) + } + if len(d.remoteTargets) == 0 { + return fmt.Errorf("can't find remote storage: %w", errs.ObjectNotFound) } - d.remoteStorage = storage + d.remoteStorage = d.remoteTargets[0].Storage return nil } func (d *Chunker) Drop(ctx context.Context) error { d.remoteStorage = nil + d.remoteTargets = nil return nil } @@ -93,22 +105,26 @@ func (d *Chunker) Get(ctx context.Context, pathStr string) (model.Obj, error) { func (d *Chunker) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { obj := d.linkedObject(file) if obj == nil || !obj.Chunked { - actualPath, err := d.getActualPathForRemote(file.GetPath()) + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + actualPath, err := d.getActualPathForRemoteOnTarget(file.GetPath(), remoteIndex) if err != nil { return nil, fmt.Errorf("failed to convert path to remote path: %w", err) } - link, _, err := op.Link(ctx, d.remoteStorage, actualPath, args) + link, _, err := op.Link(ctx, d.remoteTargets[remoteIndex].Storage, actualPath, args) return link, err } linkedParts := make([]linkedPart, 0, len(obj.Parts)) baseClosers := utils.EmptyClosers() for _, part := range obj.Parts { - actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID) + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID, part.RemoteIndex) if err != nil { return nil, fmt.Errorf("failed to convert chunk path: %w", err) } - link, _, err := op.Link(ctx, d.remoteStorage, actualPath, args) + link, _, err := op.Link(ctx, d.remoteTargets[part.RemoteIndex].Storage, actualPath, args) if err != nil { return nil, err } @@ -135,37 +151,50 @@ func (d *Chunker) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Chunker) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - dstDirActualPath, err := d.getActualPathForRemote(parentDir.GetPath()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - return op.MakeDir(ctx, d.remoteStorage, path.Join(dstDirActualPath, dirName)) + return d.ensureDirOnAllTargets(ctx, path.Join(parentDir.GetPath(), dirName)) } func (d *Chunker) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj.IsDir() { + return d.moveDirAcrossTargets(ctx, srcObj.GetPath(), dstDir.GetPath()) + } obj := d.linkedObject(srcObj) - if srcObj.IsDir() || obj == nil || !obj.Chunked { - srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDir.GetPath()); err != nil { + return err + } + srcRemoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - return op.Move(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + return op.Move(ctx, d.remoteTargets[remoteIndex].Storage, srcRemoteActualPath, dstRemoteActualPath) } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - for _, logicalPath := range d.chunkPathsForObject(obj) { - actualPath, err := d.getActualPathForRemote(logicalPath) + ensuredTargets := map[int]struct{}{} + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := ensuredTargets[location.RemoteIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, location.RemoteIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[location.RemoteIndex] = struct{}{} + } + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) if err != nil { return err } - if err := op.Move(ctx, d.remoteStorage, actualPath, dstRemoteActualPath); err != nil { + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), location.RemoteIndex) + if err != nil { + return err + } + if err := op.Move(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath, dstRemoteActualPath); err != nil { return err } } @@ -173,31 +202,38 @@ func (d *Chunker) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Chunker) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if srcObj.IsDir() { + return d.renameDirAcrossTargets(ctx, srcObj.GetPath(), newName) + } obj := d.linkedObject(srcObj) - if srcObj.IsDir() || obj == nil || !obj.Chunked { - remoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + remoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - return op.Rename(ctx, d.remoteStorage, remoteActualPath, newName) + return op.Rename(ctx, d.remoteTargets[remoteIndex].Storage, remoteActualPath, newName) } for _, part := range obj.Parts { - actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID) + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID, part.RemoteIndex) if err != nil { return err } newChunkName := d.chunkPartBaseName(path.Join(path.Dir(obj.GetPath()), newName), part.No, part.XactID) - if err := op.Rename(ctx, d.remoteStorage, actualPath, newChunkName); err != nil { + if err := op.Rename(ctx, d.remoteTargets[part.RemoteIndex].Storage, actualPath, newChunkName); err != nil { return err } } if obj.UsesMeta { - actualPath, err := d.getActualPathForRemote(obj.GetPath()) + actualPath, err := d.getActualPathForRemoteOnTarget(obj.GetPath(), obj.MainRemoteIndex) if err != nil { return err } - if err := op.Rename(ctx, d.remoteStorage, actualPath, newName); err != nil { + if err := op.Rename(ctx, d.remoteTargets[obj.MainRemoteIndex].Storage, actualPath, newName); err != nil { return err } } @@ -205,29 +241,46 @@ func (d *Chunker) Rename(ctx context.Context, srcObj model.Obj, newName string) } func (d *Chunker) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj.IsDir() { + return d.copyDirAcrossTargets(ctx, srcObj.GetPath(), dstDir.GetPath()) + } obj := d.linkedObject(srcObj) - if srcObj.IsDir() || obj == nil || !obj.Chunked { - srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDir.GetPath()); err != nil { + return err + } + srcRemoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - return op.Copy(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + return op.Copy(ctx, d.remoteTargets[remoteIndex].Storage, srcRemoteActualPath, dstRemoteActualPath) } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - for _, logicalPath := range d.chunkPathsForObject(obj) { - actualPath, err := d.getActualPathForRemote(logicalPath) + ensuredTargets := map[int]struct{}{} + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := ensuredTargets[location.RemoteIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, location.RemoteIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[location.RemoteIndex] = struct{}{} + } + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) if err != nil { return err } - if err := op.Copy(ctx, d.remoteStorage, actualPath, dstRemoteActualPath); err != nil { + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), location.RemoteIndex) + if err != nil { + return err + } + if err := op.Copy(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath, dstRemoteActualPath); err != nil { return err } } @@ -235,21 +288,28 @@ func (d *Chunker) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Chunker) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + return d.removeDirAcrossTargets(ctx, obj.GetPath()) + } chunkedObj := d.linkedObject(obj) - if obj.IsDir() || chunkedObj == nil || !chunkedObj.Chunked { - remoteActualPath, err := d.getActualPathForRemote(obj.GetPath()) + if chunkedObj == nil || !chunkedObj.Chunked { + remoteIndex := 0 + if chunkedObj != nil { + remoteIndex = chunkedObj.MainRemoteIndex + } + remoteActualPath, err := d.getActualPathForRemoteOnTarget(obj.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - return op.Remove(ctx, d.remoteStorage, remoteActualPath) + return op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, remoteActualPath) } - for _, logicalPath := range d.chunkPathsForObject(chunkedObj) { - actualPath, err := d.getActualPathForRemote(logicalPath) + for _, location := range d.objectLocationsForObject(chunkedObj) { + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) if err != nil { return err } - if err := op.Remove(ctx, d.remoteStorage, actualPath); err != nil { + if err := op.Remove(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath); err != nil { return err } } @@ -257,18 +317,21 @@ func (d *Chunker) Remove(ctx context.Context, obj model.Obj) error { } func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.FileStreamer, up driver.UpdateProgress) error { - dstDirActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + primaryDirActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), 0) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } + if err := d.ensureDirOnTarget(ctx, 0, dstDir.GetPath()); err != nil { + return err + } existing := d.linkedObject(streamer.GetExist()) logicalPath := path.Join(dstDir.GetPath(), streamer.GetName()) if streamer.GetSize() <= d.ChunkSize { - if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, streamer, up, false); err != nil { + if err := op.Put(ctx, d.remoteTargets[0].Storage, primaryDirActualPath, streamer, up, false); err != nil { return err } - return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(logicalPath)) + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(d.targetLocation(logicalPath, 0))) } if up == nil { @@ -301,9 +364,21 @@ func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.File chunkCount := 0 remaining := streamer.GetSize() - keepPaths := []string{logicalPath} + keepLocations := make([]objectLocation, 0, len(d.remoteTargets)+1) + ensuredTargets := map[int]struct{}{0: {}} for remaining > 0 { chunkLen := utils.Min(remaining, d.ChunkSize) + targetIndex := d.chunkTargetIndex(chunkCount) + if _, ok := ensuredTargets[targetIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, targetIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[targetIndex] = struct{}{} + } + dstDirActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), targetIndex) + if err != nil { + return err + } chunkName := d.chunkPartBaseName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) chunkPath := d.makeChunkName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) partReader := driver.NewLimitedUploadStream(ctx, &driver.ReaderWithCtx{ @@ -323,10 +398,10 @@ func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.File WebPutAsTask: streamer.NeedStore(), ForceStreamUpload: true, } - if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, partStream, nil, false); err != nil { + if err := op.Put(ctx, d.remoteTargets[targetIndex].Storage, dstDirActualPath, partStream, nil, false); err != nil { return err } - keepPaths = append(keepPaths, chunkPath) + keepLocations = append(keepLocations, d.targetLocation(chunkPath, targetIndex)) remaining -= chunkLen chunkCount++ } @@ -358,17 +433,20 @@ func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.File WebPutAsTask: false, ForceStreamUpload: true, } - if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, metaStream, nil, false); err != nil { + if err := op.Put(ctx, d.remoteTargets[0].Storage, primaryDirActualPath, metaStream, nil, false); err != nil { return err } + keepLocations = append(keepLocations, d.targetLocation(logicalPath, 0)) } else { - actualPath, err := d.getActualPathForRemote(logicalPath) - if err == nil { - _ = op.Remove(ctx, d.remoteStorage, actualPath) + for remoteIndex := range d.remoteTargets { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) + if err == nil { + _ = op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, actualPath) + } } } - return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(keepPaths...)) + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(keepLocations...)) } func xactIDIfNeeded(metaFormat, xactID string) string { diff --git a/drivers/chunker/meta.go b/drivers/chunker/meta.go index fabfbc67545..27265fe3033 100644 --- a/drivers/chunker/meta.go +++ b/drivers/chunker/meta.go @@ -7,19 +7,21 @@ import ( const ( defaultChunkSize int64 = 2147483648 - defaultChunkNameFmt = "*.rclone_chunk.###" + defaultChunkNameFmt = "{name}.rclone_chunk.{chunk:3}" defaultMetaFormat = "simplejson" defaultHashType = "md5" defaultStartFrom = 1 ) type Addition struct { - RemotePath string `json:"remote_path" required:"true" help:"AList mounted folder path used to store chunked data, e.g. /my-storage/chunks"` - ChunkSize int64 `json:"chunk_size" type:"number" required:"true" default:"2147483648" help:"Files larger than this will be split into chunks"` - NameFormat string `json:"name_format" required:"true" default:"*.rclone_chunk.###" help:"Compatible with rclone chunker naming"` - StartFrom int `json:"start_from" type:"number" required:"true" default:"1" help:"Chunk number base, usually 0 or 1"` - MetaFormat string `json:"meta_format" type:"select" required:"true" options:"simplejson,none" default:"simplejson" help:"simplejson is compatible with rclone chunker metadata"` - HashType string `json:"hash_type" type:"select" required:"true" options:"none,md5,sha1" default:"md5" help:"Hash stored in metadata for chunked files"` + RemotePath string `json:"remote_path" required:"true" help:"Primary AList mounted folder path used to store metadata and small files, e.g. /my-storage/chunks"` + RemotePaths string `json:"remote_paths" type:"text" help:"Additional AList mounted folder paths, one per line. Chunk files will be distributed across remote_path and these extra paths."` + StoreChunksInPrimary bool `json:"store_chunks_in_primary" type:"bool" default:"true" help:"When extra remote paths are configured, also store chunk files in remote_path"` + ChunkSize int64 `json:"chunk_size" type:"number" required:"true" default:"2147483648" help:"Files larger than this will be split into chunks"` + NameFormat string `json:"name_format" required:"true" default:"{name}.rclone_chunk.{chunk:3}" help:"Magic tokens: {name}, {chunk}, {chunk:N}. Name token must appear before chunk token."` + StartFrom int `json:"start_from" type:"number" required:"true" default:"1" help:"Chunk number base, usually 0 or 1"` + MetaFormat string `json:"meta_format" type:"select" required:"true" options:"simplejson,none" default:"simplejson" help:"simplejson is compatible with rclone chunker metadata"` + HashType string `json:"hash_type" type:"select" required:"true" options:"none,md5,sha1" default:"md5" help:"Hash stored in metadata for chunked files"` } var config = driver.Config{ diff --git a/drivers/chunker/types.go b/drivers/chunker/types.go index e89fa45c5cd..ff0f6fc4d85 100644 --- a/drivers/chunker/types.go +++ b/drivers/chunker/types.go @@ -19,15 +19,32 @@ const ( ) var ctrlTypeRegexp = regexp.MustCompile(`^` + ctrlTypeRegStr + `$`) +var chunkTokenRegexp = regexp.MustCompile(`\{chunk(?::([0-9]+))?\}`) type Chunker struct { model.Storage Addition remoteStorage driver.Driver + remoteTargets []remoteTarget dataNameFmt string nameRegexp *regexp.Regexp } +type remoteTarget struct { + MountPath string + Storage driver.Driver +} + +type locatedObj struct { + Obj model.Obj + RemoteIndex int +} + +type objectLocation struct { + LogicalPath string + RemoteIndex int +} + type metadataJSON struct { Version *int `json:"ver"` Size *int64 `json:"size"` @@ -47,23 +64,25 @@ type chunkMetadata struct { } type chunkPart struct { - No int - Size int64 - XactID string + No int + Size int64 + XactID string + RemoteIndex int } type groupInfo struct { - base model.Obj + base *locatedObj partsByXact map[string]map[int]chunkPart } type Object struct { model.Object - Main model.Obj - Parts []chunkPart - Meta *chunkMetadata - Chunked bool - UsesMeta bool + Main model.Obj + MainRemoteIndex int + Parts []chunkPart + Meta *chunkMetadata + Chunked bool + UsesMeta bool } type linkedPart struct { diff --git a/drivers/chunker/util.go b/drivers/chunker/util.go index 5711b2126f9..ebb740e8194 100644 --- a/drivers/chunker/util.go +++ b/drivers/chunker/util.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" @@ -23,7 +24,7 @@ import ( ) func (d *Chunker) validateOptions() error { - if d.RemotePath == "" { + if strings.TrimSpace(d.RemotePath) == "" { return errors.New("remote_path is required") } if d.ChunkSize <= 0 { @@ -45,50 +46,113 @@ func (d *Chunker) validateOptions() error { return nil } -func (d *Chunker) setChunkNameFormat(pattern string) error { - if strings.Count(pattern, "*") != 1 { - return errors.New("pattern must have exactly one asterisk (*)") - } - hashCount := strings.Count(pattern, "#") - if hashCount < 1 { - return errors.New("pattern must contain a hash character (#)") - } - if strings.Index(pattern, "*") > strings.Index(pattern, "#") { - return errors.New("asterisk (*) in pattern must come before hashes (#)") +func (d *Chunker) configuredRemotePaths() []string { + seen := map[string]struct{}{} + paths := make([]string, 0, 1) + addPath := func(p string) { + p = strings.TrimSpace(p) + if p == "" { + return + } + p = utils.FixAndCleanPath(p) + if _, ok := seen[p]; ok { + return + } + seen[p] = struct{}{} + paths = append(paths, p) } - if ok, _ := regexp.MatchString("^[^#]*[#]+[^#]*$", pattern); !ok { - return errors.New("hashes (#) in pattern must be consecutive") + + addPath(d.RemotePath) + for _, line := range strings.Split(d.RemotePaths, "\n") { + addPath(line) } + return paths +} + +func (d *Chunker) setChunkNameFormat(pattern string) error { if dir, _ := path.Split(pattern); dir != "" { return errors.New("directory separator prohibited") } - if pattern[0] != '*' { - return errors.New("pattern must start with asterisk") + + nameStart, nameEnd, err := parseNameToken(pattern) + if err != nil { + return err + } + chunkStart, chunkEnd, chunkWidth, err := parseChunkToken(pattern) + if err != nil { + return err + } + if nameStart > chunkStart { + return errors.New("name token must come before chunk token") } - reHashes := regexp.MustCompile("[#]+") reDigits := "[0-9]+" - if hashCount > 1 { - reDigits = fmt.Sprintf("[0-9]{%d,}", hashCount) + if chunkWidth > 0 { + reDigits = fmt.Sprintf("[0-9]{%d,}", chunkWidth) } reDataOrCtrl := fmt.Sprintf("(?:(%s)|_(%s))", reDigits, ctrlTypeRegStr) - strRegex := regexp.QuoteMeta(pattern) - strRegex = reHashes.ReplaceAllLiteralString(strRegex, reDataOrCtrl) - strRegex = strings.Replace(strRegex, "\\*", "(.+?)", 1) - strRegex = fmt.Sprintf("^%s(?:%s|%s)?$", strRegex, tempSuffixRegStr, tempSuffixRegOld) + beforeName := pattern[:nameStart] + between := pattern[nameEnd:chunkStart] + afterChunk := pattern[chunkEnd:] + + strRegex := fmt.Sprintf( + "^%s(.+?)%s%s%s(?:%s|%s)?$", + regexp.QuoteMeta(beforeName), + regexp.QuoteMeta(between), + reDataOrCtrl, + regexp.QuoteMeta(afterChunk), + tempSuffixRegStr, + tempSuffixRegOld, + ) d.nameRegexp = regexp.MustCompile(strRegex) fmtDigits := "%d" - if hashCount > 1 { - fmtDigits = fmt.Sprintf("%%0%dd", hashCount) - } - strFmt := strings.ReplaceAll(pattern, "%", "%%") - strFmt = strings.Replace(strFmt, "*", "%s", 1) - d.dataNameFmt = reHashes.ReplaceAllLiteralString(strFmt, fmtDigits) + if chunkWidth > 0 { + fmtDigits = fmt.Sprintf("%%0%dd", chunkWidth) + } + d.dataNameFmt = strings.ReplaceAll(beforeName, "%", "%%") + + "%s" + + strings.ReplaceAll(between, "%", "%%") + + fmtDigits + + strings.ReplaceAll(afterChunk, "%", "%%") return nil } +func parseNameToken(pattern string) (start, end int, err error) { + nameMagicCount := strings.Count(pattern, "{name}") + switch nameMagicCount { + case 0: + return 0, 0, errors.New("pattern must contain one name token: {name}") + case 1: + default: + return 0, 0, errors.New("pattern must contain exactly one name token: {name}") + } + start = strings.Index(pattern, "{name}") + return start, start + len("{name}"), nil +} + +func parseChunkToken(pattern string) (start, end, width int, err error) { + chunkMatches := chunkTokenRegexp.FindAllStringSubmatchIndex(pattern, -1) + switch len(chunkMatches) { + case 0: + return 0, 0, 0, errors.New("pattern must contain one chunk token: {chunk} or {chunk:N}") + case 1: + default: + return 0, 0, 0, errors.New("pattern must contain exactly one chunk token: {chunk} or {chunk:N}") + } + match := chunkMatches[0] + start = match[0] + end = match[1] + if match[2] >= 0 && match[3] >= 0 { + width, err = strconv.Atoi(pattern[match[2]:match[3]]) + if err != nil || width <= 0 { + return 0, 0, 0, errors.New("chunk width in {chunk:N} must be a positive integer") + } + } + return start, end, width, nil +} + func (d *Chunker) makeChunkName(filePath string, chunkNo int, xactID string) string { dir, baseName := path.Split(filePath) name := fmt.Sprintf(d.dataNameFmt, baseName, chunkNo+d.StartFrom) @@ -197,75 +261,159 @@ func unmarshalMetadata(data []byte) (*chunkMetadata, error) { }, nil } -func (d *Chunker) joinRemotePath(logicalPath string) string { +func joinRemotePathWithBase(baseMountPath, logicalPath string) string { logicalPath = utils.FixAndCleanPath(logicalPath) if utils.PathEqual(logicalPath, "/") { - return d.RemotePath + return utils.FixAndCleanPath(baseMountPath) } - return path.Join(d.RemotePath, logicalPath) + return path.Join(utils.FixAndCleanPath(baseMountPath), logicalPath) +} + +func (d *Chunker) joinRemotePath(logicalPath string) string { + return joinRemotePathWithBase(d.RemotePath, logicalPath) +} + +func (d *Chunker) joinRemotePathForTarget(logicalPath string, remoteIndex int) string { + target := d.remoteTargets[remoteIndex] + return joinRemotePathWithBase(target.MountPath, logicalPath) } func (d *Chunker) getActualPathForRemote(logicalPath string) (string, error) { - _, actualPath, err := op.GetStorageAndActualPath(d.joinRemotePath(logicalPath)) + return d.getActualPathForRemoteOnTarget(logicalPath, 0) +} + +func (d *Chunker) getActualPathForRemoteOnTarget(logicalPath string, remoteIndex int) (string, error) { + _, actualPath, err := op.GetStorageAndActualPath(d.joinRemotePathForTarget(logicalPath, remoteIndex)) return actualPath, err } -func (d *Chunker) getActualChunkPath(filePath string, chunkNo int, xactID string) (string, error) { - return d.getActualPathForRemote(d.makeChunkName(filePath, chunkNo, xactID)) +func (d *Chunker) getActualChunkPath(filePath string, chunkNo int, xactID string, remoteIndex int) (string, error) { + return d.getActualPathForRemoteOnTarget(d.makeChunkName(filePath, chunkNo, xactID), remoteIndex) } -func (d *Chunker) listDirObjects(ctx context.Context, dirPath string, refresh bool) ([]model.Obj, error) { - remotePath := d.joinRemotePath(dirPath) - entries, err := fsList(ctx, remotePath, refresh) - if err != nil { - return nil, err +func (d *Chunker) chunkTargetIndex(chunkNo int) int { + targetIndexes := d.chunkTargetIndexes() + if len(targetIndexes) == 0 { + return 0 + } + if chunkNo < 0 { + return targetIndexes[0] } + return targetIndexes[chunkNo%len(targetIndexes)] +} +func (d *Chunker) chunkTargetIndexes() []int { + if len(d.remoteTargets) <= 1 { + return []int{0} + } + if d.StoreChunksInPrimary { + targets := make([]int, 0, len(d.remoteTargets)) + for i := range d.remoteTargets { + targets = append(targets, i) + } + return targets + } + targets := make([]int, 0, len(d.remoteTargets)-1) + for i := 1; i < len(d.remoteTargets); i++ { + targets = append(targets, i) + } + if len(targets) == 0 { + return []int{0} + } + return targets +} + +func (d *Chunker) targetLocation(logicalPath string, remoteIndex int) objectLocation { + return objectLocation{ + LogicalPath: utils.FixAndCleanPath(logicalPath), + RemoteIndex: remoteIndex, + } +} + +func (d *Chunker) chunkLocation(filePath string, part chunkPart) objectLocation { + return d.targetLocation(d.makeChunkName(filePath, part.No, part.XactID), part.RemoteIndex) +} + +func (d *Chunker) listDirObjects(ctx context.Context, dirPath string, refresh bool) ([]model.Obj, error) { groups := map[string]*groupInfo{} - var dirs []model.Obj - - for _, entry := range entries { - if entry.IsDir() { - dirs = append(dirs, &model.Object{ - Name: entry.GetName(), - Path: path.Join(dirPath, entry.GetName()), - Size: 0, - Modified: entry.ModTime(), - Ctime: entry.CreateTime(), - IsFolder: true, - HashInfo: entry.GetHash(), - }) - continue + dirMap := map[string]model.Obj{} + found := false + + for remoteIndex := range d.remoteTargets { + remotePath := d.joinRemotePathForTarget(dirPath, remoteIndex) + entries, err := fsList(ctx, remotePath, refresh) + if err != nil { + if errs.IsObjectNotFound(err) { + continue + } + return nil, err } + found = true + + for _, entry := range entries { + if entry.IsDir() { + if _, ok := dirMap[entry.GetName()]; !ok { + dirMap[entry.GetName()] = &model.Object{ + Name: entry.GetName(), + Path: path.Join(dirPath, entry.GetName()), + Size: 0, + Modified: entry.ModTime(), + Ctime: entry.CreateTime(), + IsFolder: true, + HashInfo: entry.GetHash(), + } + } + continue + } - mainName, chunkNo, ctrlType, xactID := d.parseChunkName(entry.GetName()) - if mainName == "" { - g := groups[entry.GetName()] + mainName, chunkNo, ctrlType, xactID := d.parseChunkName(entry.GetName()) + if mainName == "" { + g := groups[entry.GetName()] + if g == nil { + g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} + groups[entry.GetName()] = g + } + if g.base == nil || remoteIndex < g.base.RemoteIndex { + g.base = &locatedObj{ + Obj: entry, + RemoteIndex: remoteIndex, + } + } + continue + } + if chunkNo < 0 || ctrlType != "" { + continue + } + g := groups[mainName] if g == nil { g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} - groups[entry.GetName()] = g + groups[mainName] = g + } + if g.partsByXact[xactID] == nil { + g.partsByXact[xactID] = map[int]chunkPart{} + } + part := chunkPart{ + No: chunkNo, + Size: entry.GetSize(), + XactID: xactID, + RemoteIndex: remoteIndex, + } + existing, ok := g.partsByXact[xactID][chunkNo] + if !ok || part.RemoteIndex < existing.RemoteIndex { + g.partsByXact[xactID][chunkNo] = part } - g.base = entry - continue - } - if chunkNo < 0 || ctrlType != "" { - continue - } - g := groups[mainName] - if g == nil { - g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} - groups[mainName] = g - } - if g.partsByXact[xactID] == nil { - g.partsByXact[xactID] = map[int]chunkPart{} - } - g.partsByXact[xactID][chunkNo] = chunkPart{ - No: chunkNo, - Size: entry.GetSize(), - XactID: xactID, } } + if !found && !utils.PathEqual(dirPath, "/") { + return nil, errs.ObjectNotFound + } + + dirs := make([]model.Obj, 0, len(dirMap)) + for _, obj := range dirMap { + dirs = append(dirs, obj) + } + result := make([]model.Obj, 0, len(dirs)+len(groups)) result = append(result, dirs...) for name, group := range groups { @@ -280,11 +428,180 @@ func (d *Chunker) listDirObjects(ctx context.Context, dirPath string, refresh bo return result, nil } +func (d *Chunker) targetPathExists(ctx context.Context, remoteIndex int, logicalPath string) (bool, error) { + _, err := fs.Get(ctx, d.joinRemotePathForTarget(logicalPath, remoteIndex), &fs.GetArgs{NoLog: true}) + if err == nil { + return true, nil + } + if errs.IsObjectNotFound(err) { + return false, nil + } + return false, err +} + +func (d *Chunker) ensureDirOnTarget(ctx context.Context, remoteIndex int, logicalDirPath string) error { + logicalDirPath = utils.FixAndCleanPath(logicalDirPath) + if utils.PathEqual(logicalDirPath, "/") { + return nil + } + return fs.MakeDir(ctx, d.joinRemotePathForTarget(logicalDirPath, remoteIndex)) +} + +func (d *Chunker) ensureDirOnAllTargets(ctx context.Context, logicalDirPath string) error { + var errsList []error + for remoteIndex := range d.remoteTargets { + if err := d.ensureDirOnTarget(ctx, remoteIndex, logicalDirPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) existingLocationsForDir(ctx context.Context, logicalDirPath string) ([]int, error) { + locations := make([]int, 0, len(d.remoteTargets)) + for remoteIndex := range d.remoteTargets { + exists, err := d.targetPathExists(ctx, remoteIndex, logicalDirPath) + if err != nil { + return nil, err + } + if exists { + locations = append(locations, remoteIndex) + } + } + return locations, nil +} + +func (d *Chunker) dirLocationsOrAll(ctx context.Context, logicalDirPath string) ([]int, error) { + locations, err := d.existingLocationsForDir(ctx, logicalDirPath) + if err != nil { + return nil, err + } + if len(locations) > 0 { + return locations, nil + } + all := make([]int, 0, len(d.remoteTargets)) + for remoteIndex := range d.remoteTargets { + all = append(all, remoteIndex) + } + return all, nil +} + +func (d *Chunker) moveDirAcrossTargets(ctx context.Context, srcPath, dstDirPath string) error { + locations, err := d.existingLocationsForDir(ctx, srcPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDirPath); err != nil { + errsList = append(errsList, err) + continue + } + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + dstActualPath, err := d.getActualPathForRemoteOnTarget(dstDirPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Move(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, dstActualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) copyDirAcrossTargets(ctx context.Context, srcPath, dstDirPath string) error { + locations, err := d.dirLocationsOrAll(ctx, srcPath) + if err != nil { + return err + } + var errsList []error + for _, remoteIndex := range locations { + exists, err := d.targetPathExists(ctx, remoteIndex, srcPath) + if err != nil { + errsList = append(errsList, err) + continue + } + if !exists { + continue + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDirPath); err != nil { + errsList = append(errsList, err) + continue + } + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + dstActualPath, err := d.getActualPathForRemoteOnTarget(dstDirPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Copy(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, dstActualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) renameDirAcrossTargets(ctx context.Context, srcPath, newName string) error { + locations, err := d.existingLocationsForDir(ctx, srcPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Rename(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, newName); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) removeDirAcrossTargets(ctx context.Context, logicalPath string) error { + locations, err := d.existingLocationsForDir(ctx, logicalPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, actualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, group *groupInfo) (model.Obj, bool, error) { var meta *chunkMetadata var err error - if group.base != nil && group.base.GetSize() <= maxMetadataSizeRead && len(group.partsByXact) > 0 { - meta, err = d.readMetadata(ctx, path.Join(dirPath, name), group.base.GetSize()) + if group.base != nil && group.base.Obj.GetSize() <= maxMetadataSizeRead && len(group.partsByXact) > 0 { + meta, err = d.readMetadata(ctx, path.Join(dirPath, name), group.base.Obj.GetSize(), group.base.RemoteIndex) if err != nil { meta = nil } @@ -295,13 +612,14 @@ func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, g Object: model.Object{ Name: name, Path: path.Join(dirPath, name), - Size: group.base.GetSize(), - Modified: group.base.ModTime(), - Ctime: group.base.CreateTime(), + Size: group.base.Obj.GetSize(), + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), IsFolder: false, - HashInfo: group.base.GetHash(), + HashInfo: group.base.Obj.GetHash(), }, - Main: group.base, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, }, true, nil } @@ -316,13 +634,14 @@ func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, g Object: model.Object{ Name: name, Path: path.Join(dirPath, name), - Size: group.base.GetSize(), - Modified: group.base.ModTime(), - Ctime: group.base.CreateTime(), + Size: group.base.Obj.GetSize(), + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), IsFolder: false, - HashInfo: group.base.GetHash(), + HashInfo: group.base.Obj.GetHash(), }, - Main: group.base, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, }, true, nil } @@ -334,15 +653,16 @@ func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, g Name: name, Path: path.Join(dirPath, name), Size: meta.Size, - Modified: group.base.ModTime(), - Ctime: group.base.CreateTime(), + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), IsFolder: false, HashInfo: buildHashInfo(meta), }, - Main: group.base, - Meta: meta, - Chunked: true, - UsesMeta: true, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, + Meta: meta, + Chunked: true, + UsesMeta: true, }, true, nil } return nil, false, nil @@ -355,13 +675,20 @@ func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, g modified := time.Time{} ctime := time.Time{} if group.base != nil { - modified = group.base.ModTime() - ctime = group.base.CreateTime() + modified = group.base.Obj.ModTime() + ctime = group.base.Obj.CreateTime() } if meta != nil { size = meta.Size } + mainRemoteIndex := 0 + var mainObj model.Obj + if group.base != nil { + mainRemoteIndex = group.base.RemoteIndex + mainObj = group.base.Obj + } + return &Object{ Object: model.Object{ Name: name, @@ -372,20 +699,21 @@ func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, g IsFolder: false, HashInfo: buildHashInfo(meta), }, - Main: group.base, - Parts: parts, - Meta: meta, - Chunked: true, - UsesMeta: meta != nil, + Main: mainObj, + MainRemoteIndex: mainRemoteIndex, + Parts: parts, + Meta: meta, + Chunked: true, + UsesMeta: meta != nil, }, true, nil } -func (d *Chunker) readMetadata(ctx context.Context, logicalPath string, size int64) (*chunkMetadata, error) { - actualPath, err := d.getActualPathForRemote(logicalPath) +func (d *Chunker) readMetadata(ctx context.Context, logicalPath string, size int64, remoteIndex int) (*chunkMetadata, error) { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) if err != nil { return nil, err } - link, obj, err := op.Link(ctx, d.remoteStorage, actualPath, model.LinkArgs{}) + link, obj, err := op.Link(ctx, d.remoteTargets[remoteIndex].Storage, actualPath, model.LinkArgs{}) if err != nil { return nil, err } @@ -441,22 +769,22 @@ func (d *Chunker) linkedObject(obj model.Obj) *Object { return nil } -func (d *Chunker) chunkPathsForObject(obj *Object) []string { +func (d *Chunker) objectLocationsForObject(obj *Object) []objectLocation { if obj == nil { return nil } - paths := make([]string, 0, len(obj.Parts)+1) + locations := make([]objectLocation, 0, len(obj.Parts)+1) if obj.Chunked && obj.UsesMeta { - paths = append(paths, obj.GetPath()) + locations = append(locations, d.targetLocation(obj.GetPath(), obj.MainRemoteIndex)) } if !obj.Chunked { - paths = append(paths, obj.GetPath()) - return paths + locations = append(locations, d.targetLocation(obj.GetPath(), obj.MainRemoteIndex)) + return locations } for _, part := range obj.Parts { - paths = append(paths, d.makeChunkName(obj.GetPath(), part.No, part.XactID)) + locations = append(locations, d.chunkLocation(obj.GetPath(), part)) } - return paths + return locations } func (d *Chunker) cleanupReplacedObject(ctx context.Context, obj *Object, keep map[string]struct{}) error { @@ -464,29 +792,33 @@ func (d *Chunker) cleanupReplacedObject(ctx context.Context, obj *Object, keep m return nil } var errs []error - for _, logicalPath := range d.chunkPathsForObject(obj) { - if _, ok := keep[logicalPath]; ok { + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := keep[d.keepKey(location)]; ok { continue } - actualPath, err := d.getActualPathForRemote(logicalPath) + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) if err != nil { errs = append(errs, err) continue } - if err := op.Remove(ctx, d.remoteStorage, actualPath); err != nil { + if err := op.Remove(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath); err != nil { errs = append(errs, err) } } return errors.Join(errs...) } -func (d *Chunker) buildKeepSet(paths ...string) map[string]struct{} { - keep := make(map[string]struct{}, len(paths)) - for _, p := range paths { - if p == "" { +func (d *Chunker) keepKey(location objectLocation) string { + return fmt.Sprintf("%d:%s", location.RemoteIndex, utils.FixAndCleanPath(location.LogicalPath)) +} + +func (d *Chunker) buildKeepSet(locations ...objectLocation) map[string]struct{} { + keep := make(map[string]struct{}, len(locations)) + for _, location := range locations { + if location.LogicalPath == "" { continue } - keep[utils.FixAndCleanPath(p)] = struct{}{} + keep[d.keepKey(location)] = struct{}{} } return keep } diff --git a/drivers/chunker/util_test.go b/drivers/chunker/util_test.go index 96f06051b75..23786adc07e 100644 --- a/drivers/chunker/util_test.go +++ b/drivers/chunker/util_test.go @@ -41,6 +41,41 @@ func TestParseChunkName(t *testing.T) { } } +func TestNamedMagicNameFormat(t *testing.T) { + d := &Chunker{ + Addition: Addition{ + NameFormat: "chunk-{name}-{chunk:4}.bin", + StartFrom: defaultStartFrom, + }, + } + if err := d.setChunkNameFormat(d.NameFormat); err != nil { + t.Fatalf("setChunkNameFormat named magic: %v", err) + } + + got := d.makeChunkName("/video/movie.mkv", 0, "") + want := "/video/chunk-movie.mkv-0001.bin" + if got != want { + t.Fatalf("makeChunkName = %q, want %q", got, want) + } + + mainName, chunkNo, ctrlType, xactID := d.parseChunkName("chunk-movie.mkv-0003.bin") + if mainName != "movie.mkv" || chunkNo != 2 || ctrlType != "" || xactID != "" { + t.Fatalf("unexpected named magic parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) + } +} + +func TestLegacyNameFormatRejected(t *testing.T) { + d := &Chunker{ + Addition: Addition{ + NameFormat: "chunk-*.###.bin", + StartFrom: defaultStartFrom, + }, + } + if err := d.setChunkNameFormat(d.NameFormat); err == nil { + t.Fatal("expected legacy syntax to be rejected") + } +} + func TestMarshalAndUnmarshalMetadata(t *testing.T) { data, err := marshalMetadata(123, 2, "5d41402abc4b2a76b9719d911017c592", "", "") if err != nil { @@ -94,11 +129,14 @@ func TestBuildListedObjectWithoutMetadata(t *testing.T) { } raw, ok, err := d.buildListedObject(context.Background(), "/", "raw.txt", &groupInfo{ - base: &model.Object{ - Name: "raw.txt", - Size: 9, - Modified: now, - Ctime: now, + base: &locatedObj{ + Obj: &model.Object{ + Name: "raw.txt", + Size: 9, + Modified: now, + Ctime: now, + }, + RemoteIndex: 0, }, partsByXact: map[string]map[int]chunkPart{}, }) @@ -113,3 +151,61 @@ func TestBuildListedObjectWithoutMetadata(t *testing.T) { t.Fatalf("unexpected raw object: %+v", rawObj) } } + +func TestConfiguredRemotePaths(t *testing.T) { + d := &Chunker{ + Addition: Addition{ + RemotePath: "/s1/chunks", + RemotePaths: "\n/s2/chunks\n/s1/chunks\n /s3/chunks \n", + }, + } + got := d.configuredRemotePaths() + want := []string{"/s1/chunks", "/s2/chunks", "/s3/chunks"} + if len(got) != len(want) { + t.Fatalf("configuredRemotePaths length = %d, want %d (%v)", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("configuredRemotePaths[%d] = %q, want %q", i, got[i], want[i]) + } + } +} + +func TestBuildKeepSetSeparatesRemoteTargets(t *testing.T) { + d := newTestChunker(t) + keep := d.buildKeepSet( + objectLocation{LogicalPath: "/movie.bin", RemoteIndex: 0}, + objectLocation{LogicalPath: "/movie.bin", RemoteIndex: 1}, + ) + if len(keep) != 2 { + t.Fatalf("buildKeepSet should keep distinct entries per remote target, got %d", len(keep)) + } +} + +func TestChunkTargetIndexes(t *testing.T) { + d := newTestChunker(t) + d.remoteTargets = []remoteTarget{{}, {}, {}} + d.StoreChunksInPrimary = true + if got := d.chunkTargetIndexes(); len(got) != 3 || got[0] != 0 || got[1] != 1 || got[2] != 2 { + t.Fatalf("chunkTargetIndexes with primary = %v", got) + } + if got := d.chunkTargetIndex(4); got != 1 { + t.Fatalf("chunkTargetIndex with primary = %d, want 1", got) + } + + d.StoreChunksInPrimary = false + if got := d.chunkTargetIndexes(); len(got) != 2 || got[0] != 1 || got[1] != 2 { + t.Fatalf("chunkTargetIndexes without primary = %v", got) + } + if got := d.chunkTargetIndex(0); got != 1 { + t.Fatalf("chunkTargetIndex without primary for first chunk = %d, want 1", got) + } + if got := d.chunkTargetIndex(3); got != 2 { + t.Fatalf("chunkTargetIndex without primary for chunk 3 = %d, want 2", got) + } + + d.remoteTargets = []remoteTarget{{}} + if got := d.chunkTargetIndex(0); got != 0 { + t.Fatalf("single target chunkTargetIndex = %d, want 0", got) + } +} From 428e04ee3c9cd2d04e985ed534ab17d54f8b8918 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 26 Mar 2026 15:54:27 +0800 Subject: [PATCH 094/133] test(chunker): remove util tests --- drivers/chunker/util_test.go | 211 ----------------------------------- 1 file changed, 211 deletions(-) delete mode 100644 drivers/chunker/util_test.go diff --git a/drivers/chunker/util_test.go b/drivers/chunker/util_test.go deleted file mode 100644 index 23786adc07e..00000000000 --- a/drivers/chunker/util_test.go +++ /dev/null @@ -1,211 +0,0 @@ -package chunker - -import ( - "context" - "testing" - "time" - - "github.com/alist-org/alist/v3/internal/model" -) - -func newTestChunker(t *testing.T) *Chunker { - t.Helper() - d := &Chunker{ - Addition: Addition{ - NameFormat: defaultChunkNameFmt, - StartFrom: defaultStartFrom, - }, - } - if err := d.setChunkNameFormat(d.NameFormat); err != nil { - t.Fatalf("setChunkNameFormat: %v", err) - } - return d -} - -func TestParseChunkName(t *testing.T) { - d := newTestChunker(t) - - mainName, chunkNo, ctrlType, xactID := d.parseChunkName("movie.mkv.rclone_chunk.001") - if mainName != "movie.mkv" || chunkNo != 0 || ctrlType != "" || xactID != "" { - t.Fatalf("unexpected parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) - } - - mainName, chunkNo, ctrlType, xactID = d.parseChunkName("movie.mkv.rclone_chunk.003_abcd") - if mainName != "movie.mkv" || chunkNo != 2 || ctrlType != "" || xactID != "abcd" { - t.Fatalf("unexpected temp parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) - } - - mainName, chunkNo, ctrlType, xactID = d.parseChunkName("movie.mkv.rclone_chunk._meta") - if mainName != "movie.mkv" || chunkNo != -1 || ctrlType != "meta" || xactID != "" { - t.Fatalf("unexpected control parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) - } -} - -func TestNamedMagicNameFormat(t *testing.T) { - d := &Chunker{ - Addition: Addition{ - NameFormat: "chunk-{name}-{chunk:4}.bin", - StartFrom: defaultStartFrom, - }, - } - if err := d.setChunkNameFormat(d.NameFormat); err != nil { - t.Fatalf("setChunkNameFormat named magic: %v", err) - } - - got := d.makeChunkName("/video/movie.mkv", 0, "") - want := "/video/chunk-movie.mkv-0001.bin" - if got != want { - t.Fatalf("makeChunkName = %q, want %q", got, want) - } - - mainName, chunkNo, ctrlType, xactID := d.parseChunkName("chunk-movie.mkv-0003.bin") - if mainName != "movie.mkv" || chunkNo != 2 || ctrlType != "" || xactID != "" { - t.Fatalf("unexpected named magic parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) - } -} - -func TestLegacyNameFormatRejected(t *testing.T) { - d := &Chunker{ - Addition: Addition{ - NameFormat: "chunk-*.###.bin", - StartFrom: defaultStartFrom, - }, - } - if err := d.setChunkNameFormat(d.NameFormat); err == nil { - t.Fatal("expected legacy syntax to be rejected") - } -} - -func TestMarshalAndUnmarshalMetadata(t *testing.T) { - data, err := marshalMetadata(123, 2, "5d41402abc4b2a76b9719d911017c592", "", "") - if err != nil { - t.Fatalf("marshalMetadata: %v", err) - } - meta, err := unmarshalMetadata(data) - if err != nil { - t.Fatalf("unmarshalMetadata: %v", err) - } - if meta.Version != 1 || meta.Size != 123 || meta.NChunks != 2 || meta.MD5 == "" || meta.XactID != "" { - t.Fatalf("unexpected metadata: %+v", meta) - } - - data, err = marshalMetadata(456, 3, "", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "txn1") - if err != nil { - t.Fatalf("marshalMetadata with txn: %v", err) - } - meta, err = unmarshalMetadata(data) - if err != nil { - t.Fatalf("unmarshalMetadata with txn: %v", err) - } - if meta.Version != 2 || meta.Size != 456 || meta.NChunks != 3 || meta.SHA1 == "" || meta.XactID != "txn1" { - t.Fatalf("unexpected txn metadata: %+v", meta) - } -} - -func TestBuildListedObjectWithoutMetadata(t *testing.T) { - d := newTestChunker(t) - now := time.Now() - - obj, ok, err := d.buildListedObject(context.Background(), "/", "archive.bin", &groupInfo{ - partsByXact: map[string]map[int]chunkPart{ - "": { - 0: {No: 0, Size: 5}, - 1: {No: 1, Size: 7}, - }, - }, - }) - if err != nil { - t.Fatalf("buildListedObject: %v", err) - } - if !ok { - t.Fatal("expected grouped object") - } - grouped, ok := obj.(*Object) - if !ok { - t.Fatalf("expected *Object, got %T", obj) - } - if !grouped.Chunked || grouped.GetSize() != 12 || len(grouped.Parts) != 2 { - t.Fatalf("unexpected grouped object: %+v", grouped) - } - - raw, ok, err := d.buildListedObject(context.Background(), "/", "raw.txt", &groupInfo{ - base: &locatedObj{ - Obj: &model.Object{ - Name: "raw.txt", - Size: 9, - Modified: now, - Ctime: now, - }, - RemoteIndex: 0, - }, - partsByXact: map[string]map[int]chunkPart{}, - }) - if err != nil { - t.Fatalf("build raw listed object: %v", err) - } - if !ok { - t.Fatal("expected raw object") - } - rawObj := raw.(*Object) - if rawObj.Chunked || rawObj.GetSize() != 9 { - t.Fatalf("unexpected raw object: %+v", rawObj) - } -} - -func TestConfiguredRemotePaths(t *testing.T) { - d := &Chunker{ - Addition: Addition{ - RemotePath: "/s1/chunks", - RemotePaths: "\n/s2/chunks\n/s1/chunks\n /s3/chunks \n", - }, - } - got := d.configuredRemotePaths() - want := []string{"/s1/chunks", "/s2/chunks", "/s3/chunks"} - if len(got) != len(want) { - t.Fatalf("configuredRemotePaths length = %d, want %d (%v)", len(got), len(want), got) - } - for i := range want { - if got[i] != want[i] { - t.Fatalf("configuredRemotePaths[%d] = %q, want %q", i, got[i], want[i]) - } - } -} - -func TestBuildKeepSetSeparatesRemoteTargets(t *testing.T) { - d := newTestChunker(t) - keep := d.buildKeepSet( - objectLocation{LogicalPath: "/movie.bin", RemoteIndex: 0}, - objectLocation{LogicalPath: "/movie.bin", RemoteIndex: 1}, - ) - if len(keep) != 2 { - t.Fatalf("buildKeepSet should keep distinct entries per remote target, got %d", len(keep)) - } -} - -func TestChunkTargetIndexes(t *testing.T) { - d := newTestChunker(t) - d.remoteTargets = []remoteTarget{{}, {}, {}} - d.StoreChunksInPrimary = true - if got := d.chunkTargetIndexes(); len(got) != 3 || got[0] != 0 || got[1] != 1 || got[2] != 2 { - t.Fatalf("chunkTargetIndexes with primary = %v", got) - } - if got := d.chunkTargetIndex(4); got != 1 { - t.Fatalf("chunkTargetIndex with primary = %d, want 1", got) - } - - d.StoreChunksInPrimary = false - if got := d.chunkTargetIndexes(); len(got) != 2 || got[0] != 1 || got[1] != 2 { - t.Fatalf("chunkTargetIndexes without primary = %v", got) - } - if got := d.chunkTargetIndex(0); got != 1 { - t.Fatalf("chunkTargetIndex without primary for first chunk = %d, want 1", got) - } - if got := d.chunkTargetIndex(3); got != 2 { - t.Fatalf("chunkTargetIndex without primary for chunk 3 = %d, want 2", got) - } - - d.remoteTargets = []remoteTarget{{}} - if got := d.chunkTargetIndex(0); got != 0 { - t.Fatalf("single target chunkTargetIndex = %d, want 0", got) - } -} From bd725d547c52387068117515a3680312118c1642 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 27 Mar 2026 08:53:39 +0800 Subject: [PATCH 095/133] feat: add burn-after-read support for shares --- internal/db/share.go | 11 +++++++++++ internal/model/share.go | 2 ++ server/handles/share.go | 29 +++++++++++++++++++++++++++++ server/handles/share_public.go | 14 ++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/internal/db/share.go b/internal/db/share.go index 54eb3d5f2f9..de96e62d2ee 100644 --- a/internal/db/share.go +++ b/internal/db/share.go @@ -64,3 +64,14 @@ func TouchShareDownload(shareID string) error { "download_count": gorm.Expr("download_count + ?", 1), }).Error } + +func ConsumeShare(shareID string) error { + now := time.Now() + return db.Model(&model.Share{}). + Where("share_id = ? AND burn_after_read = ? AND consumed_at IS NULL", shareID, true). + Updates(map[string]interface{}{ + "enabled": false, + "consumed_at": now, + "last_access_at": now, + }).Error +} diff --git a/internal/model/share.go b/internal/model/share.go index a3fc90af008..4e7f5379484 100644 --- a/internal/model/share.go +++ b/internal/model/share.go @@ -11,12 +11,14 @@ type Share struct { IsDir bool `json:"is_dir"` PasswordHash string `json:"-" gorm:"size:64"` PasswordSalt string `json:"-" gorm:"size:32"` + BurnAfterRead bool `json:"burn_after_read" gorm:"default:false"` AllowPreview bool `json:"allow_preview" gorm:"default:true"` AllowDownload bool `json:"allow_download" gorm:"default:true"` Enabled bool `json:"enabled" gorm:"default:true;index"` ViewCount int64 `json:"view_count"` DownloadCount int64 `json:"download_count"` LastAccessAt *time.Time `json:"last_access_at"` + ConsumedAt *time.Time `json:"consumed_at"` ExpiresAt *time.Time `json:"expires_at"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` diff --git a/server/handles/share.go b/server/handles/share.go index 4626da8f0a2..f0a44bac664 100644 --- a/server/handles/share.go +++ b/server/handles/share.go @@ -26,6 +26,7 @@ type CreateShareReq struct { Name string `json:"name"` Password string `json:"password"` ExpireHours int64 `json:"expire_hours"` + BurnAfterRead *bool `json:"burn_after_read"` AllowPreview *bool `json:"allow_preview"` AllowDownload *bool `json:"allow_download"` } @@ -59,12 +60,14 @@ type ShareResp struct { RootPath string `json:"root_path"` IsDir bool `json:"is_dir"` HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` AllowPreview bool `json:"allow_preview"` AllowDownload bool `json:"allow_download"` Enabled bool `json:"enabled"` ViewCount int64 `json:"view_count"` DownloadCount int64 `json:"download_count"` LastAccessAt *time.Time `json:"last_access_at"` + ConsumedAt *time.Time `json:"consumed_at"` ExpiresAt *time.Time `json:"expires_at"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` @@ -76,9 +79,11 @@ type PublicShareInfoResp struct { Name string `json:"name"` IsDir bool `json:"is_dir"` HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` AllowPreview bool `json:"allow_preview"` AllowDownload bool `json:"allow_download"` Authed bool `json:"authed"` + ConsumedAt *time.Time `json:"consumed_at"` ExpiresAt *time.Time `json:"expires_at"` CreatedAt time.Time `json:"created_at"` } @@ -123,12 +128,14 @@ func toShareResp(c *gin.Context, share *model.Share) ShareResp { RootPath: share.RootPath, IsDir: share.IsDir, HasPassword: share.HasPassword(), + BurnAfterRead: share.BurnAfterRead, AllowPreview: share.AllowPreview, AllowDownload: share.AllowDownload, Enabled: share.Enabled, ViewCount: share.ViewCount, DownloadCount: share.DownloadCount, LastAccessAt: share.LastAccessAt, + ConsumedAt: share.ConsumedAt, ExpiresAt: share.ExpiresAt, CreatedAt: share.CreatedAt, UpdatedAt: share.UpdatedAt, @@ -177,6 +184,10 @@ func getShareAccessToken(c *gin.Context, fallback string) string { func ensureShareAvailable(c *gin.Context, share *model.Share) bool { now := time.Now() + if share.ConsumedAt != nil { + common.ErrorStrResp(c, "share has been consumed", 410) + return false + } if !share.Enabled { common.ErrorStrResp(c, "share is disabled", 404) return false @@ -188,6 +199,19 @@ func ensureShareAvailable(c *gin.Context, share *model.Share) bool { return true } +func consumeShareIfNeeded(share *model.Share) error { + if !share.BurnAfterRead || share.ConsumedAt != nil { + return nil + } + now := time.Now() + if err := db.ConsumeShare(share.ShareID); err != nil { + return err + } + share.Enabled = false + share.ConsumedAt = &now + return nil +} + func ensureShareAccess(c *gin.Context, share *model.Share, token string) bool { if !share.HasPassword() { return true @@ -303,6 +327,10 @@ func CreateShare(c *gin.Context) { if req.AllowDownload != nil { allowDownload = *req.AllowDownload } + burnAfterRead := false + if req.BurnAfterRead != nil { + burnAfterRead = *req.BurnAfterRead + } var expiresAt *time.Time if req.ExpireHours > 0 { expires := time.Now().Add(time.Duration(req.ExpireHours) * time.Hour) @@ -314,6 +342,7 @@ func CreateShare(c *gin.Context) { Name: normalizeShareName(obj, req.Name), RootPath: reqPath, IsDir: obj.IsDir(), + BurnAfterRead: burnAfterRead, AllowPreview: allowPreview, AllowDownload: allowDownload, Enabled: true, diff --git a/server/handles/share_public.go b/server/handles/share_public.go index 36476bae466..e1f7cf17012 100644 --- a/server/handles/share_public.go +++ b/server/handles/share_public.go @@ -40,9 +40,11 @@ func GetPublicShareInfo(c *gin.Context) { Name: share.Name, IsDir: share.IsDir, HasPassword: share.HasPassword(), + BurnAfterRead: share.BurnAfterRead, AllowPreview: share.AllowPreview, AllowDownload: share.AllowDownload, Authed: authed, + ConsumedAt: share.ConsumedAt, ExpiresAt: share.ExpiresAt, CreatedAt: share.CreatedAt, }) @@ -138,6 +140,10 @@ func ListPublicShare(c *gin.Context) { } content = append(content, toPublicShareObjResp(c, share, item, itemTargetPath, itemRelPath, token)) } + if err := consumeShareIfNeeded(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } common.SuccessResp(c, PublicShareListResp{ Content: content, Total: int64(total), @@ -219,6 +225,10 @@ func ShareDown(c *gin.Context) { return } _ = db.TouchShareDownload(share.ShareID) + if err := consumeShareIfNeeded(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } c.Set("path", targetPath) Down(c) } @@ -254,6 +264,10 @@ func ShareProxy(c *gin.Context) { common.ErrorStrResp(c, "directory preview is not supported", 400) return } + if err := consumeShareIfNeeded(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } c.Set("path", targetPath) Proxy(c) } From 4f60c3126138e5852467ea5f28dd2c2058133212 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 27 Mar 2026 18:56:10 +0800 Subject: [PATCH 096/133] feat: extend share lifecycle controls --- internal/db/share.go | 63 ++++++- internal/model/share.go | 31 ++- server/handles/share.go | 331 ++++++++++++++++++++++++++------- server/handles/share_public.go | 31 +-- server/router.go | 2 + 5 files changed, 369 insertions(+), 89 deletions(-) diff --git a/internal/db/share.go b/internal/db/share.go index de96e62d2ee..c92e546755e 100644 --- a/internal/db/share.go +++ b/internal/db/share.go @@ -5,6 +5,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "gorm.io/gorm" + "gorm.io/gorm/clause" ) func GetShareByShareID(shareID string) (*model.Share, error) { @@ -15,6 +16,14 @@ func GetShareByShareID(shareID string) (*model.Share, error) { return &share, nil } +func GetShareByCreatorAndShareID(creatorID uint, shareID string) (*model.Share, error) { + var share model.Share + if err := db.Where("creator_id = ? AND share_id = ?", creatorID, shareID).Take(&share).Error; err != nil { + return nil, err + } + return &share, nil +} + func ShareIDExists(shareID string) bool { var count int64 if err := db.Model(&model.Share{}).Where("share_id = ?", shareID).Count(&count).Error; err != nil { @@ -23,6 +32,14 @@ func ShareIDExists(shareID string) bool { return count > 0 } +func ShareIDExistsExceptID(shareID string, id uint) bool { + var count int64 + if err := db.Model(&model.Share{}).Where("share_id = ? AND id <> ?", shareID, id).Count(&count).Error; err != nil { + return false + } + return count > 0 +} + func CreateShare(share *model.Share) error { return db.Create(share).Error } @@ -45,6 +62,12 @@ func DeleteShareByShareID(creatorID uint, shareID string) error { return db.Where("creator_id = ? AND share_id = ?", creatorID, shareID).Delete(&model.Share{}).Error } +func DisableShareByShareID(creatorID uint, shareID string) error { + return db.Model(&model.Share{}). + Where("creator_id = ? AND share_id = ?", creatorID, shareID). + Update("enabled", false).Error +} + func TouchShareView(shareID string) error { now := time.Now() return db.Model(&model.Share{}). @@ -65,13 +88,37 @@ func TouchShareDownload(shareID string) error { }).Error } -func ConsumeShare(shareID string) error { - now := time.Now() - return db.Model(&model.Share{}). - Where("share_id = ? AND burn_after_read = ? AND consumed_at IS NULL", shareID, true). - Updates(map[string]interface{}{ - "enabled": false, - "consumed_at": now, +func RecordShareAccess(shareID string) (*model.Share, error) { + var updated model.Share + err := db.Transaction(func(tx *gorm.DB) error { + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). + Where("share_id = ?", shareID). + Take(&updated).Error; err != nil { + return err + } + + now := time.Now() + updated.AccessCount++ + updated.LastAccessAt = &now + updates := map[string]interface{}{ + "access_count": updated.AccessCount, "last_access_at": now, - }).Error + } + + limit := updated.EffectiveAccessLimit() + if limit > 0 && updated.AccessCount >= limit { + updated.Enabled = false + updated.ConsumedAt = &now + updates["enabled"] = false + updates["consumed_at"] = now + } + + return tx.Model(&model.Share{}). + Where("id = ?", updated.ID). + Updates(updates).Error + }) + if err != nil { + return nil, err + } + return &updated, nil } diff --git a/internal/model/share.go b/internal/model/share.go index 4e7f5379484..30fbb8cb7d4 100644 --- a/internal/model/share.go +++ b/internal/model/share.go @@ -12,6 +12,8 @@ type Share struct { PasswordHash string `json:"-" gorm:"size:64"` PasswordSalt string `json:"-" gorm:"size:32"` BurnAfterRead bool `json:"burn_after_read" gorm:"default:false"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` AllowPreview bool `json:"allow_preview" gorm:"default:true"` AllowDownload bool `json:"allow_download" gorm:"default:true"` Enabled bool `json:"enabled" gorm:"default:true;index"` @@ -28,6 +30,33 @@ func (s Share) HasPassword() bool { return s.PasswordHash != "" } +func (s Share) EffectiveAccessLimit() int64 { + if s.AccessLimit > 0 { + return s.AccessLimit + } + if s.BurnAfterRead { + return 1 + } + return 0 +} + +func (s Share) RemainingAccesses() int64 { + limit := s.EffectiveAccessLimit() + if limit <= 0 { + return 0 + } + remaining := limit - s.AccessCount + if remaining < 0 { + return 0 + } + return remaining +} + +func (s Share) IsConsumed() bool { + limit := s.EffectiveAccessLimit() + return s.ConsumedAt != nil || (limit > 0 && s.AccessCount >= limit) +} + func (s Share) IsExpired(now time.Time) bool { - return s.ExpiresAt != nil && s.ExpiresAt.Before(now) + return s.ExpiresAt != nil && !s.ExpiresAt.After(now) } diff --git a/server/handles/share.go b/server/handles/share.go index f0a44bac664..99a693b7a5f 100644 --- a/server/handles/share.go +++ b/server/handles/share.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" stdpath "path" + "regexp" "strings" "time" @@ -21,16 +22,32 @@ import ( const shareAccessTokenLifetime = 24 * time.Hour +var shareIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,32}$`) + type CreateShareReq struct { Path string `json:"path" binding:"required"` + ShareID string `json:"share_id"` Name string `json:"name"` Password string `json:"password"` + ExpireAt string `json:"expire_at"` ExpireHours int64 `json:"expire_hours"` + AccessLimit int64 `json:"access_limit"` BurnAfterRead *bool `json:"burn_after_read"` AllowPreview *bool `json:"allow_preview"` AllowDownload *bool `json:"allow_download"` } +type UpdateShareReq struct { + ShareID string `json:"share_id" binding:"required"` + NewShareID string `json:"new_share_id"` + Name string `json:"name"` + Password string `json:"password"` + ExpireAt *string `json:"expire_at"` + AccessLimit *int64 `json:"access_limit"` + AllowPreview *bool `json:"allow_preview"` + AllowDownload *bool `json:"allow_download"` +} + type ShareDeleteReq struct { ShareID string `json:"share_id" binding:"required"` } @@ -54,38 +71,44 @@ type PublicShareListReq struct { } type ShareResp struct { - ID uint `json:"id"` - ShareID string `json:"share_id"` - Name string `json:"name"` - RootPath string `json:"root_path"` - IsDir bool `json:"is_dir"` - HasPassword bool `json:"has_password"` - BurnAfterRead bool `json:"burn_after_read"` - AllowPreview bool `json:"allow_preview"` - AllowDownload bool `json:"allow_download"` - Enabled bool `json:"enabled"` - ViewCount int64 `json:"view_count"` - DownloadCount int64 `json:"download_count"` - LastAccessAt *time.Time `json:"last_access_at"` - ConsumedAt *time.Time `json:"consumed_at"` - ExpiresAt *time.Time `json:"expires_at"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - URL string `json:"url"` + ID uint `json:"id"` + ShareID string `json:"share_id"` + Name string `json:"name"` + RootPath string `json:"root_path"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` + RemainingAccesses int64 `json:"remaining_accesses"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Enabled bool `json:"enabled"` + ViewCount int64 `json:"view_count"` + DownloadCount int64 `json:"download_count"` + LastAccessAt *time.Time `json:"last_access_at"` + ConsumedAt *time.Time `json:"consumed_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + URL string `json:"url"` } type PublicShareInfoResp struct { - ShareID string `json:"share_id"` - Name string `json:"name"` - IsDir bool `json:"is_dir"` - HasPassword bool `json:"has_password"` - BurnAfterRead bool `json:"burn_after_read"` - AllowPreview bool `json:"allow_preview"` - AllowDownload bool `json:"allow_download"` - Authed bool `json:"authed"` - ConsumedAt *time.Time `json:"consumed_at"` - ExpiresAt *time.Time `json:"expires_at"` - CreatedAt time.Time `json:"created_at"` + ShareID string `json:"share_id"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` + RemainingAccesses int64 `json:"remaining_accesses"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Authed bool `json:"authed"` + ConsumedAt *time.Time `json:"consumed_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` } type PublicShareObjResp struct { @@ -121,33 +144,41 @@ func shareURL(c *gin.Context, shareID string) string { } func toShareResp(c *gin.Context, share *model.Share) ShareResp { + accessLimit := share.EffectiveAccessLimit() return ShareResp{ - ID: share.ID, - ShareID: share.ShareID, - Name: share.Name, - RootPath: share.RootPath, - IsDir: share.IsDir, - HasPassword: share.HasPassword(), - BurnAfterRead: share.BurnAfterRead, - AllowPreview: share.AllowPreview, - AllowDownload: share.AllowDownload, - Enabled: share.Enabled, - ViewCount: share.ViewCount, - DownloadCount: share.DownloadCount, - LastAccessAt: share.LastAccessAt, - ConsumedAt: share.ConsumedAt, - ExpiresAt: share.ExpiresAt, - CreatedAt: share.CreatedAt, - UpdatedAt: share.UpdatedAt, - URL: shareURL(c, share.ShareID), + ID: share.ID, + ShareID: share.ShareID, + Name: share.Name, + RootPath: share.RootPath, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + BurnAfterRead: accessLimit == 1, + AccessLimit: accessLimit, + AccessCount: share.AccessCount, + RemainingAccesses: share.RemainingAccesses(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Enabled: share.Enabled, + ViewCount: share.ViewCount, + DownloadCount: share.DownloadCount, + LastAccessAt: share.LastAccessAt, + ConsumedAt: share.ConsumedAt, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, + UpdatedAt: share.UpdatedAt, + URL: shareURL(c, share.ShareID), } } func normalizeShareName(obj model.Obj, name string) string { + return normalizeOptionalShareName(name, obj.GetName()) +} + +func normalizeOptionalShareName(name, fallback string) string { if strings.TrimSpace(name) != "" { return strings.TrimSpace(name) } - return obj.GetName() + return fallback } func generateShareID() (string, error) { @@ -164,6 +195,84 @@ func sharePasswordHash(password, salt string) string { return model.HashPwd(model.StaticHash(password), salt) } +func validateCustomShareID(shareID string) error { + if shareID == "" { + return nil + } + if !shareIDPattern.MatchString(shareID) { + return fmt.Errorf("share_id must be 1-32 characters of letters, numbers, underscore or hyphen") + } + return nil +} + +func resolveRequestedShareID(rawShareID, fallback string, excludeID uint) (string, error) { + shareID := strings.TrimSpace(rawShareID) + if shareID == "" { + if fallback != "" { + return fallback, nil + } + return generateShareID() + } + if err := validateCustomShareID(shareID); err != nil { + return "", err + } + if excludeID == 0 { + if db.ShareIDExists(shareID) { + return "", fmt.Errorf("share link already exists") + } + return shareID, nil + } + if db.ShareIDExistsExceptID(shareID, excludeID) { + return "", fmt.Errorf("share link already exists") + } + return shareID, nil +} + +func normalizeShareAccessLimit(accessLimit int64, burnAfterRead *bool) (int64, bool, error) { + if accessLimit < 0 { + return 0, false, fmt.Errorf("access_limit must be 0 or greater") + } + if accessLimit == 0 && burnAfterRead != nil && *burnAfterRead { + accessLimit = 1 + } + return accessLimit, accessLimit == 1, nil +} + +func parseShareExpireAt(raw string) (*time.Time, error) { + value := strings.TrimSpace(raw) + if value == "" { + return nil, nil + } + if parsed, err := time.Parse(time.RFC3339, value); err == nil { + return &parsed, nil + } + layouts := []string{ + "2006-01-02T15:04:05", + "2006-01-02T15:04", + "2006-01-02 15:04:05", + } + for _, layout := range layouts { + if parsed, err := time.ParseInLocation(layout, value, time.Local); err == nil { + return &parsed, nil + } + } + return nil, fmt.Errorf("invalid expire_at") +} + +func resolveShareExpireAt(expireAt string, expireHours int64) (*time.Time, error) { + if strings.TrimSpace(expireAt) != "" { + return parseShareExpireAt(expireAt) + } + if expireHours < 0 { + return nil, fmt.Errorf("expire_hours must be 0 or greater") + } + if expireHours == 0 { + return nil, nil + } + expires := time.Now().Add(time.Duration(expireHours) * time.Hour) + return &expires, nil +} + func sharePasswordMatched(share *model.Share, password string) bool { if !share.HasPassword() { return true @@ -184,7 +293,7 @@ func getShareAccessToken(c *gin.Context, fallback string) string { func ensureShareAvailable(c *gin.Context, share *model.Share) bool { now := time.Now() - if share.ConsumedAt != nil { + if share.IsConsumed() { common.ErrorStrResp(c, "share has been consumed", 410) return false } @@ -199,16 +308,14 @@ func ensureShareAvailable(c *gin.Context, share *model.Share) bool { return true } -func consumeShareIfNeeded(share *model.Share) error { - if !share.BurnAfterRead || share.ConsumedAt != nil { - return nil - } - now := time.Now() - if err := db.ConsumeShare(share.ShareID); err != nil { +func recordShareAccess(share *model.Share) error { + updated, err := db.RecordShareAccess(share.ShareID) + if err != nil { return err } - share.Enabled = false - share.ConsumedAt = &now + if updated != nil { + *share = *updated + } return nil } @@ -314,9 +421,9 @@ func CreateShare(c *gin.Context) { common.ErrorResp(c, err, 500) return } - shareID, err := generateShareID() + shareID, err := resolveRequestedShareID(req.ShareID, "", 0) if err != nil { - common.ErrorResp(c, err, 500, true) + common.ErrorResp(c, err, 400) return } allowPreview := true @@ -327,14 +434,15 @@ func CreateShare(c *gin.Context) { if req.AllowDownload != nil { allowDownload = *req.AllowDownload } - burnAfterRead := false - if req.BurnAfterRead != nil { - burnAfterRead = *req.BurnAfterRead + accessLimit, burnAfterRead, err := normalizeShareAccessLimit(req.AccessLimit, req.BurnAfterRead) + if err != nil { + common.ErrorResp(c, err, 400) + return } - var expiresAt *time.Time - if req.ExpireHours > 0 { - expires := time.Now().Add(time.Duration(req.ExpireHours) * time.Hour) - expiresAt = &expires + expiresAt, err := resolveShareExpireAt(req.ExpireAt, req.ExpireHours) + if err != nil { + common.ErrorResp(c, err, 400) + return } share := &model.Share{ ShareID: shareID, @@ -343,6 +451,7 @@ func CreateShare(c *gin.Context) { RootPath: reqPath, IsDir: obj.IsDir(), BurnAfterRead: burnAfterRead, + AccessLimit: accessLimit, AllowPreview: allowPreview, AllowDownload: allowDownload, Enabled: true, @@ -359,6 +468,78 @@ func CreateShare(c *gin.Context) { common.SuccessResp(c, toShareResp(c, share)) } +func UpdateShare(c *gin.Context) { + var req UpdateShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + share, err := db.GetShareByCreatorAndShareID(user.ID, req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + + shareID, err := resolveRequestedShareID(req.NewShareID, share.ShareID, share.ID) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + + allowPreview := share.AllowPreview + if req.AllowPreview != nil { + allowPreview = *req.AllowPreview + } + allowDownload := share.AllowDownload + if req.AllowDownload != nil { + allowDownload = *req.AllowDownload + } + + accessLimit := share.EffectiveAccessLimit() + burnAfterRead := accessLimit == 1 + if req.AccessLimit != nil { + accessLimit, burnAfterRead, err = normalizeShareAccessLimit(*req.AccessLimit, nil) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + } + + expiresAt := share.ExpiresAt + if req.ExpireAt != nil { + expiresAt, err = parseShareExpireAt(*req.ExpireAt) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + } + + share.ShareID = shareID + share.Name = normalizeOptionalShareName(req.Name, share.Name) + share.BurnAfterRead = burnAfterRead + share.AccessLimit = accessLimit + share.AllowPreview = allowPreview + share.AllowDownload = allowDownload + share.ExpiresAt = expiresAt + if req.Password != "" { + share.PasswordSalt = random.String(16) + share.PasswordHash = sharePasswordHash(req.Password, share.PasswordSalt) + } + if share.Enabled && accessLimit > 0 && share.AccessCount >= accessLimit { + now := time.Now() + share.Enabled = false + if share.ConsumedAt == nil { + share.ConsumedAt = &now + } + } + if err := db.UpdateShare(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, toShareResp(c, share)) +} + func ListShares(c *gin.Context) { var req model.PageReq if err := c.ShouldBind(&req); err != nil { @@ -382,6 +563,24 @@ func ListShares(c *gin.Context) { }) } +func DisableShare(c *gin.Context) { + var req ShareDeleteReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if _, err := db.GetShareByCreatorAndShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 404) + return + } + if err := db.DisableShareByShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + func DeleteShare(c *gin.Context) { var req ShareDeleteReq if err := c.ShouldBind(&req); err != nil { diff --git a/server/handles/share_public.go b/server/handles/share_public.go index e1f7cf17012..22aba45bc4c 100644 --- a/server/handles/share_public.go +++ b/server/handles/share_public.go @@ -36,17 +36,20 @@ func GetPublicShareInfo(c *gin.Context) { _ = db.TouchShareView(share.ShareID) } common.SuccessResp(c, PublicShareInfoResp{ - ShareID: share.ShareID, - Name: share.Name, - IsDir: share.IsDir, - HasPassword: share.HasPassword(), - BurnAfterRead: share.BurnAfterRead, - AllowPreview: share.AllowPreview, - AllowDownload: share.AllowDownload, - Authed: authed, - ConsumedAt: share.ConsumedAt, - ExpiresAt: share.ExpiresAt, - CreatedAt: share.CreatedAt, + ShareID: share.ShareID, + Name: share.Name, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + BurnAfterRead: share.EffectiveAccessLimit() == 1, + AccessLimit: share.EffectiveAccessLimit(), + AccessCount: share.AccessCount, + RemainingAccesses: share.RemainingAccesses(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Authed: authed, + ConsumedAt: share.ConsumedAt, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, }) } @@ -140,7 +143,7 @@ func ListPublicShare(c *gin.Context) { } content = append(content, toPublicShareObjResp(c, share, item, itemTargetPath, itemRelPath, token)) } - if err := consumeShareIfNeeded(share); err != nil { + if err := recordShareAccess(share); err != nil { common.ErrorResp(c, err, 500, true) return } @@ -225,7 +228,7 @@ func ShareDown(c *gin.Context) { return } _ = db.TouchShareDownload(share.ShareID) - if err := consumeShareIfNeeded(share); err != nil { + if err := recordShareAccess(share); err != nil { common.ErrorResp(c, err, 500, true) return } @@ -264,7 +267,7 @@ func ShareProxy(c *gin.Context) { common.ErrorStrResp(c, "directory preview is not supported", 400) return } - if err := consumeShareIfNeeded(share); err != nil { + if err := recordShareAccess(share); err != nil { common.ErrorResp(c, err, 500, true) return } diff --git a/server/router.go b/server/router.go index e63b667297e..34df40096b4 100644 --- a/server/router.go +++ b/server/router.go @@ -111,6 +111,8 @@ func Init(e *gin.Engine) { _fs(auth.Group("/fs")) share := auth.Group("/share", middlewares.AuthNotGuest) share.POST("/create", handles.CreateShare) + share.POST("/update", handles.UpdateShare) + share.POST("/disable", handles.DisableShare) share.GET("/list", handles.ListShares) share.POST("/delete", handles.DeleteShare) _task(auth.Group("/task", middlewares.AuthNotGuest)) From fe0c5b8dc52de5ad6e67a3301478f6ddd219ffbd Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:04:29 +0800 Subject: [PATCH 097/133] feat(driver/streamtape): add response types for remote download, file info, and conversion status --- drivers/streamtape/types.go | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/drivers/streamtape/types.go b/drivers/streamtape/types.go index 4e470ff927e..347e89d8a67 100644 --- a/drivers/streamtape/types.go +++ b/drivers/streamtape/types.go @@ -52,3 +52,46 @@ type createFolderResult struct { type uploadURLResult struct { URL string `json:"url"` } + +type remoteDlAddResult struct { + ID string `json:"id"` + FolderID string `json:"folderid"` +} + +type remoteDlStatusResult map[string]remoteDlStatusItem + +type remoteDlStatusItem struct { + ID string `json:"id"` + RemoteURL string `json:"remoteurl"` + Status string `json:"status"` + BytesLoaded interface{} `json:"bytes_loaded"` + BytesTotal interface{} `json:"bytes_total"` + FolderID string `json:"folderid"` + Added string `json:"added"` + LastUpdate string `json:"last_update"` + ExtID bool `json:"extid"` + URL bool `json:"url"` +} + +type fileInfoResult map[string]fileInfoItem + +type fileInfoItem struct { + ID string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + Type string `json:"type"` + Converted bool `json:"converted"` + Status int `json:"status"` +} + +type conversionResult []conversionItem + +type conversionItem struct { + Name string `json:"name"` + FolderID string `json:"folderid"` + Status string `json:"status"` + Progress int `json:"progress"` + Retries int `json:"retries"` + Link string `json:"link"` + LinkID string `json:"linkid"` +} From 11254ddba154b135a530423d8bd98331a6210deb Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:06:12 +0800 Subject: [PATCH 098/133] feat(driver/streamtape): add remote upload ID helpers --- drivers/streamtape/util.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/streamtape/util.go b/drivers/streamtape/util.go index f6d8373763e..51ad73d59bb 100644 --- a/drivers/streamtape/util.go +++ b/drivers/streamtape/util.go @@ -144,3 +144,20 @@ func extractFileIDFromUploadBody(body []byte) string { } return "" } + +const remoteUploadPrefix = "ru:" + +func encodeRemoteUploadID(id string) string { + return remoteUploadPrefix + id +} + +func remoteUploadIDFromObjID(id string) string { + if strings.HasPrefix(id, remoteUploadPrefix) { + return strings.TrimPrefix(id, remoteUploadPrefix) + } + return "" +} + +func isRemoteUploadID(id string) bool { + return strings.HasPrefix(id, remoteUploadPrefix) +} From d59c58916cf6529f4a8d5785b426f079e49acdd4 Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:08:26 +0800 Subject: [PATCH 099/133] feat(driver/streamtape): implement PutURL for remote uploads --- drivers/streamtape/driver.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go index 042df42cbe6..7eb9d6c9e33 100644 --- a/drivers/streamtape/driver.go +++ b/drivers/streamtape/driver.go @@ -381,6 +381,35 @@ func (d *Streamtape) Put(ctx context.Context, dstDir model.Obj, file model.FileS }, nil } +// PutURL initiates a remote upload from an external URL +func (d *Streamtape) PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + params := map[string]string{ + "url": url, + } + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + if name != "" { + params["name"] = name + } + + var result remoteDlAddResult + if err := d.callAPI(ctx, "/remotedl/add", params, &result); err != nil { + return nil, err + } + + return &model.Object{ + ID: encodeRemoteUploadID(result.ID), + Name: name, + IsFolder: false, + }, nil +} + func (d *Streamtape) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { return nil, errs.NotImplement } From 6761fe7a1ac69b21bfcac54c7385b4ac96426987 Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:11:35 +0800 Subject: [PATCH 100/133] feat(driver/streamtape): implement Other method with remotedl_status, remotedl_remove, file_info, thumbnail, conversion_status --- drivers/streamtape/driver.go | 112 +++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go index 7eb9d6c9e33..b261f86100e 100644 --- a/drivers/streamtape/driver.go +++ b/drivers/streamtape/driver.go @@ -426,4 +426,116 @@ func (d *Streamtape) ArchiveDecompress(ctx context.Context, srcObj, dstDir model return nil, errs.NotImplement } +func (d *Streamtape) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch strings.ToLower(args.Method) { + case "remotedl_status": + return d.remoteDlStatus(ctx, args) + case "remotedl_remove": + return d.remoteDlRemove(ctx, args) + case "file_info": + return d.fileInfo(ctx, args) + case "thumbnail": + return d.thumbnail(ctx, args) + case "conversion_status": + return d.conversionStatus(ctx, args) + default: + return nil, errs.NotSupport + } +} + +func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + uploadID := remoteUploadIDFromObjID(args.Obj.GetID()) + if uploadID == "" { + // Try to get from data + if data, ok := args.Data.(map[string]interface{}); ok { + if id, ok := data["id"].(string); ok { + uploadID = id + } + } + } + if uploadID == "" { + return nil, fmt.Errorf("remote upload ID required") + } + + var result remoteDlStatusResult + if err := d.callAPI(ctx, "/remotedl/status", map[string]string{"id": uploadID}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) remoteDlRemove(ctx context.Context, args model.OtherArgs) (interface{}, error) { + uploadID := remoteUploadIDFromObjID(args.Obj.GetID()) + if uploadID == "" { + // Try to get from data + if data, ok := args.Data.(map[string]interface{}); ok { + if id, ok := data["id"].(string); ok { + uploadID = id + } + } + } + if uploadID == "" { + return nil, fmt.Errorf("remote upload ID required") + } + + if err := d.callAPI(ctx, "/remotedl/remove", map[string]string{"id": uploadID}, nil); err != nil { + return nil, err + } + return true, nil +} + +func (d *Streamtape) fileInfo(ctx context.Context, args model.OtherArgs) (interface{}, error) { + var fileIDs string + if data, ok := args.Data.(map[string]interface{}); ok { + if ids, ok := data["file_ids"].(string); ok { + fileIDs = ids + } + } + if fileIDs == "" { + fileIDs = fileIDFromObjID(args.Obj.GetID()) + } + if fileIDs == "" { + return nil, fmt.Errorf("file IDs required") + } + + var result fileInfoResult + if err := d.callAPI(ctx, "/file/info", map[string]string{"file": fileIDs}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) thumbnail(ctx context.Context, args model.OtherArgs) (interface{}, error) { + fileID := fileIDFromObjID(args.Obj.GetID()) + if fileID == "" { + return nil, fmt.Errorf("file ID required") + } + + var result string + if err := d.callAPI(ctx, "/file/getsplash", map[string]string{"file": fileID}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) conversionStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + isFailed := false + if data, ok := args.Data.(map[string]interface{}); ok { + if t, ok := data["type"].(string); ok && t == "failed" { + isFailed = true + } + } + + endpoint := "/file/runningconverts" + if isFailed { + endpoint = "/file/failedconverts" + } + + var result conversionResult + if err := d.callAPI(ctx, endpoint, nil, &result); err != nil { + return nil, err + } + return result, nil +} + var _ driver.Driver = (*Streamtape)(nil) From 24aaf0ce2c5642b1e5154e429ad68bc50418e3c5 Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:13:59 +0800 Subject: [PATCH 101/133] refactor(driver/streamtape): extract remote upload ID helper to reduce duplication --- drivers/streamtape/driver.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go index b261f86100e..f632d22f31f 100644 --- a/drivers/streamtape/driver.go +++ b/drivers/streamtape/driver.go @@ -443,10 +443,9 @@ func (d *Streamtape) Other(ctx context.Context, args model.OtherArgs) (interface } } -func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { +func (d *Streamtape) extractRemoteUploadID(args model.OtherArgs) (string, error) { uploadID := remoteUploadIDFromObjID(args.Obj.GetID()) if uploadID == "" { - // Try to get from data if data, ok := args.Data.(map[string]interface{}); ok { if id, ok := data["id"].(string); ok { uploadID = id @@ -454,7 +453,15 @@ func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) ( } } if uploadID == "" { - return nil, fmt.Errorf("remote upload ID required") + return "", fmt.Errorf("remote upload ID required") + } + return uploadID, nil +} + +func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + uploadID, err := d.extractRemoteUploadID(args) + if err != nil { + return nil, err } var result remoteDlStatusResult @@ -465,17 +472,9 @@ func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) ( } func (d *Streamtape) remoteDlRemove(ctx context.Context, args model.OtherArgs) (interface{}, error) { - uploadID := remoteUploadIDFromObjID(args.Obj.GetID()) - if uploadID == "" { - // Try to get from data - if data, ok := args.Data.(map[string]interface{}); ok { - if id, ok := data["id"].(string); ok { - uploadID = id - } - } - } - if uploadID == "" { - return nil, fmt.Errorf("remote upload ID required") + uploadID, err := d.extractRemoteUploadID(args) + if err != nil { + return nil, err } if err := d.callAPI(ctx, "/remotedl/remove", map[string]string{"id": uploadID}, nil); err != nil { From 066386fac4cb53d75e0513d79f7dbf03aec39fdd Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:15:40 +0800 Subject: [PATCH 102/133] feat(driver/streamtape): add Sha256 upload support and move-to-root alert --- drivers/streamtape/driver.go | 3 +++ drivers/streamtape/meta.go | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go index f632d22f31f..9ce32e25b69 100644 --- a/drivers/streamtape/driver.go +++ b/drivers/streamtape/driver.go @@ -331,6 +331,9 @@ func (d *Streamtape) Put(ctx context.Context, dstDir model.Obj, file model.FileS if folderID != "" && folderID != "0" { params["folder"] = folderID } + if d.Sha256 != "" { + params["sha256"] = d.Sha256 + } var uploadURL uploadURLResult if err := d.callAPI(ctx, "/file/ul", params, &uploadURL); err != nil { diff --git a/drivers/streamtape/meta.go b/drivers/streamtape/meta.go index 50b09f14d41..50dd90291ef 100644 --- a/drivers/streamtape/meta.go +++ b/drivers/streamtape/meta.go @@ -14,6 +14,7 @@ type Addition struct { RangeConcurrency int `json:"range_concurrency" type:"number" default:"4" help:"Chunk mode concurrent upstream requests"` RangePercent int `json:"range_percent" type:"number" default:"15" help:"Percent mode part size percentage (1-100)"` EnableRangeControl bool `json:"enable_range_control" default:"true" help:"Enable driver-level range shaping for smoother streaming"` + Sha256 string `json:"sha256" help:"Expected SHA256 hash for upload verification (optional)"` } var config = driver.Config{ @@ -26,7 +27,7 @@ var config = driver.Config{ NeedMs: false, DefaultRoot: "0", CheckStatus: false, - Alert: "", + Alert: "Moving files to root folder is not supported by Streamtape API", NoOverwriteUpload: false, ProxyRangeOption: true, } From 9dd6823391080e9b67c69c084d3089fc636e49bf Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:17:32 +0800 Subject: [PATCH 103/133] fix(driver/streamtape): add warning type prefix to Alert message --- drivers/streamtape/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/streamtape/meta.go b/drivers/streamtape/meta.go index 50dd90291ef..55e97894dbe 100644 --- a/drivers/streamtape/meta.go +++ b/drivers/streamtape/meta.go @@ -27,7 +27,7 @@ var config = driver.Config{ NeedMs: false, DefaultRoot: "0", CheckStatus: false, - Alert: "Moving files to root folder is not supported by Streamtape API", + Alert: "warning|Moving files to root folder is not supported by Streamtape API", NoOverwriteUpload: false, ProxyRangeOption: true, } From e3f3fc40f026186cc91b83c11ac23f0d2627d6a8 Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Sat, 28 Mar 2026 13:57:40 +0800 Subject: [PATCH 104/133] test(streamtape): add comprehensive unit tests for Streamtape driver - Initialize RestyClient with custom settings for testing - Add TestDriverList to verify listing folders and files - Add TestDriverPutURL to test remote upload functionality and status checks - Add TestDriverFileInfo to validate file_info method via Other call - Add TestDriverThumbnail to test thumbnail retrieval via Other call - Add TestDriverConversionStatus to test conversion status queries (running and failed) --- drivers/streamtape/api_test.go | 224 +++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 drivers/streamtape/api_test.go diff --git a/drivers/streamtape/api_test.go b/drivers/streamtape/api_test.go new file mode 100644 index 00000000000..d12fc1320ab --- /dev/null +++ b/drivers/streamtape/api_test.go @@ -0,0 +1,224 @@ +package streamtape + +import ( + "context" + "crypto/tls" + "testing" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" +) + +const ( + testLogin = "" + testKey = "" +) + +func init() { + // Initialize RestyClient for testing + base.RestyClient = resty.New(). + SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"). + SetRetryCount(3). + SetRetryResetReaders(true). + SetTimeout(30 * time.Second). + SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) +} + +func newTestDriver() *Streamtape { + return &Streamtape{ + Addition: Addition{ + APILogin: testLogin, + APIKey: testKey, + }, + } +} + +// TestDriverList tests List method +func TestDriverList(t *testing.T) { + d := newTestDriver() + d.RootID.RootFolderID = "0" + + ctx := context.Background() + objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("List failed: %v", err) + } + t.Logf("List returned %d objects", len(objs)) + for _, obj := range objs { + t.Logf(" - %s (folder=%v)", obj.GetName(), obj.IsDir()) + } +} + +// TestDriverPutURL tests PutURL method - remote upload +func TestDriverPutURL(t *testing.T) { + d := newTestDriver() + d.RootID.RootFolderID = "0" + + t.Logf("Driver initialized: APILogin=%s, APIKey=%s, RootFolderID=%s", + d.APILogin, d.APIKey, d.RootID.RootFolderID) + + ctx := context.Background() + // Pass a valid root directory object instead of nil + rootDir := &model.Object{ID: "d:0", IsFolder: true} + obj, err := d.PutURL(ctx, rootDir, "test.txt", "https://example.com/test.txt") + if err != nil { + t.Fatalf("PutURL failed: %v", err) + } + t.Logf("PutURL returned: id=%s name=%s", obj.GetID(), obj.GetName()) + + // Extract upload ID + uploadID := remoteUploadIDFromObjID(obj.GetID()) + if uploadID == "" { + t.Fatal("PutURL returned invalid ID format") + } + t.Logf("Upload ID: %s", uploadID) + + // Pass Obj with the upload ID so extractRemoteUploadID can work + uploadObj := &model.Object{ID: obj.GetID()} + + // Test remotedl_status via Other method + statusResult, err := d.Other(ctx, model.OtherArgs{ + Obj: uploadObj, + Method: "remotedl_status", + Data: map[string]interface{}{"id": uploadID}, + }) + if err != nil { + t.Fatalf("remotedl_status failed: %v", err) + } + t.Logf("remotedl_status: %+v", statusResult) + + // Test remotedl_remove via Other method + removeResult, err := d.Other(ctx, model.OtherArgs{ + Obj: uploadObj, + Method: "remotedl_remove", + Data: map[string]interface{}{"id": uploadID}, + }) + if err != nil { + t.Fatalf("remotedl_remove failed: %v", err) + } + t.Logf("remotedl_remove: %v", removeResult) +} + +// TestDriverFileInfo tests file_info via Other method +func TestDriverFileInfo(t *testing.T) { + d := newTestDriver() + + // First get a file ID from list + ctx := context.Background() + objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("List failed: %v", err) + } + + // Find a file in subfolders + var fileID string + var fileName string + for _, obj := range objs { + if obj.IsDir() { + subObjs, err := d.List(ctx, obj, model.ListArgs{}) + if err != nil { + continue + } + for _, subObj := range subObjs { + if !subObj.IsDir() { + fileID = subObj.GetID() + fileName = subObj.GetName() + break + } + } + if fileID != "" { + break + } + } + } + + if fileID == "" { + t.Skip("No files found for file_info test") + } + t.Logf("Testing file_info with file: %s (id=%s)", fileName, fileID) + + // Test file_info via Other method + infoResult, err := d.Other(ctx, model.OtherArgs{ + Method: "file_info", + Obj: &model.Object{ID: fileID}, + }) + if err != nil { + t.Fatalf("file_info failed: %v", err) + } + t.Logf("file_info: %+v", infoResult) +} + +// TestDriverThumbnail tests thumbnail via Other method +func TestDriverThumbnail(t *testing.T) { + d := newTestDriver() + + // First get a file ID from list + ctx := context.Background() + objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("List failed: %v", err) + } + + // Find a file in subfolders + var fileID string + for _, obj := range objs { + if obj.IsDir() { + subObjs, err := d.List(ctx, obj, model.ListArgs{}) + if err != nil { + continue + } + for _, subObj := range subObjs { + if !subObj.IsDir() { + fileID = subObj.GetID() + break + } + } + if fileID != "" { + break + } + } + } + + if fileID == "" { + t.Skip("No files found for thumbnail test") + } + t.Logf("Testing thumbnail with file id=%s", fileID) + + // Test thumbnail via Other method + thumbResult, err := d.Other(ctx, model.OtherArgs{ + Method: "thumbnail", + Obj: &model.Object{ID: fileID}, + }) + if err != nil { + t.Fatalf("thumbnail failed: %v", err) + } + t.Logf("thumbnail: %v", thumbResult) +} + +// TestDriverConversionStatus tests conversion_status via Other method +func TestDriverConversionStatus(t *testing.T) { + d := newTestDriver() + + ctx := context.Background() + + // Test running conversions + runningResult, err := d.Other(ctx, model.OtherArgs{ + Method: "conversion_status", + }) + if err != nil { + t.Fatalf("conversion_status (running) failed: %v", err) + } + t.Logf("conversion_status (running): %+v", runningResult) + + // Test failed conversions + failedResult, err := d.Other(ctx, model.OtherArgs{ + Method: "conversion_status", + Data: map[string]interface{}{"type": "failed"}, + }) + if err != nil { + t.Fatalf("conversion_status (failed) failed: %v", err) + } + t.Logf("conversion_status (failed): %+v", failedResult) +} \ No newline at end of file From 1d05bdb0d575d845eb1096ad8a5aa353f6628a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sat, 28 Mar 2026 14:45:30 +0800 Subject: [PATCH 105/133] chore(streamtape): delete drivers/streamtape/api_test.go --- drivers/streamtape/api_test.go | 224 --------------------------------- 1 file changed, 224 deletions(-) delete mode 100644 drivers/streamtape/api_test.go diff --git a/drivers/streamtape/api_test.go b/drivers/streamtape/api_test.go deleted file mode 100644 index d12fc1320ab..00000000000 --- a/drivers/streamtape/api_test.go +++ /dev/null @@ -1,224 +0,0 @@ -package streamtape - -import ( - "context" - "crypto/tls" - "testing" - "time" - - "github.com/alist-org/alist/v3/drivers/base" - "github.com/alist-org/alist/v3/internal/model" - "github.com/go-resty/resty/v2" -) - -const ( - testLogin = "" - testKey = "" -) - -func init() { - // Initialize RestyClient for testing - base.RestyClient = resty.New(). - SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"). - SetRetryCount(3). - SetRetryResetReaders(true). - SetTimeout(30 * time.Second). - SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) -} - -func newTestDriver() *Streamtape { - return &Streamtape{ - Addition: Addition{ - APILogin: testLogin, - APIKey: testKey, - }, - } -} - -// TestDriverList tests List method -func TestDriverList(t *testing.T) { - d := newTestDriver() - d.RootID.RootFolderID = "0" - - ctx := context.Background() - objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) - if err != nil { - t.Fatalf("List failed: %v", err) - } - t.Logf("List returned %d objects", len(objs)) - for _, obj := range objs { - t.Logf(" - %s (folder=%v)", obj.GetName(), obj.IsDir()) - } -} - -// TestDriverPutURL tests PutURL method - remote upload -func TestDriverPutURL(t *testing.T) { - d := newTestDriver() - d.RootID.RootFolderID = "0" - - t.Logf("Driver initialized: APILogin=%s, APIKey=%s, RootFolderID=%s", - d.APILogin, d.APIKey, d.RootID.RootFolderID) - - ctx := context.Background() - // Pass a valid root directory object instead of nil - rootDir := &model.Object{ID: "d:0", IsFolder: true} - obj, err := d.PutURL(ctx, rootDir, "test.txt", "https://example.com/test.txt") - if err != nil { - t.Fatalf("PutURL failed: %v", err) - } - t.Logf("PutURL returned: id=%s name=%s", obj.GetID(), obj.GetName()) - - // Extract upload ID - uploadID := remoteUploadIDFromObjID(obj.GetID()) - if uploadID == "" { - t.Fatal("PutURL returned invalid ID format") - } - t.Logf("Upload ID: %s", uploadID) - - // Pass Obj with the upload ID so extractRemoteUploadID can work - uploadObj := &model.Object{ID: obj.GetID()} - - // Test remotedl_status via Other method - statusResult, err := d.Other(ctx, model.OtherArgs{ - Obj: uploadObj, - Method: "remotedl_status", - Data: map[string]interface{}{"id": uploadID}, - }) - if err != nil { - t.Fatalf("remotedl_status failed: %v", err) - } - t.Logf("remotedl_status: %+v", statusResult) - - // Test remotedl_remove via Other method - removeResult, err := d.Other(ctx, model.OtherArgs{ - Obj: uploadObj, - Method: "remotedl_remove", - Data: map[string]interface{}{"id": uploadID}, - }) - if err != nil { - t.Fatalf("remotedl_remove failed: %v", err) - } - t.Logf("remotedl_remove: %v", removeResult) -} - -// TestDriverFileInfo tests file_info via Other method -func TestDriverFileInfo(t *testing.T) { - d := newTestDriver() - - // First get a file ID from list - ctx := context.Background() - objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) - if err != nil { - t.Fatalf("List failed: %v", err) - } - - // Find a file in subfolders - var fileID string - var fileName string - for _, obj := range objs { - if obj.IsDir() { - subObjs, err := d.List(ctx, obj, model.ListArgs{}) - if err != nil { - continue - } - for _, subObj := range subObjs { - if !subObj.IsDir() { - fileID = subObj.GetID() - fileName = subObj.GetName() - break - } - } - if fileID != "" { - break - } - } - } - - if fileID == "" { - t.Skip("No files found for file_info test") - } - t.Logf("Testing file_info with file: %s (id=%s)", fileName, fileID) - - // Test file_info via Other method - infoResult, err := d.Other(ctx, model.OtherArgs{ - Method: "file_info", - Obj: &model.Object{ID: fileID}, - }) - if err != nil { - t.Fatalf("file_info failed: %v", err) - } - t.Logf("file_info: %+v", infoResult) -} - -// TestDriverThumbnail tests thumbnail via Other method -func TestDriverThumbnail(t *testing.T) { - d := newTestDriver() - - // First get a file ID from list - ctx := context.Background() - objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) - if err != nil { - t.Fatalf("List failed: %v", err) - } - - // Find a file in subfolders - var fileID string - for _, obj := range objs { - if obj.IsDir() { - subObjs, err := d.List(ctx, obj, model.ListArgs{}) - if err != nil { - continue - } - for _, subObj := range subObjs { - if !subObj.IsDir() { - fileID = subObj.GetID() - break - } - } - if fileID != "" { - break - } - } - } - - if fileID == "" { - t.Skip("No files found for thumbnail test") - } - t.Logf("Testing thumbnail with file id=%s", fileID) - - // Test thumbnail via Other method - thumbResult, err := d.Other(ctx, model.OtherArgs{ - Method: "thumbnail", - Obj: &model.Object{ID: fileID}, - }) - if err != nil { - t.Fatalf("thumbnail failed: %v", err) - } - t.Logf("thumbnail: %v", thumbResult) -} - -// TestDriverConversionStatus tests conversion_status via Other method -func TestDriverConversionStatus(t *testing.T) { - d := newTestDriver() - - ctx := context.Background() - - // Test running conversions - runningResult, err := d.Other(ctx, model.OtherArgs{ - Method: "conversion_status", - }) - if err != nil { - t.Fatalf("conversion_status (running) failed: %v", err) - } - t.Logf("conversion_status (running): %+v", runningResult) - - // Test failed conversions - failedResult, err := d.Other(ctx, model.OtherArgs{ - Method: "conversion_status", - Data: map[string]interface{}{"type": "failed"}, - }) - if err != nil { - t.Fatalf("conversion_status (failed) failed: %v", err) - } - t.Logf("conversion_status (failed): %+v", failedResult) -} \ No newline at end of file From a0545cb4366ce62060bfee402ea149a98a757883 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Sat, 28 Mar 2026 22:46:33 +0800 Subject: [PATCH 106/133] chore(deps): bump github.com/shoenig/go-m1cpu to v0.2.1 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bc2475c548d..17c09201ab0 100644 --- a/go.mod +++ b/go.mod @@ -250,7 +250,7 @@ require ( github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/shirou/gopsutil/v3 v3.24.4 // indirect - github.com/shoenig/go-m1cpu v0.2.0 // indirect + github.com/shoenig/go-m1cpu v0.2.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index e6b04779ede..370ac6d206f 100644 --- a/go.sum +++ b/go.sum @@ -591,6 +591,8 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/go-m1cpu v0.2.0 h1:t4GNqvPZ84Vjtpboo/kT3pIkbaK3vc+JIlD/Wz1zSFY= github.com/shoenig/go-m1cpu v0.2.0/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= +github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= +github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= From c4595e59f3ecd81ad508c5c310a1f15d6ff09b7f Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Mon, 30 Mar 2026 10:00:03 +0800 Subject: [PATCH 107/133] Fix share access accounting and validation --- internal/db/share.go | 12 ++++---- server/handles/share.go | 55 +++++++++++++++++++++++++++++----- server/handles/share_public.go | 27 ++++++++--------- 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/internal/db/share.go b/internal/db/share.go index c92e546755e..711c3f278c3 100644 --- a/internal/db/share.go +++ b/internal/db/share.go @@ -24,20 +24,20 @@ func GetShareByCreatorAndShareID(creatorID uint, shareID string) (*model.Share, return &share, nil } -func ShareIDExists(shareID string) bool { +func ShareIDExists(shareID string) (bool, error) { var count int64 if err := db.Model(&model.Share{}).Where("share_id = ?", shareID).Count(&count).Error; err != nil { - return false + return false, err } - return count > 0 + return count > 0, nil } -func ShareIDExistsExceptID(shareID string, id uint) bool { +func ShareIDExistsExceptID(shareID string, id uint) (bool, error) { var count int64 if err := db.Model(&model.Share{}).Where("share_id = ? AND id <> ?", shareID, id).Count(&count).Error; err != nil { - return false + return false, err } - return count > 0 + return count > 0, nil } func CreateShare(share *model.Share) error { diff --git a/server/handles/share.go b/server/handles/share.go index 99a693b7a5f..4de7da63672 100644 --- a/server/handles/share.go +++ b/server/handles/share.go @@ -2,7 +2,9 @@ package handles import ( "crypto/subtle" + "errors" "fmt" + "net/http" "net/url" stdpath "path" "regexp" @@ -24,6 +26,11 @@ const shareAccessTokenLifetime = 24 * time.Hour var shareIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,32}$`) +var ( + errShareIDInvalid = errors.New("share_id must be 1-32 characters of letters, numbers, underscore or hyphen") + errShareIDExists = errors.New("share link already exists") +) + type CreateShareReq struct { Path string `json:"path" binding:"required"` ShareID string `json:"share_id"` @@ -184,7 +191,11 @@ func normalizeOptionalShareName(name, fallback string) string { func generateShareID() (string, error) { for range 10 { shareID := random.String(8) - if !db.ShareIDExists(shareID) { + exists, err := db.ShareIDExists(shareID) + if err != nil { + return "", err + } + if !exists { return shareID, nil } } @@ -200,7 +211,7 @@ func validateCustomShareID(shareID string) error { return nil } if !shareIDPattern.MatchString(shareID) { - return fmt.Errorf("share_id must be 1-32 characters of letters, numbers, underscore or hyphen") + return errShareIDInvalid } return nil } @@ -217,13 +228,21 @@ func resolveRequestedShareID(rawShareID, fallback string, excludeID uint) (strin return "", err } if excludeID == 0 { - if db.ShareIDExists(shareID) { - return "", fmt.Errorf("share link already exists") + exists, err := db.ShareIDExists(shareID) + if err != nil { + return "", fmt.Errorf("check share id availability: %w", err) + } + if exists { + return "", errShareIDExists } return shareID, nil } - if db.ShareIDExistsExceptID(shareID, excludeID) { - return "", fmt.Errorf("share link already exists") + exists, err := db.ShareIDExistsExceptID(shareID, excludeID) + if err != nil { + return "", fmt.Errorf("check share id availability: %w", err) + } + if exists { + return "", errShareIDExists } return shareID, nil } @@ -334,6 +353,10 @@ func ensureShareAccess(c *gin.Context, share *model.Share, token string) bool { return true } +func shouldTrackShareContentAccess(c *gin.Context) bool { + return c.Request.Method != http.MethodHead +} + func resolveShareTarget(share *model.Share, rawRelPath string) (string, string, error) { cleanRelPath := utils.FixAndCleanPath(rawRelPath) if !share.IsDir && cleanRelPath != "/" { @@ -349,6 +372,14 @@ func resolveShareTarget(share *model.Share, rawRelPath string) (string, string, return target, cleanRelPath, nil } +func resolveShareWildcardTarget(share *model.Share, rawPath string) (string, string, error) { + path, err := url.PathUnescape(rawPath) + if err != nil { + return "", "", err + } + return resolveShareTarget(share, strings.TrimPrefix(path, "/")) +} + func buildPublicShareAssetURL(c *gin.Context, prefix, shareID, relPath, token string, preview bool) string { base := common.GetApiUrl(c.Request) + prefix + shareID cleanPath := utils.FixAndCleanPath(relPath) @@ -423,7 +454,11 @@ func CreateShare(c *gin.Context) { } shareID, err := resolveRequestedShareID(req.ShareID, "", 0) if err != nil { - common.ErrorResp(c, err, 400) + if errors.Is(err, errShareIDInvalid) || errors.Is(err, errShareIDExists) { + common.ErrorResp(c, err, 400) + return + } + common.ErrorResp(c, err, 500, true) return } allowPreview := true @@ -483,7 +518,11 @@ func UpdateShare(c *gin.Context) { shareID, err := resolveRequestedShareID(req.NewShareID, share.ShareID, share.ID) if err != nil { - common.ErrorResp(c, err, 400) + if errors.Is(err, errShareIDInvalid) || errors.Is(err, errShareIDExists) { + common.ErrorResp(c, err, 400) + return + } + common.ErrorResp(c, err, 500, true) return } diff --git a/server/handles/share_public.go b/server/handles/share_public.go index 22aba45bc4c..83ca79f3591 100644 --- a/server/handles/share_public.go +++ b/server/handles/share_public.go @@ -2,7 +2,6 @@ package handles import ( stdpath "path" - "strings" "time" "github.com/alist-org/alist/v3/internal/db" @@ -143,10 +142,6 @@ func ListPublicShare(c *gin.Context) { } content = append(content, toPublicShareObjResp(c, share, item, itemTargetPath, itemRelPath, token)) } - if err := recordShareAccess(share); err != nil { - common.ErrorResp(c, err, 500, true) - return - } common.SuccessResp(c, PublicShareListResp{ Content: content, Total: int64(total), @@ -213,7 +208,7 @@ func ShareDown(c *gin.Context) { if !ensureShareAccess(c, share, token) { return } - targetPath, _, err := resolveShareTarget(share, strings.TrimPrefix(c.Param("path"), "/")) + targetPath, _, err := resolveShareWildcardTarget(share, c.Param("path")) if err != nil { common.ErrorResp(c, err, 400) return @@ -227,10 +222,12 @@ func ShareDown(c *gin.Context) { common.ErrorStrResp(c, "directory download is not supported", 400) return } - _ = db.TouchShareDownload(share.ShareID) - if err := recordShareAccess(share); err != nil { - common.ErrorResp(c, err, 500, true) - return + if shouldTrackShareContentAccess(c) { + _ = db.TouchShareDownload(share.ShareID) + if err := recordShareAccess(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } } c.Set("path", targetPath) Down(c) @@ -253,7 +250,7 @@ func ShareProxy(c *gin.Context) { if !ensureShareAccess(c, share, token) { return } - targetPath, _, err := resolveShareTarget(share, strings.TrimPrefix(c.Param("path"), "/")) + targetPath, _, err := resolveShareWildcardTarget(share, c.Param("path")) if err != nil { common.ErrorResp(c, err, 400) return @@ -267,9 +264,11 @@ func ShareProxy(c *gin.Context) { common.ErrorStrResp(c, "directory preview is not supported", 400) return } - if err := recordShareAccess(share); err != nil { - common.ErrorResp(c, err, 500, true) - return + if shouldTrackShareContentAccess(c) { + if err := recordShareAccess(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } } c.Set("path", targetPath) Proxy(c) From 71898a4443f8217f62d29ea0f735b84848f3cc42 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 31 Mar 2026 16:27:12 +0800 Subject: [PATCH 108/133] feat: add 360AI YunPan driver --- drivers/all.go | 1 + drivers/yunpan360/driver.go | 286 ++++++++++ drivers/yunpan360/meta.go | 34 ++ drivers/yunpan360/types.go | 462 +++++++++++++++ drivers/yunpan360/upload.go | 926 +++++++++++++++++++++++++++++++ drivers/yunpan360/upload_test.go | 25 + drivers/yunpan360/util.go | 909 ++++++++++++++++++++++++++++++ drivers/yunpan360/util_test.go | 109 ++++ 8 files changed, 2752 insertions(+) create mode 100644 drivers/yunpan360/driver.go create mode 100644 drivers/yunpan360/meta.go create mode 100644 drivers/yunpan360/types.go create mode 100644 drivers/yunpan360/upload.go create mode 100644 drivers/yunpan360/upload_test.go create mode 100644 drivers/yunpan360/util.go create mode 100644 drivers/yunpan360/util_test.go diff --git a/drivers/all.go b/drivers/all.go index a4fce9d0cac..ef872771961 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -83,6 +83,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/wopan" _ "github.com/alist-org/alist/v3/drivers/wukong" _ "github.com/alist-org/alist/v3/drivers/yandex_disk" + _ "github.com/alist-org/alist/v3/drivers/yunpan360" ) // All do nothing,just for import diff --git a/drivers/yunpan360/driver.go b/drivers/yunpan360/driver.go new file mode 100644 index 00000000000..c92e918a0ad --- /dev/null +++ b/drivers/yunpan360/driver.go @@ -0,0 +1,286 @@ +package yunpan360 + +import ( + "context" + "errors" + stdpath "path" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Yunpan360 struct { + model.Storage + Addition + + authMu sync.Mutex + cachedOpenAuth *OpenAuthInfo + openAuthExpire time.Time + + cachedCookieSession *CookieDownloadSession + cookieSessionExpire time.Time +} + +func (d *Yunpan360) Config() driver.Config { + return config +} + +func (d *Yunpan360) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Yunpan360) Init(ctx context.Context) error { + if d.PageSize <= 0 { + d.PageSize = 100 + } + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + if d.RootFolderPath == "" { + d.RootFolderPath = "/" + } + d.OrderDirection = strings.ToLower(strings.TrimSpace(d.OrderDirection)) + if d.OrderDirection != "desc" { + d.OrderDirection = "asc" + } + d.AuthType = strings.ToLower(strings.TrimSpace(d.AuthType)) + if d.AuthType == "" { + d.AuthType = authTypeCookie + } + d.SubChannel = strings.TrimSpace(d.SubChannel) + if d.SubChannel == "" { + d.SubChannel = defaultSubChannel + } + d.EcsEnv = strings.ToLower(strings.TrimSpace(d.EcsEnv)) + if d.EcsEnv == "" { + d.EcsEnv = openEnvProd + } + d.Cookie = strings.TrimSpace(d.Cookie) + d.APIKey = strings.TrimSpace(d.APIKey) + d.OwnerQID = strings.TrimSpace(d.OwnerQID) + d.DownloadToken = strings.TrimSpace(d.DownloadToken) + d.cachedOpenAuth = nil + d.openAuthExpire = time.Time{} + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} + + switch d.authMode() { + case authTypeAPIKey: + if d.APIKey == "" { + return errors.New("api_key is empty") + } + _, err := d.openUserInfo(ctx) + return err + case authTypeCookie: + if d.Cookie == "" { + return errors.New("cookie is empty") + } + // Web download URLs require browser-session headers; force local proxying + // so AList can forward Referer/Origin instead of exposing a bare 302 URL. + d.WebProxy = true + _, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + return err + default: + return errors.New("invalid auth_type") + } +} + +func (d *Yunpan360) Drop(ctx context.Context) error { + d.cachedOpenAuth = nil + d.openAuthExpire = time.Time{} + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} + return nil +} + +func (d *Yunpan360) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + dirPath := dir.GetPath() + if dirPath == "" { + dirPath = d.RootFolderPath + } + + objs := make([]model.Obj, 0, d.PageSize) + for page := 0; ; page++ { + resp, err := d.listPage(ctx, dirPath, page, d.PageSize) + if err != nil { + return nil, err + } + pageObjs := resp.Objects(dirPath) + for _, item := range pageObjs { + objs = append(objs, item) + } + if len(pageObjs) == 0 { + break + } + if d.authMode() == authTypeAPIKey { + if len(pageObjs) < d.PageSize { + break + } + continue + } + if !resp.GetHasNextPage() { + break + } + } + return objs, nil +} + +func (d *Yunpan360) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if d.authMode() == authTypeCookie { + resp, err := d.cookieDownloadURL(ctx, file) + if err != nil { + return nil, err + } + downloadURL := strings.TrimSpace(resp.GetURL()) + if downloadURL == "" { + return nil, errors.New("download url is empty") + } + return &model.Link{ + URL: downloadURL, + Header: map[string][]string{ + "Accept": {"text/javascript, text/html, application/xml, text/xml, */*"}, + "Origin": {baseURL}, + "Referer": {baseURL + indexPath}, + }, + }, nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + resp, err := d.openDownloadURL(ctx, file) + if err != nil { + return nil, err + } + downloadURL := strings.TrimSpace(resp.GetURL()) + if downloadURL == "" { + return nil, errors.New("download url is empty") + } + return &model.Link{URL: downloadURL}, nil +} + +func (d *Yunpan360) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if d.authMode() == authTypeCookie { + fullPath := ensureDirAPIPath(stdpath.Join(parentDir.GetPath(), dirName)) + resp, err := d.cookieMakeDir(ctx, fullPath) + if err != nil { + return nil, err + } + return &YunpanObject{ + Object: model.Object{ + ID: resp.Data.NID, + Path: normalizeRemotePath(fullPath), + Name: dirName, + Size: 0, + Modified: time.Now(), + IsFolder: true, + }, + }, nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + fullPath := ensureDirAPIPath(stdpath.Join(parentDir.GetPath(), dirName)) + resp, err := d.openMakeDir(ctx, fullPath) + if err != nil { + return nil, err + } + obj := &model.Object{ + ID: resp.Data.NID, + Path: normalizeRemotePath(fullPath), + Name: dirName, + Size: 0, + Modified: time.Now(), + IsFolder: true, + } + return obj, nil +} + +func (d *Yunpan360) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.authMode() == authTypeCookie { + srcPath := apiPathForObj(srcObj) + dstPath := ensureDirAPIPath(dstDir.GetPath()) + if err := d.cookieMove(ctx, srcPath, dstPath); err != nil { + return nil, err + } + return cloneObj(srcObj, stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.GetName()), nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + srcPath := apiPathForObj(srcObj) + dstPath := ensureDirAPIPath(dstDir.GetPath()) + if err := d.openMove(ctx, srcPath, dstPath); err != nil { + return nil, err + } + return cloneObj(srcObj, stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.GetName()), nil +} + +func (d *Yunpan360) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.authMode() == authTypeCookie { + targetName := strings.TrimSuffix(strings.TrimSpace(newName), "/") + if targetName == "" { + return nil, errors.New("new name is empty") + } + if err := d.cookieRename(ctx, srcObj, targetName); err != nil { + return nil, err + } + parentPath := stdpath.Dir(srcObj.GetPath()) + if parentPath == "." { + parentPath = "/" + } + return cloneObj(srcObj, stdpath.Join(parentPath, targetName), targetName), nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + srcPath := apiPathForObj(srcObj) + targetName := newName + if srcObj.IsDir() { + targetName = ensureDirSuffix(newName) + } + if err := d.openRename(ctx, srcPath, targetName); err != nil { + return nil, err + } + + parentPath := stdpath.Dir(srcObj.GetPath()) + if parentPath == "." { + parentPath = "/" + } + return cloneObj(srcObj, stdpath.Join(parentPath, strings.TrimSuffix(newName, "/")), strings.TrimSuffix(newName, "/")), nil +} + +func (d *Yunpan360) Remove(ctx context.Context, obj model.Obj) error { + if d.authMode() == authTypeCookie { + return d.cookieRecycle(ctx, obj) + } + if d.authMode() != authTypeAPIKey { + return errs.NotImplement + } + return d.openDelete(ctx, apiPathForObj(obj)) +} + +func (d *Yunpan360) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if d.authMode() == authTypeCookie { + return nil, errs.NotImplement + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + return d.putOpenFile(ctx, dstDir, file, up) +} + +func (d *Yunpan360) authMode() string { + if d.AuthType == authTypeAPIKey { + return authTypeAPIKey + } + return authTypeCookie +} + +var _ driver.Driver = (*Yunpan360)(nil) diff --git a/drivers/yunpan360/meta.go b/drivers/yunpan360/meta.go new file mode 100644 index 00000000000..5c4e1c8d245 --- /dev/null +++ b/drivers/yunpan360/meta.go @@ -0,0 +1,34 @@ +package yunpan360 + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + AuthType string `json:"auth_type" type:"select" options:"cookie,api_key" default:"cookie"` + Cookie string `json:"cookie" type:"text" help:"Cookie copied from a logged-in yunpan.com session; used when auth_type=cookie"` + OwnerQID string `json:"owner_qid" type:"text" help:"Optional owner_qid for cookie-mode download; leave empty to auto-detect"` + DownloadToken string `json:"download_token" type:"text" help:"Optional web token for cookie-mode download; leave empty to auto-detect"` + APIKey string `json:"api_key" type:"text" help:"360 AI YunPan API key; used when auth_type=api_key"` + EcsEnv string `json:"ecs_env" type:"select" options:"prod,test,hgtest" default:"prod"` + SubChannel string `json:"sub_channel" default:"open"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + PageSize int `json:"page_size" type:"number" default:"100" help:"List page size"` +} + +var config = driver.Config{ + Name: "360AIYunPan", + LocalSort: false, + CheckStatus: true, + NoUpload: false, + DefaultRoot: "/", + Alert: "info|api_key mode supports list/link/upload/mkdir/rename/move/delete; cookie mode supports list/link/mkdir/rename/move/delete only, and forces web proxy because direct download URLs require web headers.", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Yunpan360{} + }) +} diff --git a/drivers/yunpan360/types.go b/drivers/yunpan360/types.go new file mode 100644 index 00000000000..f5c1b7a2135 --- /dev/null +++ b/drivers/yunpan360/types.go @@ -0,0 +1,462 @@ +package yunpan360 + +import ( + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + authTypeCookie = "cookie" + authTypeAPIKey = "api_key" + openEnvProd = "prod" + defaultSubChannel = "open" + openSignSecret = "e7b24b112a44fdd9ee93bdf998c6ca0e" + openClientID = "e4757e933b6486c08ed206ecb6d5d9e684fcb4e2" + openClientSecret = "885fd3231f1c1e37c9f462261a09b8c38cde0c2b" + openClientSecretQA = "b11b8fff1c75a5d227c8cc93aaeb0bb70c8eee47" +) + +type BaseResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` +} + +type CookieDownloadSession struct { + OwnerQID string + Token string +} + +type ListResp interface { + Objects(parentPath string) []model.Obj + GetHasNextPage() bool +} + +type CookieListResp struct { + BaseResp + Token string `json:"token"` + OwnerQid string `json:"owner_qid"` + Qid string `json:"qid"` + Data []ListItem `json:"data"` + HasNextPage bool `json:"has_next_page"` +} + +func (r *CookieListResp) Objects(parentPath string) []model.Obj { + ownerQID := r.GetOwnerQID() + return utils.MustSliceConvert(r.Data, func(src ListItem) model.Obj { + return src.toObj(parentPath, ownerQID, r.Token) + }) +} + +func (r *CookieListResp) GetHasNextPage() bool { + return r.HasNextPage +} + +func (r *CookieListResp) GetOwnerQID() string { + return firstNonEmpty(r.OwnerQid, r.Qid) +} + +type ListItem struct { + NID string `json:"nid"` + FileName string `json:"file_name"` + FilePath string `json:"file_path"` + FileSize string `json:"file_size"` + IsDir bool `json:"is_dir"` + Fhash string `json:"fhash"` + CreateTime string `json:"create_time"` + ModifyTime string `json:"modify_time"` + Mtime string `json:"mtime"` + ServerTime string `json:"server_time"` + Preview string `json:"preview"` + Thumb string `json:"thumb"` + SrcPic string `json:"srcpic"` + OwnerQid string `json:"owner_qid"` + Qid string `json:"qid"` + Token string `json:"token"` +} + +func (i ListItem) toObj(parentPath, ownerQID, token string) model.Obj { + objPath := normalizeRemotePath(i.FilePath) + if objPath == "" || !pathLooksLikeObject(objPath, i.FileName) { + objPath = joinRemotePath(parentPath, i.FileName) + } + thumb := "" + if !i.IsDir { + thumb = absoluteURL(firstNonEmpty(i.Thumb, i.SrcPic, i.Preview)) + } + + return &YunpanObject{ + Object: model.Object{ + ID: i.NID, + Path: objPath, + Name: i.FileName, + Size: parseSize(i.FileSize), + Modified: parseYunpanTime(i.ModifyTime, i.Mtime), + Ctime: parseYunpanTime(i.CreateTime, i.ServerTime), + IsFolder: i.IsDir, + HashInfo: parseHash(i.Fhash), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + OwnerQID: firstNonEmpty(i.OwnerQid, i.Qid, ownerQID), + DownloadToken: firstNonEmpty(i.Token, token), + } +} + +func parseSize(raw string) int64 { + size, _ := strconv.ParseInt(raw, 10, 64) + return size +} + +func parseHash(raw string) utils.HashInfo { + if len(raw) == 40 { + return utils.NewHashInfo(utils.SHA1, raw) + } + return utils.HashInfo{} +} + +func parseYunpanTime(unixStr, text string) time.Time { + if t := parseUnixTime(unixStr); !t.IsZero() { + return t + } + return parseTextTime(text) +} + +func parseUnixTime(raw string) time.Time { + if raw != "" { + sec, err := strconv.ParseInt(raw, 10, 64) + if err == nil && sec > 0 { + return time.Unix(sec, 0) + } + } + return time.Time{} +} + +func parseTextTime(text string) time.Time { + if text == "" { + return time.Time{} + } + t, err := time.ParseInLocation("2006-01-02 15:04:05", text, utils.CNLoc) + if err == nil { + return t + } + return time.Time{} +} + +func normalizeRemotePath(p string) string { + if p == "" { + return "" + } + if p != "/" { + p = strings.TrimSuffix(p, "/") + } + return utils.FixAndCleanPath(p) +} + +type OpenAuthResp struct { + BaseResp + Data struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + AccessTokenExpire int64 `json:"access_token_expire"` + Qid string `json:"qid"` + } `json:"data"` +} + +type OpenAuthInfo struct { + AccessToken string + Qid string + Token string + SubChannel string +} + +type OpenListResp struct { + BaseResp + Data struct { + NodeList []OpenNode `json:"node_list"` + List []OpenNode `json:"list"` + Data []OpenNode `json:"data"` + TotalCount int `json:"total_count"` + Total int `json:"total"` + PageNum int `json:"page_num"` + } `json:"data"` +} + +func (r *OpenListResp) Objects(parentPath string) []model.Obj { + nodes := r.Data.NodeList + if len(nodes) == 0 { + nodes = r.Data.List + } + if len(nodes) == 0 { + nodes = r.Data.Data + } + return utils.MustSliceConvert(nodes, func(src OpenNode) model.Obj { + return src.toObj(parentPath) + }) +} + +func (r *OpenListResp) GetHasNextPage() bool { + total := r.Data.TotalCount + if total <= 0 { + total = r.Data.Total + } + if total <= 0 { + return false + } + loaded := len(r.Data.NodeList) + if loaded == 0 { + loaded = len(r.Data.List) + } + if loaded == 0 { + loaded = len(r.Data.Data) + } + return loaded > 0 && loaded < total +} + +type OpenNode struct { + NID string `json:"nid"` + Name string `json:"name"` + FName string `json:"fname"` + Path string `json:"path"` + FPath string `json:"fpath"` + Type interface{} `json:"type"` + IsDir interface{} `json:"is_dir"` + CountSize interface{} `json:"count_size"` + Size interface{} `json:"size"` + CreateTime interface{} `json:"create_time"` + ModifyTime interface{} `json:"modify_time"` + MTime interface{} `json:"mtime"` + FileHash string `json:"file_hash"` + Fhash string `json:"fhash"` + Thumb string `json:"thumb"` + Preview string `json:"preview"` + SrcPic string `json:"srcpic"` +} + +func (n OpenNode) toObj(parentPath string) model.Obj { + name := firstNonEmpty(strings.TrimSpace(n.Name), strings.TrimSpace(n.FName)) + objPath := normalizeRemotePath(firstNonEmpty(n.FPath, n.Path)) + if objPath == "" || !pathLooksLikeObject(objPath, name) { + objPath = joinRemotePath(parentPath, name) + } + isDir := parseOpenDir(n.IsDir, n.Type) + thumb := "" + if !isDir { + thumb = absoluteURL(firstNonEmpty(n.Thumb, n.SrcPic, n.Preview)) + } + + return &YunpanObject{ + Object: model.Object{ + ID: n.NID, + Path: objPath, + Name: name, + Size: parseAnySize(n.CountSize, n.Size), + Modified: parseAnyTime(n.ModifyTime, n.MTime), + Ctime: parseAnyTime(n.CreateTime), + IsFolder: isDir, + HashInfo: parseHash(firstNonEmpty(n.FileHash, n.Fhash)), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + } +} + +type OpenUserInfoResp struct { + BaseResp + Data map[string]interface{} `json:"data"` +} + +type OpenMkdirResp struct { + BaseResp + Data struct { + NID string `json:"nid"` + } `json:"data"` +} + +type CookieMkdirResp struct { + BaseResp + Data struct { + NID string `json:"nid"` + } `json:"data"` +} + +type CookieMoveResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + IsAsync bool `json:"is_async"` + } `json:"data"` +} + +type CookieRecycleResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + IsAsync bool `json:"is_async"` + } `json:"data"` +} + +type CookieAsyncQueryResp struct { + BaseResp + Data map[string]CookieAsyncTask `json:"data"` +} + +type CookieAsyncTask struct { + MessageID string `json:"message_id"` + SendTime string `json:"send_time"` + Status int `json:"status"` + Action string `json:"action"` + Errno int `json:"errno"` + Errstr string `json:"errstr"` + Result string `json:"result"` +} + +type CookieDownloadResp struct { + BaseResp + Data struct { + DownloadURL string `json:"download_url"` + Store string `json:"store"` + Host string `json:"host"` + } `json:"data"` +} + +func (r *CookieDownloadResp) GetURL() string { + return r.Data.DownloadURL +} + +type OpenDownloadResp struct { + BaseResp + Data struct { + DownloadURL string `json:"downloadUrl"` + } `json:"data"` + DownloadURL string `json:"downloadUrl"` +} + +func (r *OpenDownloadResp) GetURL() string { + return firstNonEmpty(r.Data.DownloadURL, r.DownloadURL) +} + +type YunpanObject struct { + model.Object + model.Thumbnail + OwnerQID string + DownloadToken string +} + +func parseAnySize(values ...interface{}) int64 { + for _, value := range values { + switch v := value.(type) { + case string: + if v == "" { + continue + } + size, err := strconv.ParseInt(v, 10, 64) + if err == nil { + return size + } + case float64: + return int64(v) + case int64: + return v + case int: + return int64(v) + } + } + return 0 +} + +func parseAnyTime(values ...interface{}) time.Time { + for _, value := range values { + switch v := value.(type) { + case string: + if t := parseUnixTime(v); !t.IsZero() { + return t + } + if t := parseTextTime(v); !t.IsZero() { + return t + } + case float64: + if v > 0 { + return time.Unix(int64(v), 0) + } + case int64: + if v > 0 { + return time.Unix(v, 0) + } + case int: + if v > 0 { + return time.Unix(int64(v), 0) + } + } + } + return time.Time{} +} + +func parseOpenDir(values ...interface{}) bool { + for _, value := range values { + switch v := value.(type) { + case bool: + if v { + return true + } + case string: + switch strings.ToLower(strings.TrimSpace(v)) { + case "1", "true", "dir", "folder": + return true + } + case float64: + if int64(v) == 1 { + return true + } + case int64: + if v == 1 { + return true + } + case int: + if v == 1 { + return true + } + } + } + return false +} + +func pathLooksLikeObject(objPath, name string) bool { + if objPath == "" || name == "" { + return false + } + return strings.TrimSuffix(stdPathBase(objPath), "/") == strings.TrimSuffix(name, "/") +} + +func stdPathBase(p string) string { + if p == "/" { + return "/" + } + idx := strings.LastIndex(strings.TrimSuffix(p, "/"), "/") + if idx < 0 { + return p + } + return p[idx+1:] +} + +func joinRemotePath(parentPath, name string) string { + parentPath = normalizeRemotePath(parentPath) + if parentPath == "" { + parentPath = "/" + } + return normalizeRemotePath(strings.TrimSuffix(parentPath, "/") + "/" + strings.TrimPrefix(name, "/")) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} diff --git a/drivers/yunpan360/upload.go b/drivers/yunpan360/upload.go new file mode 100644 index 00000000000..38c3fded57b --- /dev/null +++ b/drivers/yunpan360/upload.go @@ -0,0 +1,926 @@ +package yunpan360 + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/textproto" + "net/url" + stdpath "path" + "runtime" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + aliststream "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + yunpanUploadChunkSize = int64(512 * 1024) + yunpanUploadBoundary = "WebKitFormBoundaryQ5OJVvzZwEkg4ttY" + yunpanUploadVersion = "1.0.1" + yunpanUploadDevType = "ecs_openapi" + yunpanUploadDevName = "EYUN_WEB_UPLOAD" +) + +type openUploadPlan struct { + DirPath string + TargetPath string + FileName string + Size int64 + FileHash string + FileSHA1 string + FileSum string + CreatedAt int64 + DeviceID string + Chunks []openUploadChunk +} + +type openUploadChunk struct { + Index int + Offset int64 + Size int64 + Hash string +} + +type openUploadDetectResp struct { + BaseResp + Data struct { + Exists []openUploadDuplicate `json:"exists"` + IsSlice int `json:"is_slice"` + } `json:"data"` +} + +type openUploadDuplicate struct { + FullName string `json:"fullName"` +} + +type openUploadAddressResp struct { + BaseResp + Data struct { + HTTP string `json:"http"` + Addr1 string `json:"addr_1"` + Addr2 string `json:"addr_2"` + Backup string `json:"backup"` + TK string `json:"tk"` + GroupSize string `json:"group_size"` + AutoCommit interface{} `json:"autoCommit"` + IsHTTPS interface{} `json:"is_https"` + } `json:"data"` +} + +type openUploadRequestResp struct { + BaseResp + Data struct { + Tid string `json:"tid"` + BlockInfo []map[string]interface{} `json:"block_info"` + } `json:"data"` +} + +type openUploadFinalizeResp struct { + BaseResp + Data map[string]interface{} `json:"data"` +} + +type uploadEnvelope struct { + Errno *int `json:"errno"` + Errmsg string `json:"errmsg"` + Data json.RawMessage `json:"data"` +} + +func (d *Yunpan360) putOpenFile(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return nil, err + } + + dirPath := dstDir.GetPath() + if dirPath == "" { + dirPath = d.RootFolderPath + } + dirPath = ensureDirAPIPath(dirPath) + targetPath := joinRemotePath(dirPath, file.GetName()) + + cached, err := d.cacheUploadSource(ctx, file, progressRange(up, 0, 5)) + if err != nil { + return nil, err + } + defer func() { + _, _ = cached.Seek(0, io.SeekStart) + }() + + plan, err := d.buildUploadPlan(ctx, cached, targetPath, file.GetSize(), file.ModTime(), progressRange(up, 5, 10)) + if err != nil { + return nil, err + } + + detectResp, err := d.openDetectUpload(ctx, auth, plan) + if err != nil { + return nil, err + } + if detectResp.Data.IsSlice == 0 { + detectResp.Data.IsSlice = 1 + } + + addrResp, err := d.openGetUploadAddress(ctx, auth, plan) + if err != nil { + return nil, err + } + + finalResp := &openUploadFinalizeResp{BaseResp: BaseResp{Errno: 0}, Data: map[string]interface{}{}} + if strings.TrimSpace(addrResp.Data.HTTP) == "" { + finalResp.Data = map[string]interface{}{"autoCommit": true} + if tk := strings.TrimSpace(addrResp.Data.TK); tk != "" { + finalResp.Data["tk"] = tk + finalResp.Data["autoCommit"] = false + } + } else { + reqResp, err := d.openRequestUpload(ctx, auth, plan, addrResp) + if err != nil { + return nil, err + } + if err := d.openUploadBlocks(ctx, auth, cached, plan, addrResp, reqResp, up); err != nil { + return nil, err + } + finalResp, err = d.openCommitUpload(ctx, auth, plan, addrResp, reqResp) + if err != nil { + return nil, err + } + } + + if err := d.openFinalizeUpload(ctx, auth, finalResp); err != nil { + return nil, err + } + if up != nil { + up(100) + } + + obj, err := d.findUploadedObject(ctx, targetPath) + if err == nil { + return obj, nil + } + if !errors.Is(err, errs.ObjectNotFound) { + return nil, err + } + + return &model.Object{ + Path: normalizeRemotePath(targetPath), + Name: file.GetName(), + Size: file.GetSize(), + Modified: time.Now(), + Ctime: time.Now(), + HashInfo: utils.NewHashInfo(utils.SHA1, firstNonEmpty(plan.FileSHA1, plan.FileHash)), + }, nil +} + +func (d *Yunpan360) cacheUploadSource(ctx context.Context, file model.FileStreamer, up driver.UpdateProgress) (model.File, error) { + if cached := file.GetFile(); cached != nil { + _, _ = cached.Seek(0, io.SeekStart) + if up != nil { + up(100) + } + return cached, nil + } + if up == nil { + return file.CacheFullInTempFile() + } + return aliststream.CacheFullInTempFileAndUpdateProgress(file, up) +} + +func (d *Yunpan360) buildUploadPlan(ctx context.Context, cached model.File, targetPath string, size int64, modTime time.Time, up driver.UpdateProgress) (*openUploadPlan, error) { + if _, err := cached.Seek(0, io.SeekStart); err != nil { + return nil, err + } + + createdAt := time.Now().Unix() + if !modTime.IsZero() { + createdAt = modTime.Unix() + } + plan := &openUploadPlan{ + DirPath: ensureDirAPIPath(stdpath.Dir(targetPath)), + TargetPath: normalizeRemotePath(targetPath), + FileName: stdpath.Base(targetPath), + Size: size, + CreatedAt: createdAt, + DeviceID: sha1HexString("node-sdk-" + runtime.Version()), + } + + if plan.DirPath == "./" || plan.DirPath == "." { + plan.DirPath = "/" + } + + totalChunks := 0 + if size > 0 { + totalChunks = int((size + yunpanUploadChunkSize - 1) / yunpanUploadChunkSize) + } + chunks := make([]openUploadChunk, 0, totalChunks) + var hashConcat strings.Builder + var hashed int64 + + for idx := 0; idx < totalChunks; idx++ { + if err := ctx.Err(); err != nil { + return nil, err + } + offset := int64(idx) * yunpanUploadChunkSize + chunkSize := yunpanUploadChunkSize + if remain := size - offset; remain < chunkSize { + chunkSize = remain + } + + chunkHash, err := sha1HexReader(io.NewSectionReader(cached, offset, chunkSize)) + if err != nil { + return nil, err + } + + chunks = append(chunks, openUploadChunk{ + Index: idx + 1, + Offset: offset, + Size: chunkSize, + Hash: chunkHash, + }) + hashConcat.WriteString(chunkHash) + hashed += chunkSize + reportByteProgress(up, hashed, size) + } + + plan.Chunks = chunks + plan.FileHash = sha1HexString(hashConcat.String()) + if _, err := cached.Seek(0, io.SeekStart); err != nil { + return nil, err + } + sha1Hasher := sha1.New() + md5Hasher := md5.New() + if _, err := io.Copy(io.MultiWriter(sha1Hasher, md5Hasher), cached); err != nil { + return nil, err + } + plan.FileSHA1 = hex.EncodeToString(sha1Hasher.Sum(nil)) + plan.FileSum = hex.EncodeToString(md5Hasher.Sum(nil)) + if size == 0 && up != nil { + up(100) + } + return plan, nil +} + +func (d *Yunpan360) openDetectUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan) (*openUploadDetectResp, error) { + payload, err := json.Marshal([]map[string]interface{}{ + {"fname": plan.FileName, "fsize": plan.Size}, + }) + if err != nil { + return nil, err + } + + signParams := map[string]string{ + "data": string(payload), + "path": plan.DirPath, + } + body, contentType, err := createMultipartForm("", map[string]string{ + "qid": auth.Qid, + "data": string(payload), + "path": plan.DirPath, + "sign": openSign(auth.AccessToken, auth.Qid, "Sync.detectFileExists", signParams), + }, nil) + if err != nil { + return nil, err + } + + var resp openUploadDetectResp + err = d.uploadRequest(ctx, http.MethodPost, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.detectFileExists", nil), auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openGetUploadAddress(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan) (*openUploadAddressResp, error) { + query := d.uploadCookieParams(auth, plan, "") + signParams := map[string]string{ + "access_token": auth.AccessToken, + "fhash": plan.FileHash, + "fname": plan.TargetPath, + "fsize": strconv.FormatInt(plan.Size, 10), + } + query["sign"] = openSign(auth.AccessToken, auth.Qid, "Sync.getUploadFileAddr", signParams) + + var resp openUploadAddressResp + err := d.uploadRequest(ctx, http.MethodGet, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.getUploadFileAddr", query), auth.AccessToken, "", nil, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openRequestUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan, addrResp *openUploadAddressResp) (*openUploadRequestResp, error) { + chunkInfos := make([]map[string]interface{}, 0, len(plan.Chunks)) + for _, chunk := range plan.Chunks { + chunkInfos = append(chunkInfos, map[string]interface{}{ + "bhash": chunk.Hash, + "bidx": chunk.Index, + "boffset": chunk.Offset, + "bsize": chunk.Size, + }) + } + payload, err := json.Marshal(map[string]interface{}{ + "request": map[string]interface{}{ + "block_info": chunkInfos, + }, + }) + if err != nil { + return nil, err + } + + body, contentType, err := createMultipartForm( + yunpanUploadBoundary, + d.uploadCookieParams(auth, plan, strings.TrimSpace(addrResp.Data.TK)), + &multipartFile{ + FieldName: "file", + FileName: "file.dat", + ContentType: "application/octet-stream", + Content: payload, + }, + ) + if err != nil { + return nil, err + } + + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.request4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var resp openUploadRequestResp + err = d.uploadRequest(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openUploadBlocks(ctx context.Context, auth *OpenAuthInfo, cached model.File, plan *openUploadPlan, addrResp *openUploadAddressResp, reqResp *openUploadRequestResp, up driver.UpdateProgress) error { + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.block4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var uploaded int64 + for _, chunk := range plan.Chunks { + info := reqResp.blockInfoForChunk(chunk.Index) + if info.found() > 0 { + uploaded += chunk.Size + reportUploadProgress(up, uploaded, plan.Size) + continue + } + + chunkBytes := make([]byte, chunk.Size) + if _, err := cached.ReadAt(chunkBytes, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { + return err + } + + fields := map[string]string{ + "bhash": chunk.Hash, + "bidx": strconv.Itoa(chunk.Index), + "boffset": strconv.FormatInt(chunk.Offset, 10), + "bsize": strconv.FormatInt(chunk.Size, 10), + "filename": plan.TargetPath, + "filesize": strconv.FormatInt(plan.Size, 10), + "q": info.stringValue("q"), + "t": info.stringValue("t"), + "token": auth.Token, + "tid": info.stringValue("tid"), + } + for key, value := range info.extraFields() { + fields[key] = value + } + + body, contentType, err := createMultipartForm( + yunpanUploadBoundary, + fields, + &multipartFile{ + FieldName: "file", + FileName: "file.dat", + ContentType: "application/octet-stream", + Content: chunkBytes, + }, + ) + if err != nil { + return err + } + + chunkStart := uploaded + chunkSize := chunk.Size + err = d.uploadRequestWithProgress(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, func(p float64) { + done := chunkStart + int64(float64(chunkSize)*(p/100.0)) + reportUploadProgress(up, done, plan.Size) + }, nil) + if err != nil { + return err + } + + uploaded += chunk.Size + reportUploadProgress(up, uploaded, plan.Size) + } + return nil +} + +func (d *Yunpan360) openCommitUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan, addrResp *openUploadAddressResp, reqResp *openUploadRequestResp) (*openUploadFinalizeResp, error) { + body, contentType, err := createMultipartForm("", map[string]string{ + "q": "", + "t": "", + "token": auth.Token, + "tid": strings.TrimSpace(reqResp.Data.Tid), + }, nil) + if err != nil { + return nil, err + } + + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.commit4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var resp openUploadFinalizeResp + err = d.uploadRequest(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openFinalizeUpload(ctx context.Context, auth *OpenAuthInfo, resp *openUploadFinalizeResp) error { + if resp == nil || resp.autoCommit() { + return nil + } + tk := strings.TrimSpace(resp.stringValue("tk")) + if tk == "" { + return errors.New("upload tk is empty") + } + + signParams := map[string]string{ + "tk": tk, + } + body, contentType, err := createMultipartForm("", map[string]string{ + "qid": auth.Qid, + "tk": tk, + "sign": openSign(auth.AccessToken, auth.Qid, "Sync.addFileToApi", signParams), + }, nil) + if err != nil { + return err + } + + return d.uploadRequest(ctx, http.MethodPost, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.addFileToApi", nil), auth.AccessToken, contentType, body, nil) +} + +func (d *Yunpan360) findUploadedObject(ctx context.Context, targetPath string) (model.Obj, error) { + targetPath = normalizeRemotePath(targetPath) + parentPath := normalizeRemotePath(stdpath.Dir(targetPath)) + if parentPath == "." || parentPath == "" { + parentPath = "/" + } + targetName := stdpath.Base(targetPath) + + for page := 0; ; page++ { + resp, err := d.listPage(ctx, parentPath, page, d.PageSize) + if err != nil { + return nil, err + } + pageObjs := resp.Objects(parentPath) + for _, obj := range pageObjs { + if normalizeRemotePath(obj.GetPath()) == targetPath || obj.GetName() == targetName { + return obj, nil + } + } + if len(pageObjs) == 0 || len(pageObjs) < d.PageSize { + break + } + } + return nil, errs.ObjectNotFound +} + +func (d *Yunpan360) uploadCookieParams(auth *OpenAuthInfo, plan *openUploadPlan, uploadTK string) map[string]string { + params := map[string]string{ + "owner_qid": auth.Qid, + "fname": plan.TargetPath, + "fsize": strconv.FormatInt(plan.Size, 10), + "fctime": strconv.FormatInt(plan.CreatedAt, 10), + "fmtime": strconv.FormatInt(plan.CreatedAt, 10), + "fhash": plan.FileHash, + "qid": auth.Qid, + "fattr": "0", + "token": auth.Token, + "devtype": yunpanUploadDevType, + } + if uploadTK != "" { + params["tk"] = uploadTK + } + return params +} + +func (d *Yunpan360) uploadDataParams(auth *OpenAuthInfo, plan *openUploadPlan) map[string]string { + return map[string]string{ + "owner_qid": auth.Qid, + "qid": auth.Qid, + "devtype": yunpanUploadDevType, + "devid": plan.DeviceID, + "v": yunpanUploadVersion, + "ofmt": "json", + "devname": yunpanUploadDevName, + "rtick": strconv.FormatInt(time.Now().UnixMilli(), 10), + } +} + +func (d *Yunpan360) uploadBaseURL(addrResp *openUploadAddressResp) string { + host := "" + isHTTPS := false + if addrResp != nil { + host = strings.TrimSpace(addrResp.Data.HTTP) + isHTTPS = parseOpenDir(addrResp.Data.IsHTTPS) + } + scheme := "http" + if isHTTPS { + scheme = "https" + } + if host == "" { + return openAPIURL(d.EcsEnv) + } + return fmt.Sprintf("%s://%s/intf.php", scheme, host) +} + +func (d *Yunpan360) uploadRequest(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, out interface{}) error { + return d.uploadRequestWithProgress(ctx, method, reqURL, accessToken, contentType, body, nil, out) +} + +func (d *Yunpan360) uploadRequestWithProgress(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, progress driver.UpdateProgress, out interface{}) error { + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + if err := sleepWithContext(ctx, time.Duration(attempt)*500*time.Millisecond); err != nil { + return err + } + } + err := d.doUploadRequest(ctx, method, reqURL, accessToken, contentType, body, progress, out) + if err == nil { + return nil + } + lastErr = err + if ctx.Err() != nil { + return ctx.Err() + } + } + return lastErr +} + +func (d *Yunpan360) doUploadRequest(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, progress driver.UpdateProgress, out interface{}) error { + var bodyReader io.ReadCloser + if body != nil { + reader := &driver.SimpleReaderWithSize{ + Reader: bytes.NewReader(body), + Size: int64(len(body)), + } + if progress != nil { + bodyReader = driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: reader, + UpdateProgress: progress, + }) + } else { + bodyReader = driver.NewLimitedUploadStream(ctx, reader) + } + } + + req, err := http.NewRequestWithContext(ctx, method, reqURL, bodyReader) + if err != nil { + if bodyReader != nil { + _ = bodyReader.Close() + } + return err + } + if body != nil { + req.ContentLength = int64(len(body)) + } + req.Header.Set("Accept", "application/json") + if accessToken != "" { + req.Header.Set("Access-Token", accessToken) + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode >= http.StatusBadRequest { + return fmt.Errorf("yunpan upload request failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(respBody))) + } + return decodeUploadResp(respBody, out) +} + +func decodeUploadResp(body []byte, out interface{}) error { + var env uploadEnvelope + if err := utils.Json.Unmarshal(body, &env); err != nil { + return err + } + if env.Errno != nil && *env.Errno != 0 { + if env.Errmsg == "" { + return fmt.Errorf("yunpan upload request failed: errno=%d", *env.Errno) + } + return errors.New(env.Errmsg) + } + if env.Errno == nil && strings.TrimSpace(env.Errmsg) != "" && len(env.Data) > 0 && string(env.Data) == "[]" { + return errors.New(env.Errmsg) + } + if out == nil { + return nil + } + if err := utils.Json.Unmarshal(body, out); err != nil { + if strings.TrimSpace(env.Errmsg) != "" { + return errors.New(env.Errmsg) + } + return err + } + return nil +} + +type multipartFile struct { + FieldName string + FileName string + ContentType string + Content []byte +} + +func createMultipartForm(boundary string, fields map[string]string, file *multipartFile) ([]byte, string, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + if boundary != "" { + if err := writer.SetBoundary(boundary); err != nil { + return nil, "", err + } + } + + for key, value := range fields { + if err := writer.WriteField(key, value); err != nil { + return nil, "", err + } + } + + if file != nil { + partHeader := make(textproto.MIMEHeader) + partHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, file.FieldName, file.FileName)) + partHeader.Set("Content-Type", firstNonEmpty(file.ContentType, "application/octet-stream")) + part, err := writer.CreatePart(partHeader) + if err != nil { + return nil, "", err + } + if _, err := part.Write(file.Content); err != nil { + return nil, "", err + } + } + + if err := writer.Close(); err != nil { + return nil, "", err + } + return body.Bytes(), writer.FormDataContentType(), nil +} + +func buildJSQueryURL(baseURL, method string, params map[string]string) string { + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sort.Strings(keys) + + var builder strings.Builder + builder.WriteString(baseURL) + if strings.Contains(baseURL, "?") { + builder.WriteByte('&') + } else { + builder.WriteByte('?') + } + builder.WriteString("method=") + builder.WriteString(jsQueryEscape(method)) + for _, key := range keys { + value := params[key] + if value == "" { + continue + } + builder.WriteByte('&') + builder.WriteString(key) + builder.WriteByte('=') + builder.WriteString(jsQueryEscape(value)) + } + return builder.String() +} + +func buildQueryURL(baseURL string, params map[string]string) string { + u, err := url.Parse(baseURL) + if err != nil { + return baseURL + } + u.RawQuery = encodeSortedQuery(params) + return u.String() +} + +func encodeSortedQuery(params map[string]string) string { + if len(params) == 0 { + return "" + } + q := make(url.Values, len(params)) + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + q.Set(key, params[key]) + } + return q.Encode() +} + +func appendHostQuery(rawURL, host string) string { + host = strings.TrimSpace(host) + if host == "" { + return rawURL + } + return rawURL + "&host=" + jsQueryEscape(host) +} + +func jsQueryEscape(raw string) string { + replacer := strings.NewReplacer( + "+", "%20", + "%21", "!", + "%27", "'", + "%28", "(", + "%29", ")", + "%2A", "*", + "%7E", "~", + ) + return replacer.Replace(url.QueryEscape(raw)) +} + +func sha1HexReader(r io.Reader) (string, error) { + h := sha1.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func sha1HexString(raw string) string { + sum := sha1.Sum([]byte(raw)) + return hex.EncodeToString(sum[:]) +} + +func progressRange(up driver.UpdateProgress, start, end float64) driver.UpdateProgress { + if up == nil { + return nil + } + return model.UpdateProgressWithRange(up, start, end) +} + +func reportByteProgress(up driver.UpdateProgress, done, total int64) { + if up == nil { + return + } + if total <= 0 { + up(100) + return + } + up(float64(done) / float64(total) * 100) +} + +func reportUploadProgress(up driver.UpdateProgress, done, total int64) { + if up == nil { + return + } + if total <= 0 { + up(100) + return + } + if done > total { + done = total + } + up(10 + float64(done)/float64(total)*90) +} + +func sleepWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +type blockInfoMap map[string]interface{} + +func (r *openUploadRequestResp) blockInfoForChunk(index int) blockInfoMap { + if index <= 0 || index > len(r.Data.BlockInfo) { + return blockInfoMap{} + } + return blockInfoMap(r.Data.BlockInfo[index-1]) +} + +func (m blockInfoMap) stringValue(key string) string { + value, ok := m[key] + if !ok || value == nil { + return "" + } + switch v := value.(type) { + case string: + return v + case float64: + return strconv.FormatInt(int64(v), 10) + case int: + return strconv.Itoa(v) + case int64: + return strconv.FormatInt(v, 10) + case bool: + if v { + return "1" + } + return "0" + default: + return fmt.Sprint(v) + } +} + +func (m blockInfoMap) found() int64 { + raw := strings.TrimSpace(m.stringValue("found")) + if raw == "" { + return 0 + } + value, _ := strconv.ParseInt(raw, 10, 64) + return value +} + +func (m blockInfoMap) extraFields() map[string]string { + extras := make(map[string]string) + for key := range m { + switch key { + case "bhash", "bidx", "boffset", "bsize", "filename", "filesize", "q", "t", "token", "tid", "found", "url": + continue + } + value := strings.TrimSpace(m.stringValue(key)) + if value != "" { + extras[key] = value + } + } + return extras +} + +func (r *openUploadFinalizeResp) autoCommit() bool { + raw, ok := r.Data["autoCommit"] + if !ok { + return false + } + switch v := raw.(type) { + case bool: + return v + case string: + return strings.EqualFold(v, "true") || v == "1" + case float64: + return int64(v) == 1 + case int: + return v == 1 + case int64: + return v == 1 + default: + return false + } +} + +func (r *openUploadFinalizeResp) stringValue(key string) string { + if r == nil || r.Data == nil { + return "" + } + value, ok := r.Data[key] + if !ok || value == nil { + return "" + } + switch v := value.(type) { + case string: + return v + case float64: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(v, 10) + case int: + return strconv.Itoa(v) + default: + return fmt.Sprint(v) + } +} diff --git a/drivers/yunpan360/upload_test.go b/drivers/yunpan360/upload_test.go new file mode 100644 index 00000000000..092570ff5e8 --- /dev/null +++ b/drivers/yunpan360/upload_test.go @@ -0,0 +1,25 @@ +package yunpan360 + +import ( + "bytes" + "testing" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +func TestBuildUploadPlanComputesFileSHA1AndMD5(t *testing.T) { + d := &Yunpan360{} + data := []byte("hello yunpan") + file := model.NewNopMFile(bytes.NewReader(data)) + plan, err := d.buildUploadPlan(t.Context(), file, "/hello.txt", int64(len(data)), time.Unix(1700000000, 0), nil) + if err != nil { + t.Fatalf("buildUploadPlan() error = %v", err) + } + if plan.FileSHA1 != "254ec33af17332a3964145f8c6a3dc12833c7ea2" { + t.Fatalf("FileSHA1 = %q", plan.FileSHA1) + } + if plan.FileSum != "fefeffa5b6ae9f39851050b44cacfcb1" { + t.Fatalf("FileSum = %q", plan.FileSum) + } +} diff --git a/drivers/yunpan360/util.go b/drivers/yunpan360/util.go new file mode 100644 index 00000000000..2718f9c99cc --- /dev/null +++ b/drivers/yunpan360/util.go @@ -0,0 +1,909 @@ +package yunpan360 + +import ( + "context" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "html" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + baseURL = "https://www.yunpan.com" + indexPath = "/file/index" + listPath = "/file/list" + downloadPath = "/file/download" + openAPIProdURL = "https://openapi.eyun.360.cn/intf.php" + openAPITestURL = "https://qaopen.eyun.360.cn/intf.php" + openAPIHGTestURL = "https://hg-openapi.eyun.360.cn/intf.php" +) + +func (d *Yunpan360) listPage(ctx context.Context, dirPath string, page, pageSize int) (ListResp, error) { + if d.authMode() == authTypeAPIKey { + return d.listOpenPage(ctx, dirPath, page, pageSize) + } + return d.listCookiePage(ctx, dirPath, page, pageSize) +} + +func (d *Yunpan360) listCookiePage(ctx context.Context, dirPath string, page, pageSize int) (*CookieListResp, error) { + var resp CookieListResp + err := d.cookieRequestForm(ctx, listPath, map[string]string{ + "path": requestPath(dirPath), + "page": strconv.Itoa(page), + "page_size": strconv.Itoa(pageSize), + "order": requestOrder(d.OrderDirection), + "field": "file_name", + "focus_nid": "0", + }, &resp) + if err != nil { + return nil, err + } + d.cacheCookieDownloadSession(resp.GetOwnerQID(), resp.Token) + return &resp, nil +} + +func (d *Yunpan360) cookieRequestForm(ctx context.Context, apiPath string, form map[string]string, out interface{}) error { + req := base.RestyClient.R(). + SetContext(ctx). + SetHeaders(map[string]string{ + "Accept": "text/javascript, text/html, application/xml, text/xml, */*", + "Content-Type": "application/x-www-form-urlencoded", + "Cookie": d.Cookie, + "Origin": baseURL, + "Referer": baseURL + "/file/index", + "X-Requested-With": "XMLHttpRequest", + }). + SetFormData(form) + + res, err := req.Execute(http.MethodPost, baseURL+apiPath) + if err != nil { + return err + } + + var baseResp BaseResp + if err := utils.Json.Unmarshal(res.Body(), &baseResp); err != nil { + return err + } + if baseResp.Errno != 0 { + if baseResp.Errmsg == "" { + return fmt.Errorf("yunpan request failed: errno=%d", baseResp.Errno) + } + return errors.New(baseResp.Errmsg) + } + if out == nil { + return nil + } + return utils.Json.Unmarshal(res.Body(), out) +} + +func requestPath(dirPath string) string { + path := normalizeRemotePath(dirPath) + if path == "" { + return "/" + } + return path +} + +func requestOrder(order string) string { + if strings.EqualFold(order, "desc") { + return "desc" + } + return "asc" +} + +func (d *Yunpan360) cookiePage(ctx context.Context, pagePath string) ([]byte, error) { + req := base.RestyClient.R(). + SetContext(ctx). + SetHeaders(map[string]string{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Cookie": d.Cookie, + "Referer": baseURL + indexPath, + }) + res, err := req.Get(baseURL + pagePath) + if err != nil { + return nil, err + } + return res.Body(), nil +} + +func openAPIURL(env string) string { + switch env { + case "test": + return openAPITestURL + case "hgtest": + return openAPIHGTestURL + default: + return openAPIProdURL + } +} + +func openClientSecretForEnv(env string) string { + if env == "test" { + return openClientSecretQA + } + return openClientSecret +} + +func phpQueryEscape(raw string) string { + escaped := url.QueryEscape(raw) + return strings.ReplaceAll(escaped, "~", "%7E") +} + +func openSign(accessToken, qid, method string, extra map[string]string) string { + params := map[string]string{ + "access_token": accessToken, + "method": method, + "qid": qid, + } + for key, value := range extra { + if value != "" { + params[key] = value + } + } + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sortStrings(keys) + pairs := make([]string, 0, len(keys)) + for _, key := range keys { + pairs = append(pairs, key+"="+phpQueryEscape(params[key])) + } + sum := md5.Sum([]byte(strings.Join(pairs, "&") + openSignSecret)) + return hex.EncodeToString(sum[:]) +} + +func sortStrings(values []string) { + for i := 0; i < len(values); i++ { + for j := i + 1; j < len(values); j++ { + if values[j] < values[i] { + values[i], values[j] = values[j], values[i] + } + } + } +} + +func (d *Yunpan360) getOpenAuth(ctx context.Context) (*OpenAuthInfo, error) { + d.authMu.Lock() + defer d.authMu.Unlock() + + if d.cachedOpenAuth != nil && time.Now().Before(d.openAuthExpire) { + auth := *d.cachedOpenAuth + return &auth, nil + } + + reqURL := openAPIURL(d.EcsEnv) + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Accept", "application/json"). + SetHeader("api_key", d.APIKey). + SetQueryParams(map[string]string{ + "method": "Oauth.getAccessTokenByApiKey", + "client_id": openClientID, + "client_secret": openClientSecretForEnv(d.EcsEnv), + "grant_type": "authorization_code", + "sub_channel": d.SubChannel, + "api_key": d.APIKey, + }) + + res, err := req.Get(reqURL) + if err != nil { + return nil, err + } + + var resp OpenAuthResp + if err := utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + if resp.Errno != 0 { + if resp.Errmsg == "" { + return nil, fmt.Errorf("yunpan auth failed: errno=%d", resp.Errno) + } + return nil, errors.New(resp.Errmsg) + } + + auth := &OpenAuthInfo{ + AccessToken: resp.Data.AccessToken, + Qid: resp.Data.Qid, + Token: resp.Data.Token, + SubChannel: d.SubChannel, + } + d.cachedOpenAuth = auth + d.openAuthExpire = time.Now().Add(50 * time.Minute) + + copied := *auth + return &copied, nil +} + +func (d *Yunpan360) openBaseParams(auth *OpenAuthInfo, method string, signParams map[string]string, withSign bool) map[string]string { + params := map[string]string{ + "method": method, + "access_token": auth.AccessToken, + "qid": auth.Qid, + "sub_channel": auth.SubChannel, + } + if withSign { + params["sign"] = openSign(auth.AccessToken, auth.Qid, method, signParams) + } else { + params["sign"] = "" + } + return params +} + +func (d *Yunpan360) openGET(ctx context.Context, method string, signParams map[string]string, query map[string]string, out interface{}, withSign bool) error { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return err + } + params := d.openBaseParams(auth, method, signParams, withSign) + for key, value := range query { + params[key] = value + } + + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Access-Token", auth.AccessToken). + SetQueryParams(params) + res, err := req.Get(openAPIURL(d.EcsEnv)) + if err != nil { + return err + } + return decodeBaseResp(res.Body(), out) +} + +func (d *Yunpan360) openPOST(ctx context.Context, method string, signParams map[string]string, query, body map[string]string, out interface{}, withSign bool) error { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return err + } + queryParams := map[string]string{} + for key, value := range query { + queryParams[key] = value + } + bodyParams := map[string]string{} + for key, value := range body { + bodyParams[key] = value + } + + baseParams := d.openBaseParams(auth, method, signParams, withSign) + if len(queryParams) == 0 { + bodyParams = mergeStringMaps(baseParams, bodyParams) + } else { + queryParams = mergeStringMaps(baseParams, queryParams) + } + + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Access-Token", auth.AccessToken). + SetHeader("Content-Type", "application/x-www-form-urlencoded") + if len(queryParams) > 0 { + req.SetQueryParams(queryParams) + } + if len(bodyParams) > 0 { + req.SetFormData(bodyParams) + } + res, err := req.Post(openAPIURL(d.EcsEnv)) + if err != nil { + return err + } + return decodeBaseResp(res.Body(), out) +} + +func decodeBaseResp(body []byte, out interface{}) error { + var baseResp BaseResp + if err := utils.Json.Unmarshal(body, &baseResp); err != nil { + return err + } + if baseResp.Errno != 0 { + if baseResp.Errmsg == "" { + return fmt.Errorf("yunpan request failed: errno=%d", baseResp.Errno) + } + return errors.New(baseResp.Errmsg) + } + if out == nil { + return nil + } + return utils.Json.Unmarshal(body, out) +} + +func mergeStringMaps(baseMap, extra map[string]string) map[string]string { + merged := map[string]string{} + for key, value := range baseMap { + merged[key] = value + } + for key, value := range extra { + merged[key] = value + } + return merged +} + +func (d *Yunpan360) cookieDownloadURL(ctx context.Context, file model.Obj) (*CookieDownloadResp, error) { + resp, err := d.cookieDownloadURLOnce(ctx, file, false) + if err == nil { + return resp, nil + } + + d.invalidateCookieDownloadSession() + return d.cookieDownloadURLOnce(ctx, file, true) +} + +func (d *Yunpan360) cookieDownloadURLOnce(ctx context.Context, file model.Obj, refresh bool) (*CookieDownloadResp, error) { + nid := strings.TrimSpace(file.GetID()) + if nid == "" { + return nil, errors.New("missing file id") + } + + fname := normalizeRemotePath(file.GetPath()) + if fname == "" { + return nil, errors.New("missing file path") + } + + ownerQID, token, err := d.resolveCookieDownloadParams(ctx, file, refresh) + if err != nil { + return nil, err + } + + var resp CookieDownloadResp + err = d.cookieRequestForm(ctx, downloadPath, map[string]string{ + "nid": nid, + "fname": fname, + "owner_qid": ownerQID, + "token": token, + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) cookieRename(ctx context.Context, srcObj model.Obj, newName string) error { + path := normalizeRemotePath(srcObj.GetPath()) + if path == "" { + return errors.New("missing object path") + } + nid := strings.TrimSpace(srcObj.GetID()) + if nid == "" { + return errors.New("missing object id") + } + + ownerQID, err := d.resolveCookieOwnerQID(ctx, srcObj, false) + if err != nil { + return err + } + + return d.cookieRequestForm(ctx, "/file/rename", map[string]string{ + "path": path, + "nid": nid, + "newpath": strings.TrimSuffix(strings.TrimSpace(newName), "/"), + "owner_qid": ownerQID, + }, nil) +} + +func (d *Yunpan360) resolveCookieDownloadParams(ctx context.Context, file model.Obj, refresh bool) (string, string, error) { + ownerQID := sanitizeOwnerQID(d.OwnerQID) + token := strings.TrimSpace(d.DownloadToken) + + if obj, ok := file.(*YunpanObject); ok { + ownerQID = firstNonEmpty(sanitizeOwnerQID(obj.OwnerQID), ownerQID) + token = firstNonEmpty(strings.TrimSpace(obj.DownloadToken), token) + } + + ownerQID = firstNonEmpty(ownerQID, + sanitizeOwnerQID(getCookieValue(d.Cookie, "owner_qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "ownerQid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "QID")), + ) + token = firstNonEmpty(token, + getCookieValue(d.Cookie, "download_token"), + getCookieValue(d.Cookie, "token"), + getCookieValue(d.Cookie, "Token"), + ) + + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + return ownerQID, token, nil + } + + if !refresh { + if cached := d.getCachedCookieDownloadSession(); cached != nil { + ownerQID = firstNonEmpty(ownerQID, cached.OwnerQID) + token = firstNonEmpty(token, cached.Token) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + return ownerQID, token, nil + } + } + + resp, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + if err == nil && resp != nil { + ownerQID = firstNonEmpty(ownerQID, resp.GetOwnerQID()) + token = firstNonEmpty(token, strings.TrimSpace(resp.Token)) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + d.cacheCookieDownloadSession(ownerQID, token) + return ownerQID, token, nil + } + + pageSession, err := d.getCookieDownloadSessionFromPage(ctx) + if err == nil && pageSession != nil { + ownerQID = firstNonEmpty(ownerQID, pageSession.OwnerQID) + token = firstNonEmpty(token, pageSession.Token) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID == "" || token == "" { + return "", "", errors.New("missing owner_qid or download_token for cookie mode") + } + + d.cacheCookieDownloadSession(ownerQID, token) + return ownerQID, token, nil +} + +func (d *Yunpan360) resolveCookieOwnerQID(ctx context.Context, file model.Obj, refresh bool) (string, error) { + ownerQID := sanitizeOwnerQID(d.OwnerQID) + + if obj, ok := file.(*YunpanObject); ok { + ownerQID = firstNonEmpty(sanitizeOwnerQID(obj.OwnerQID), ownerQID) + } + + ownerQID = firstNonEmpty(ownerQID, + sanitizeOwnerQID(getCookieValue(d.Cookie, "owner_qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "ownerQid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "QID")), + ) + if ownerQID != "" { + return ownerQID, nil + } + + if !refresh { + if cached := d.getCachedCookieDownloadSession(); cached != nil { + ownerQID = firstNonEmpty(ownerQID, cached.OwnerQID) + } + if ownerQID != "" { + return ownerQID, nil + } + } + + resp, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + if err == nil && resp != nil { + ownerQID = firstNonEmpty(ownerQID, resp.GetOwnerQID()) + } + if ownerQID != "" { + return ownerQID, nil + } + + pageSession, err := d.getCookieDownloadSessionFromPage(ctx) + if err == nil && pageSession != nil { + ownerQID = firstNonEmpty(ownerQID, pageSession.OwnerQID) + } + if ownerQID == "" { + return "", errors.New("missing owner_qid for cookie mode") + } + return ownerQID, nil +} + +func (d *Yunpan360) getCookieDownloadSessionFromPage(ctx context.Context) (*CookieDownloadSession, error) { + body, err := d.cookiePage(ctx, indexPath) + if err != nil { + return nil, err + } + session := parseCookieDownloadSessionFromText(string(body)) + if session == nil { + return nil, errors.New("failed to parse cookie download session from page") + } + d.cacheCookieSession(session) + return session, nil +} + +func (d *Yunpan360) getCachedCookieDownloadSession() *CookieDownloadSession { + d.authMu.Lock() + defer d.authMu.Unlock() + + if d.cachedCookieSession == nil || time.Now().After(d.cookieSessionExpire) { + return nil + } + session := *d.cachedCookieSession + return &session +} + +func (d *Yunpan360) cacheCookieDownloadSession(ownerQID, token string) { + d.cacheCookieSession(&CookieDownloadSession{ + OwnerQID: ownerQID, + Token: token, + }) +} + +func (d *Yunpan360) cacheCookieSession(session *CookieDownloadSession) { + if session == nil { + return + } + + cached := &CookieDownloadSession{ + OwnerQID: sanitizeOwnerQID(session.OwnerQID), + Token: strings.TrimSpace(session.Token), + } + if cached.OwnerQID == "" && cached.Token != "" { + cached.OwnerQID = ownerQIDFromToken(cached.Token) + } + if cached.OwnerQID == "" || cached.Token == "" { + return + } + + d.authMu.Lock() + defer d.authMu.Unlock() + + d.cachedCookieSession = cached + d.cookieSessionExpire = time.Now().Add(10 * time.Minute) +} + +func (d *Yunpan360) invalidateCookieDownloadSession() { + d.authMu.Lock() + defer d.authMu.Unlock() + + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} +} + +func getCookieValue(rawCookie, name string) string { + for _, item := range strings.Split(rawCookie, ";") { + part := strings.TrimSpace(item) + if part == "" { + continue + } + key, value, ok := strings.Cut(part, "=") + if !ok || key != name { + continue + } + value = strings.TrimSpace(value) + value = strings.Trim(value, "\"") + unescaped, err := url.QueryUnescape(value) + if err == nil { + return strings.TrimSpace(unescaped) + } + return value + } + return "" +} + +func ownerQIDFromToken(token string) string { + parts := strings.Split(strings.TrimSpace(token), ".") + if len(parts) < 4 { + return "" + } + qid := strings.TrimSpace(parts[3]) + for _, ch := range qid { + if ch < '0' || ch > '9' { + return "" + } + } + return qid +} + +func sanitizeOwnerQID(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "0" { + return "" + } + return raw +} + +func parseCookieDownloadSessionFromText(text string) *CookieDownloadSession { + token := extractFirstMatch(text, + `(?i)["']download_token["']\s*[:=]\s*["']([^"'<>]+)["']`, + `(?i)["']token["']\s*[:=]\s*["']([^"'<>]+)["']`, + `(?i)\btoken\s*[:=]\s*["']([^"'<>]+)["']`, + ) + ownerQID := extractFirstMatch(text, + `(?i)["']owner_qid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)["']ownerQid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)["']qid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)\bowner_qid\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)\bqid\s*[:=]\s*["']?([0-9]+)["']?`, + ) + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID == "" || token == "" { + return nil + } + return &CookieDownloadSession{ + OwnerQID: ownerQID, + Token: token, + } +} + +func extractFirstMatch(text string, patterns ...string) string { + return extractFirstValidatedMatch(nil, text, patterns...) +} + +func extractFirstValidatedMatch(validate func(string) bool, text string, patterns ...string) string { + for _, pattern := range patterns { + re := regexp.MustCompile(pattern) + for _, matches := range re.FindAllStringSubmatch(text, -1) { + if len(matches) < 2 { + continue + } + value := html.UnescapeString(strings.TrimSpace(matches[1])) + value = strings.Trim(value, "\"'") + if value == "" { + continue + } + if validate == nil || validate(value) { + return value + } + } + } + return "" +} + +func (d *Yunpan360) listOpenPage(ctx context.Context, dirPath string, page, pageSize int) (*OpenListResp, error) { + var resp OpenListResp + path := ensureDirAPIPath(dirPath) + params := map[string]string{ + "path": path, + "page": strconv.Itoa(page), + "page_size": strconv.Itoa(pageSize), + } + err := d.openGET(ctx, "File.getList", params, params, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openUserInfo(ctx context.Context) (*OpenUserInfoResp, error) { + var resp OpenUserInfoResp + err := d.openGET(ctx, "User.getUserDetail", nil, nil, &resp, false) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openDownloadURL(ctx context.Context, file model.Obj) (*OpenDownloadResp, error) { + var resp OpenDownloadResp + signParams := map[string]string{} + body := map[string]string{} + + if file.GetPath() != "" { + signParams["fpath"] = normalizeRemotePath(file.GetPath()) + body["fpath"] = signParams["fpath"] + } else if file.GetID() != "" { + signParams["nid"] = file.GetID() + body["nid"] = file.GetID() + } else { + return nil, errors.New("missing file path and id") + } + + err := d.openPOST(ctx, "MCP.getDownLoadUrl", signParams, nil, body, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) cookieMakeDir(ctx context.Context, fullPath string) (*CookieMkdirResp, error) { + var resp CookieMkdirResp + body := map[string]string{ + "path": ensureDirAPIPath(fullPath), + "owner_qid": "0", + } + err := d.cookieRequestForm(ctx, "/file/mkdir", body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openMakeDir(ctx context.Context, fullPath string) (*OpenMkdirResp, error) { + var resp OpenMkdirResp + body := map[string]string{"fname": ensureDirAPIPath(fullPath)} + err := d.openPOST(ctx, "File.mkdir", body, nil, body, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openRename(ctx context.Context, srcName, newName string) error { + signParams := map[string]string{ + "src_name": srcName, + "new_name": newName, + } + return d.openPOST(ctx, "File.rename", signParams, nil, signParams, nil, true) +} + +func (d *Yunpan360) cookieMove(ctx context.Context, srcPath, dstPath string) error { + var resp CookieMoveResp + body := map[string]string{ + "path[]": srcPath, + "newpath": ensureDirAPIPath(dstPath), + } + if err := d.cookieRequestForm(ctx, "/file/move", body, &resp); err != nil { + return err + } + if !resp.Data.IsAsync { + return nil + } + return d.waitCookieAsyncTask(ctx, resp.Data.TaskID) +} + +func (d *Yunpan360) cookieRecycle(ctx context.Context, obj model.Obj) error { + path := apiPathForObj(obj) + if path == "" { + return errors.New("missing object path") + } + ownerQID, err := d.resolveCookieOwnerQID(ctx, obj, false) + if err != nil { + return err + } + + var resp CookieRecycleResp + if err := d.cookieRequestForm(ctx, "/file/recycle", map[string]string{ + "path[]": path, + "owner_qid": ownerQID, + }, &resp); err != nil { + return err + } + if !resp.Data.IsAsync { + return nil + } + return d.waitCookieAsyncTask(ctx, resp.Data.TaskID, 3008) +} + +func (d *Yunpan360) cookieAsyncQuery(ctx context.Context, taskID string) (*CookieAsyncQueryResp, error) { + var resp CookieAsyncQueryResp + err := d.cookieRequestForm(ctx, "/async/query", map[string]string{ + "task_id": strings.TrimSpace(taskID), + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) waitCookieAsyncTask(ctx context.Context, taskID string, toleratedErrnos ...int) error { + taskID = strings.TrimSpace(taskID) + if taskID == "" { + return nil + } + + tolerated := map[int]struct{}{} + for _, errno := range toleratedErrnos { + tolerated[errno] = struct{}{} + } + + for attempt := 0; attempt < 15; attempt++ { + resp, err := d.cookieAsyncQuery(ctx, taskID) + if err == nil && resp != nil { + if task, ok := resp.Data[taskID]; ok { + done, taskErr := checkCookieAsyncTask(task, tolerated) + if done { + return taskErr + } + } + } + if attempt == 14 { + break + } + if err := sleepWithContext(ctx, 300*time.Millisecond); err != nil { + return err + } + } + + // Keep prior behavior when the async task is still pending after the probe window. + return nil +} + +func checkCookieAsyncTask(task CookieAsyncTask, toleratedErrnos map[int]struct{}) (bool, error) { + if task.Status != 10 { + return false, nil + } + if task.Errno == 0 { + return true, nil + } + if _, ok := toleratedErrnos[task.Errno]; ok { + return true, nil + } + if strings.TrimSpace(task.Errstr) != "" { + return true, errors.New(task.Errstr) + } + if strings.TrimSpace(task.Action) != "" { + return true, fmt.Errorf("yunpan async task %s failed: errno=%d", task.Action, task.Errno) + } + return true, fmt.Errorf("yunpan async task failed: errno=%d", task.Errno) +} + +func (d *Yunpan360) openMove(ctx context.Context, srcName, dstPath string) error { + signParams := map[string]string{ + "src_name": srcName, + "new_name": dstPath, + } + return d.openPOST(ctx, "File.move", signParams, nil, signParams, nil, true) +} + +func (d *Yunpan360) openDelete(ctx context.Context, targetPath string) error { + return d.openPOST(ctx, "File.delete", nil, nil, map[string]string{ + "fname": targetPath, + }, nil, true) +} + +func apiPathForObj(obj model.Obj) string { + if obj.IsDir() { + return ensureDirAPIPath(obj.GetPath()) + } + return normalizeRemotePath(obj.GetPath()) +} + +func ensureDirSuffix(name string) string { + name = strings.TrimSpace(name) + if name == "" || strings.HasSuffix(name, "/") { + return name + } + return name + "/" +} + +func ensureDirAPIPath(p string) string { + p = normalizeRemotePath(p) + if p == "" || p == "/" { + return "/" + } + return p + "/" +} + +func cloneObj(src model.Obj, newPath, newName string) model.Obj { + obj := model.Object{ + ID: src.GetID(), + Path: normalizeRemotePath(newPath), + Name: newName, + Size: src.GetSize(), + Modified: src.ModTime(), + Ctime: src.CreateTime(), + IsFolder: src.IsDir(), + HashInfo: src.GetHash(), + } + if raw, ok := src.(*YunpanObject); ok { + return &YunpanObject{ + Object: obj, + Thumbnail: raw.Thumbnail, + OwnerQID: raw.OwnerQID, + DownloadToken: raw.DownloadToken, + } + } + return &obj +} + +func absoluteURL(raw string) string { + if raw == "" { + return "" + } + if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") { + return raw + } + if strings.HasPrefix(raw, "/") { + return baseURL + raw + } + return baseURL + "/" + raw +} diff --git a/drivers/yunpan360/util_test.go b/drivers/yunpan360/util_test.go new file mode 100644 index 00000000000..987ca7f928c --- /dev/null +++ b/drivers/yunpan360/util_test.go @@ -0,0 +1,109 @@ +package yunpan360 + +import "testing" + +func TestOwnerQIDFromToken(t *testing.T) { + token := "3061246061.9.cb696851.679627082.16700351802400171.1774866942" + if got := ownerQIDFromToken(token); got != "679627082" { + t.Fatalf("ownerQIDFromToken() = %q, want %q", got, "679627082") + } +} + +func TestParseCookieDownloadSessionFromText(t *testing.T) { + page := `window.__NUXT__={"token":"3061246061.9.cb696851.679627082.16700351802400171.1774866942","owner_qid":"679627082"}` + session := parseCookieDownloadSessionFromText(page) + if session == nil { + t.Fatal("parseCookieDownloadSessionFromText() = nil") + } + if session.OwnerQID != "679627082" { + t.Fatalf("OwnerQID = %q, want %q", session.OwnerQID, "679627082") + } + if session.Token == "" { + t.Fatal("Token should not be empty") + } +} + +func TestCookieListRespObjectsCarrySession(t *testing.T) { + resp := CookieListResp{ + Token: "3061246061.9.cb696851.679627082.16700351802400171.1774866942", + OwnerQid: "679627082", + Data: []ListItem{{ + NID: "17748755101917705", + FileName: "统计4.mp4", + FilePath: "/统计4.mp4", + FileSize: "1024", + }}, + } + + objs := resp.Objects("/") + if len(objs) != 1 { + t.Fatalf("len(objs) = %d, want 1", len(objs)) + } + + obj, ok := objs[0].(*YunpanObject) + if !ok { + t.Fatalf("object type = %T, want *YunpanObject", objs[0]) + } + if obj.OwnerQID != "679627082" { + t.Fatalf("OwnerQID = %q, want %q", obj.OwnerQID, "679627082") + } + if obj.DownloadToken == "" { + t.Fatal("DownloadToken should not be empty") + } +} + +func TestResolveCookieOwnerQIDFromObject(t *testing.T) { + d := &Yunpan360{} + obj := &YunpanObject{ + OwnerQID: "679627082", + } + got, err := d.resolveCookieOwnerQID(t.Context(), obj, false) + if err != nil { + t.Fatalf("resolveCookieOwnerQID() error = %v", err) + } + if got != "679627082" { + t.Fatalf("resolveCookieOwnerQID() = %q, want %q", got, "679627082") + } +} + +func TestCheckCookieAsyncTaskSuccess(t *testing.T) { + done, err := checkCookieAsyncTask(CookieAsyncTask{ + Status: 10, + Action: "File.move", + Errno: 0, + }, nil) + if !done { + t.Fatal("checkCookieAsyncTask() should be done") + } + if err != nil { + t.Fatalf("checkCookieAsyncTask() error = %v", err) + } +} + +func TestCheckCookieAsyncTaskPending(t *testing.T) { + done, err := checkCookieAsyncTask(CookieAsyncTask{ + Status: 1, + Action: "File.move", + }, nil) + if done { + t.Fatal("checkCookieAsyncTask() should be pending") + } + if err != nil { + t.Fatalf("checkCookieAsyncTask() error = %v", err) + } +} + +func TestCheckCookieAsyncTaskIgnoredErrno(t *testing.T) { + done, err := checkCookieAsyncTask(CookieAsyncTask{ + Status: 10, + Action: "File.recycle", + Errno: 3008, + Errstr: "文件（夹）已移动或删除！", + }, map[int]struct{}{3008: {}}) + if !done { + t.Fatal("checkCookieAsyncTask() should be done") + } + if err != nil { + t.Fatalf("checkCookieAsyncTask() error = %v", err) + } +} From 0fe31e042f701146734f572c32aa98fdb40e482a Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 31 Mar 2026 16:28:19 +0800 Subject: [PATCH 109/133] chore: remove yunpan360 driver tests --- drivers/yunpan360/upload_test.go | 25 ------- drivers/yunpan360/util_test.go | 109 ------------------------------- 2 files changed, 134 deletions(-) delete mode 100644 drivers/yunpan360/upload_test.go delete mode 100644 drivers/yunpan360/util_test.go diff --git a/drivers/yunpan360/upload_test.go b/drivers/yunpan360/upload_test.go deleted file mode 100644 index 092570ff5e8..00000000000 --- a/drivers/yunpan360/upload_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package yunpan360 - -import ( - "bytes" - "testing" - "time" - - "github.com/alist-org/alist/v3/internal/model" -) - -func TestBuildUploadPlanComputesFileSHA1AndMD5(t *testing.T) { - d := &Yunpan360{} - data := []byte("hello yunpan") - file := model.NewNopMFile(bytes.NewReader(data)) - plan, err := d.buildUploadPlan(t.Context(), file, "/hello.txt", int64(len(data)), time.Unix(1700000000, 0), nil) - if err != nil { - t.Fatalf("buildUploadPlan() error = %v", err) - } - if plan.FileSHA1 != "254ec33af17332a3964145f8c6a3dc12833c7ea2" { - t.Fatalf("FileSHA1 = %q", plan.FileSHA1) - } - if plan.FileSum != "fefeffa5b6ae9f39851050b44cacfcb1" { - t.Fatalf("FileSum = %q", plan.FileSum) - } -} diff --git a/drivers/yunpan360/util_test.go b/drivers/yunpan360/util_test.go deleted file mode 100644 index 987ca7f928c..00000000000 --- a/drivers/yunpan360/util_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package yunpan360 - -import "testing" - -func TestOwnerQIDFromToken(t *testing.T) { - token := "3061246061.9.cb696851.679627082.16700351802400171.1774866942" - if got := ownerQIDFromToken(token); got != "679627082" { - t.Fatalf("ownerQIDFromToken() = %q, want %q", got, "679627082") - } -} - -func TestParseCookieDownloadSessionFromText(t *testing.T) { - page := `window.__NUXT__={"token":"3061246061.9.cb696851.679627082.16700351802400171.1774866942","owner_qid":"679627082"}` - session := parseCookieDownloadSessionFromText(page) - if session == nil { - t.Fatal("parseCookieDownloadSessionFromText() = nil") - } - if session.OwnerQID != "679627082" { - t.Fatalf("OwnerQID = %q, want %q", session.OwnerQID, "679627082") - } - if session.Token == "" { - t.Fatal("Token should not be empty") - } -} - -func TestCookieListRespObjectsCarrySession(t *testing.T) { - resp := CookieListResp{ - Token: "3061246061.9.cb696851.679627082.16700351802400171.1774866942", - OwnerQid: "679627082", - Data: []ListItem{{ - NID: "17748755101917705", - FileName: "统计4.mp4", - FilePath: "/统计4.mp4", - FileSize: "1024", - }}, - } - - objs := resp.Objects("/") - if len(objs) != 1 { - t.Fatalf("len(objs) = %d, want 1", len(objs)) - } - - obj, ok := objs[0].(*YunpanObject) - if !ok { - t.Fatalf("object type = %T, want *YunpanObject", objs[0]) - } - if obj.OwnerQID != "679627082" { - t.Fatalf("OwnerQID = %q, want %q", obj.OwnerQID, "679627082") - } - if obj.DownloadToken == "" { - t.Fatal("DownloadToken should not be empty") - } -} - -func TestResolveCookieOwnerQIDFromObject(t *testing.T) { - d := &Yunpan360{} - obj := &YunpanObject{ - OwnerQID: "679627082", - } - got, err := d.resolveCookieOwnerQID(t.Context(), obj, false) - if err != nil { - t.Fatalf("resolveCookieOwnerQID() error = %v", err) - } - if got != "679627082" { - t.Fatalf("resolveCookieOwnerQID() = %q, want %q", got, "679627082") - } -} - -func TestCheckCookieAsyncTaskSuccess(t *testing.T) { - done, err := checkCookieAsyncTask(CookieAsyncTask{ - Status: 10, - Action: "File.move", - Errno: 0, - }, nil) - if !done { - t.Fatal("checkCookieAsyncTask() should be done") - } - if err != nil { - t.Fatalf("checkCookieAsyncTask() error = %v", err) - } -} - -func TestCheckCookieAsyncTaskPending(t *testing.T) { - done, err := checkCookieAsyncTask(CookieAsyncTask{ - Status: 1, - Action: "File.move", - }, nil) - if done { - t.Fatal("checkCookieAsyncTask() should be pending") - } - if err != nil { - t.Fatalf("checkCookieAsyncTask() error = %v", err) - } -} - -func TestCheckCookieAsyncTaskIgnoredErrno(t *testing.T) { - done, err := checkCookieAsyncTask(CookieAsyncTask{ - Status: 10, - Action: "File.recycle", - Errno: 3008, - Errstr: "文件（夹）已移动或删除！", - }, map[int]struct{}{3008: {}}) - if !done { - t.Fatal("checkCookieAsyncTask() should be done") - } - if err != nil { - t.Fatalf("checkCookieAsyncTask() error = %v", err) - } -} From 97a4bb39a059cdc0ee147b6f4159644730322e39 Mon Sep 17 00:00:00 2001 From: icebear0828 Date: Sun, 5 Apr 2026 12:17:41 -0500 Subject: [PATCH 110/133] fix: enable GFM extension for markdown rendering in proxy mode When `filter_readme_scripts` is enabled, markdown files served through the proxy are converted to HTML using goldmark. However, the default goldmark converter does not include GFM (GitHub Flavored Markdown) extensions, causing tables, strikethrough, and other GFM syntax to be rendered as plain text instead of proper HTML elements. This adds `extension.GFM` to the goldmark converter so that GFM tables, strikethrough, autolinks, and task lists are correctly converted to HTML. --- server/handles/down.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/handles/down.go b/server/handles/down.go index 37439f00bb3..680c33f54b1 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -18,6 +18,7 @@ import ( "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" ) func Down(c *gin.Context) { @@ -151,7 +152,8 @@ func localProxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange boo } var html bytes.Buffer - if err = goldmark.Convert(buf.Bytes(), &html); err != nil { + md := goldmark.New(goldmark.WithExtensions(extension.GFM)) + if err = md.Convert(buf.Bytes(), &html); err != nil { err = fmt.Errorf("markdown conversion failed: %w", err) } else { buf.Reset() From 9b697e452581b4da75aca0f1385f56511d0717d8 Mon Sep 17 00:00:00 2001 From: darkizone Date: Sun, 12 Apr 2026 20:05:31 +0100 Subject: [PATCH 111/133] feat(driver): add Darkibox driver Add driver for Darkibox (https://darkibox.com/), an XFileSharing-based video hosting platform. Supported operations: - List files and folders (with pagination) - Upload files (via upload server) - Download files (via direct link API with quality selection) - Create and delete folders - Delete and move files - Account validation on init Closes #8079 Co-Authored-By: Claude Opus 4.6 (1M context) --- drivers/all.go | 1 + drivers/darkibox/driver.go | 299 +++++++++++++++++++++++++++++++++++++ drivers/darkibox/meta.go | 27 ++++ drivers/darkibox/types.go | 79 ++++++++++ drivers/darkibox/util.go | 88 +++++++++++ 5 files changed, 494 insertions(+) create mode 100644 drivers/darkibox/driver.go create mode 100644 drivers/darkibox/meta.go create mode 100644 drivers/darkibox/types.go create mode 100644 drivers/darkibox/util.go diff --git a/drivers/all.go b/drivers/all.go index 1f096a86224..e62f67fbd85 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -27,6 +27,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" _ "github.com/alist-org/alist/v3/drivers/crypt" + _ "github.com/alist-org/alist/v3/drivers/darkibox" _ "github.com/alist-org/alist/v3/drivers/doubao" _ "github.com/alist-org/alist/v3/drivers/doubao_new" _ "github.com/alist-org/alist/v3/drivers/doubao_share" diff --git a/drivers/darkibox/driver.go b/drivers/darkibox/driver.go new file mode 100644 index 00000000000..0e3f2e5065b --- /dev/null +++ b/drivers/darkibox/driver.go @@ -0,0 +1,299 @@ +package darkibox + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" +) + +type Darkibox struct { + model.Storage + Addition +} + +func (d *Darkibox) Config() driver.Config { + return config +} + +func (d *Darkibox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Darkibox) Init(ctx context.Context) error { + if d.APIKey == "" { + return fmt.Errorf("API key is required") + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + + // Verify API key by calling account/info + var account accountInfoResult + if err := d.callAPI(ctx, "/account/info", nil, &account); err != nil { + return fmt.Errorf("failed to verify API key: %w", err) + } + + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Darkibox) Drop(ctx context.Context) error { + return nil +} + +func (d *Darkibox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := d.RootFolderID + if dir.GetID() != "" { + folderID = folderIDFromObjID(dir.GetID()) + } + + var objects []model.Obj + + // List sub-folders via /api/folder/list + var folders folderListResult + if err := d.callAPI(ctx, "/folder/list", map[string]string{ + "fld_id": fldIDStr(folderID), + }, &folders); err != nil { + return nil, fmt.Errorf("list folders failed: %w", err) + } + for _, f := range folders.Folders { + objects = append(objects, &model.Object{ + ID: encodeFolderID(f.FldID), + Name: f.Name, + IsFolder: true, + }) + } + + // List files via /api/file/list (paginated) + page := 1 + for { + var files fileListResult + if err := d.callAPI(ctx, "/file/list", map[string]string{ + "fld_id": fldIDStr(folderID), + "per_page": "200", + "page": strconv.Itoa(page), + }, &files); err != nil { + return nil, fmt.Errorf("list files failed: %w", err) + } + + for _, f := range files.Files { + modified := time.Now() + if f.Uploaded != "" { + if t, err := time.Parse("2006-01-02 15:04:05", f.Uploaded); err == nil { + modified = t + } + } + name := f.Name + if name == "" { + name = f.Title + } + objects = append(objects, &model.Object{ + ID: encodeFileID(f.FileCode), + Name: name, + Size: f.Size, + Modified: modified, + IsFolder: false, + }) + } + + // Check if there are more pages + if len(files.Files) < 200 { + break + } + page++ + } + + return objects, nil +} + +func (d *Darkibox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + + fileCode := fileCodeFromObjID(file.GetID()) + if fileCode == "" { + return nil, fmt.Errorf("empty file code") + } + + var result directLinkResult + if err := d.callAPI(ctx, "/file/direct_link", map[string]string{ + "file_code": fileCode, + }, &result); err != nil { + return nil, fmt.Errorf("failed to get direct link: %w", err) + } + + // Find the original quality version, fall back to first available + var dlURL string + for _, v := range result.Versions { + if v.Name == "o" { + dlURL = v.URL + break + } + } + if dlURL == "" && len(result.Versions) > 0 { + dlURL = result.Versions[0].URL + } + if dlURL == "" { + return nil, fmt.Errorf("no download URL available for file %s", fileCode) + } + + return &model.Link{ + URL: dlURL, + }, nil +} + +func (d *Darkibox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + parentID := d.RootFolderID + if parentDir.GetID() != "" { + parentID = folderIDFromObjID(parentDir.GetID()) + } + + var result folderCreateResult + if err := d.callAPI(ctx, "/folder/create", map[string]string{ + "name": dirName, + "parent_id": fldIDStr(parentID), + }, &result); err != nil { + return nil, fmt.Errorf("create folder failed: %w", err) + } + + return &model.Object{ + ID: encodeFolderID(result.FldID), + Name: dirName, + IsFolder: true, + }, nil +} + +func (d *Darkibox) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, errs.NotImplement + } + + fileCode := fileCodeFromObjID(srcObj.GetID()) + if fileCode == "" { + return nil, fmt.Errorf("empty file code") + } + + dstFolderID := d.RootFolderID + if dstDir.GetID() != "" { + dstFolderID = folderIDFromObjID(dstDir.GetID()) + } + + if err := d.callAPI(ctx, "/file/move", map[string]string{ + "file_code": fileCode, + "to_folder": fldIDStr(dstFolderID), + }, nil); err != nil { + return nil, fmt.Errorf("move file failed: %w", err) + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: false, + }, nil +} + +func (d *Darkibox) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + folderID := folderIDFromObjID(obj.GetID()) + return d.callAPI(ctx, "/folder/delete", map[string]string{ + "fld_id": folderID, + }, nil) + } + + fileCode := fileCodeFromObjID(obj.GetID()) + return d.callAPI(ctx, "/file/delete", map[string]string{ + "file_code": fileCode, + }, nil) +} + +func (d *Darkibox) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + // Step 1: Get the upload server URL + var server uploadServerResult + if err := d.callAPI(ctx, "/upload/server", nil, &server); err != nil { + return nil, fmt.Errorf("get upload server failed: %w", err) + } + if server.URL == "" { + return nil, fmt.Errorf("no upload server URL returned") + } + + // Step 2: Upload the file to the upload server + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }) + + res, err := base.RestyClient.R(). + SetContext(ctx). + SetMultipartField("file", file.GetName(), "", reader). + SetMultipartFormData(map[string]string{ + "key": d.APIKey, + "fld_id": fldIDStr(folderID), + }). + Post(server.URL) + if err != nil { + return nil, fmt.Errorf("upload failed: %w", err) + } + if res.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("upload failed: http %d", res.StatusCode()) + } + + // Try to parse upload response to get the file code + var uploadResp uploadResult + if err := base.RestyClient.JSONUnmarshal(res.Body(), &uploadResp); err == nil && len(uploadResp.Files) > 0 { + uf := uploadResp.Files[0] + return &model.Object{ + ID: encodeFileID(uf.FileCode), + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil + } + + return &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +func (d *Darkibox) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*Darkibox)(nil) diff --git a/drivers/darkibox/meta.go b/drivers/darkibox/meta.go new file mode 100644 index 00000000000..a09707707fd --- /dev/null +++ b/drivers/darkibox/meta.go @@ -0,0 +1,27 @@ +package darkibox + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APIKey string `json:"api_key" required:"true" help:"API key from your Darkibox account"` +} + +var config = driver.Config{ + Name: "Darkibox", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: true, + NoCache: false, + NoUpload: false, + DefaultRoot: "0", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Darkibox{} + }) +} diff --git a/drivers/darkibox/types.go b/drivers/darkibox/types.go new file mode 100644 index 00000000000..9f193f97e4f --- /dev/null +++ b/drivers/darkibox/types.go @@ -0,0 +1,79 @@ +package darkibox + +import "encoding/json" + +// apiResponse is the common wrapper for all Darkibox API responses. +type apiResponse struct { + Msg string `json:"msg"` + Result json.RawMessage `json:"result"` + ServerTime string `json:"server_time"` + Status int `json:"status"` +} + +// accountInfoResult represents the result of /api/account/info +type accountInfoResult struct { + Email string `json:"email"` + Balance string `json:"balance"` + StorageUsed string `json:"storage_used"` +} + +// fileListResult represents the result of /api/file/list +type fileListResult struct { + Results int `json:"results"` + ResultsTotal int `json:"results_total"` + Files []fileItem `json:"files"` +} + +type fileItem struct { + FileCode string `json:"file_code"` + Name string `json:"name"` + Title string `json:"title"` + Size int64 `json:"size"` + Uploaded string `json:"uploaded"` + FldID int64 `json:"fld_id"` +} + +// folderListResult represents the result of /api/folder/list +type folderListResult struct { + Folders []folderItem `json:"folders"` +} + +type folderItem struct { + FldID int64 `json:"fld_id"` + Name string `json:"name"` + Code string `json:"code"` +} + +// directLinkResult represents the result of /api/file/direct_link +type directLinkResult struct { + Versions []directLinkVersion `json:"versions"` +} + +type directLinkVersion struct { + Name string `json:"name"` + URL string `json:"url"` +} + +// uploadServerResult represents the result of /api/upload/server +type uploadServerResult struct { + URL string `json:"url"` +} + +// uploadResult represents the response from the upload endpoint +type uploadResult struct { + Files []uploadedFile `json:"files"` +} + +type uploadedFile struct { + FileCode string `json:"filecode"` + URL string `json:"url"` + Name string `json:"name"` + Size int64 `json:"size"` + Status int `json:"status"` +} + +// folderCreateResult represents the result of /api/folder/create +type folderCreateResult struct { + FldID int64 `json:"fld_id"` + Name string `json:"name"` +} diff --git a/drivers/darkibox/util.go b/drivers/darkibox/util.go new file mode 100644 index 00000000000..0032abab106 --- /dev/null +++ b/drivers/darkibox/util.go @@ -0,0 +1,88 @@ +package darkibox + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" +) + +const apiBase = "https://darkibox.com/api" + +// callAPI makes a GET request to the Darkibox API with the given endpoint and params. +// It automatically injects the API key. The result JSON is unmarshalled into out if non-nil. +func (d *Darkibox) callAPI(ctx context.Context, endpoint string, params map[string]string, out any) error { + query := map[string]string{ + "key": d.APIKey, + } + for k, v := range params { + if strings.TrimSpace(v) == "" { + continue + } + query[k] = v + } + + var resp apiResponse + r, err := base.RestyClient.R(). + SetContext(ctx). + SetQueryParams(query). + SetResult(&resp). + Get(apiBase + endpoint) + if err != nil { + return err + } + if r.StatusCode() != http.StatusOK { + return fmt.Errorf("darkibox http error: %d", r.StatusCode()) + } + if resp.Status != 200 { + return fmt.Errorf("darkibox api error: status=%d msg=%s", resp.Status, resp.Msg) + } + if out == nil || len(resp.Result) == 0 || string(resp.Result) == "null" { + return nil + } + if err := json.Unmarshal(resp.Result, out); err != nil { + return fmt.Errorf("decode darkibox result failed: %w", err) + } + return nil +} + +// fldIDStr converts a folder ID (which may be the root "0") to a string suitable for API params. +func fldIDStr(id string) string { + if id == "" { + return "0" + } + return id +} + +// encodeFolderID prefixes a folder ID so we can distinguish folders from files. +func encodeFolderID(id int64) string { + return "d:" + strconv.FormatInt(id, 10) +} + +// encodeFileID prefixes a file code so we can distinguish files from folders. +func encodeFileID(code string) string { + return "f:" + code +} + +// folderIDFromObjID extracts the numeric folder ID string from an object ID. +func folderIDFromObjID(id string) string { + if strings.HasPrefix(id, "d:") { + return strings.TrimPrefix(id, "d:") + } + if id == "" || id == "/" { + return "0" + } + return id +} + +// fileCodeFromObjID extracts the file code from an object ID. +func fileCodeFromObjID(id string) string { + if strings.HasPrefix(id, "f:") { + return strings.TrimPrefix(id, "f:") + } + return id +} From 06cb5ee555e6c0936d7b0780bf9a563e9aba8ea8 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 15 Apr 2026 19:19:49 +0800 Subject: [PATCH 112/133] feat(guangyapan): add full GuangYaPan driver integration Implement GuangYaPan storage adapter and register driver. Includes: - SMS/captcha login flow with token refresh - list/link operations - mkdir/rename/remove/move/copy - upload via res_center token + OSS multipart + task polling - compatibility fixes for provider type, endpoint normalization, and upload status codes --- drivers/all.go | 1 + drivers/guangyapan/driver.go | 950 +++++++++++++++++++++++++++++++++++ drivers/guangyapan/meta.go | 42 ++ drivers/guangyapan/types.go | 141 ++++++ 4 files changed, 1134 insertions(+) create mode 100644 drivers/guangyapan/driver.go create mode 100644 drivers/guangyapan/meta.go create mode 100644 drivers/guangyapan/types.go diff --git a/drivers/all.go b/drivers/all.go index fe375586a2d..3dc90424cbe 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -42,6 +42,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/gofile" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" + _ "github.com/alist-org/alist/v3/drivers/guangyapan" _ "github.com/alist-org/alist/v3/drivers/halalcloud" _ "github.com/alist-org/alist/v3/drivers/ilanzou" _ "github.com/alist-org/alist/v3/drivers/ipfs_api" diff --git a/drivers/guangyapan/driver.go b/drivers/guangyapan/driver.go new file mode 100644 index 00000000000..ff83c9a88fa --- /dev/null +++ b/drivers/guangyapan/driver.go @@ -0,0 +1,950 @@ +package guangyapan + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +const ( + accountBaseURL = "https://account.guangyapan.com" + apiBaseURL = "https://api.guangyapan.com" + defaultClient = "aMe-8VSlkrbQXpUR" +) + +type GuangYaPan struct { + model.Storage + Addition + + accountClient *resty.Client + apiClient *resty.Client +} + +func (d *GuangYaPan) Config() driver.Config { + return config +} + +func (d *GuangYaPan) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *GuangYaPan) Init(ctx context.Context) error { + d.ClientID = strings.TrimSpace(d.ClientID) + if d.ClientID == "" { + d.ClientID = defaultClient + } + d.DeviceID = normalizeDeviceID(d.DeviceID) + if d.DeviceID == "" { + d.DeviceID = randomDeviceID() + } + if d.PageSize <= 0 { + d.PageSize = 100 + } + if d.OrderBy < 0 { + d.OrderBy = 3 + } + if d.SortType != 0 && d.SortType != 1 { + d.SortType = 1 + } + + d.AccessToken = strings.TrimSpace(d.AccessToken) + d.RefreshToken = strings.TrimSpace(d.RefreshToken) + d.PhoneNumber = strings.TrimSpace(d.PhoneNumber) + d.VerifyCode = strings.TrimSpace(d.VerifyCode) + d.CaptchaToken = strings.TrimSpace(d.CaptchaToken) + d.VerificationID = strings.TrimSpace(d.VerificationID) + + d.accountClient = base.NewRestyClient(). + SetBaseURL(accountBaseURL). + SetHeader("Accept", "application/json, text/plain, */*"). + SetHeader("Content-Type", "application/json"). + SetHeader("X-Device-Model", "chrome%2F147.0.0.0"). + SetHeader("X-Device-Name", "PC-Chrome"). + SetHeader("X-Device-Sign", "wdi10."+d.DeviceID+"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"). + SetHeader("X-Net-Work-Type", "NONE"). + SetHeader("X-OS-Version", "MacIntel"). + SetHeader("X-Platform-Version", "1"). + SetHeader("X-Protocol-Version", "301"). + SetHeader("X-Provider-Name", "NONE"). + SetHeader("X-SDK-Version", "9.0.2"). + SetHeader("X-Client-Id", d.ClientID). + SetHeader("X-Client-Version", "0.0.1"). + SetHeader("X-Device-Id", d.DeviceID) + if d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + } + + d.apiClient = base.NewRestyClient(). + SetBaseURL(apiBaseURL). + SetHeader("Accept", "application/json, text/plain, */*"). + SetHeader("Content-Type", "application/json"). + SetHeader("Did", d.DeviceID). + SetHeader("Dt", "4") + + // Priority: access_token -> refresh_token -> sms login. + if d.AccessToken != "" { + if err := d.validateToken(ctx); err == nil { + return nil + } + d.AccessToken = "" + } + if d.RefreshToken != "" { + if err := d.refreshToken(ctx); err == nil { + if err2 := d.validateToken(ctx); err2 == nil { + return nil + } + } + } + // Two-stage SMS flow: + // 1) phone only + send_code=true: send code and cache verification_id (do not fail init). + // 2) phone + verify_code: complete login and save tokens. + if d.PhoneNumber != "" { + if d.canSMSLogin() { + if err := d.loginBySMSCode(ctx); err != nil { + return err + } + return d.validateToken(ctx) + } + if d.SendCode { + d.setTempStatus("SMS sending in progress...") + if err := d.prepareSMSCode(ctx); err != nil { + d.setTempStatus(fmt.Sprintf("SMS send failed: %v. Please check captcha/meta and set send_code=true to retry.", err)) + log.Warnf("guangyapan: prepare sms code failed: %v", err) + } else { + d.setTempStatus("SMS sent successfully. Please fill verify_code and save to complete login.") + } + } + return nil + } + return errors.New("login failed: provide a valid access_token, or refresh_token, or phone_number + verify_code + captcha_token") +} + +func (d *GuangYaPan) Drop(ctx context.Context) error { + return nil +} + +func (d *GuangYaPan) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + parentID := dir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + res := make([]model.Obj, 0, d.PageSize) + for page := 0; ; page++ { + var resp listResp + body := map[string]any{ + "parentId": parentID, + "page": page, + "pageSize": d.PageSize, + "orderBy": d.OrderBy, + "sortType": d.SortType, + "fileTypes": []int{}, + } + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_file_list", body, &resp); err != nil { + return nil, err + } + for _, item := range resp.Data.List { + res = append(res, &model.Object{ + ID: item.FileID, + Path: parentID, + Name: item.FileName, + Size: item.FileSize, + Modified: unixOrZero(item.UTime), + Ctime: unixOrZero(item.CTime), + IsFolder: item.ResType == 2, + }) + } + if len(resp.Data.List) < d.PageSize { + break + } + if resp.Data.Total > 0 && len(res) >= resp.Data.Total { + break + } + } + return res, nil +} + +func (d *GuangYaPan) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + var resp downloadResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_download_url", map[string]any{ + "fileId": file.GetID(), + }, &resp); err != nil { + return nil, err + } + + url := strings.TrimSpace(resp.Data.SignedURL) + if url == "" { + url = strings.TrimSpace(resp.Data.DownloadURL) + } + if url == "" { + return nil, errors.New("empty download url") + } + return &model.Link{URL: url}, nil +} + +func (d *GuangYaPan) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + name := strings.TrimSpace(dirName) + if name == "" { + return errors.New("dir name is empty") + } + + parentID := parentDir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + var out createDirResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/create_dir", map[string]any{ + "parentId": parentID, + "dirName": name, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("make dir failed: %s", strings.TrimSpace(out.Msg)) + } + return nil +} + +func (d *GuangYaPan) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + name := strings.TrimSpace(newName) + if name == "" { + return errors.New("new name is empty") + } + + var out commonResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/rename", map[string]any{ + "fileId": fileID, + "newName": name, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("rename failed: %s", strings.TrimSpace(out.Msg)) + } + return nil +} + +func (d *GuangYaPan) Remove(ctx context.Context, obj model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(obj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + + var del deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/delete_file", map[string]any{ + "fileIds": []string{fileID}, + }, &del); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(del.Msg), "success") { + return fmt.Errorf("delete failed: %s", strings.TrimSpace(del.Msg)) + } + + taskID := strings.TrimSpace(del.Data.TaskID) + if taskID == "" { + // Some backends may apply deletion synchronously. + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + parentID := dstDir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + var out deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/move_file", map[string]any{ + "fileIds": []string{fileID}, + "parentId": parentID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("move failed: %s", strings.TrimSpace(out.Msg)) + } + taskID := strings.TrimSpace(out.Data.TaskID) + if taskID == "" { + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + parentID := dstDir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + var out deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/copy_file", map[string]any{ + "fileIds": []string{fileID}, + "parentId": parentID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("copy failed: %s", strings.TrimSpace(out.Msg)) + } + taskID := strings.TrimSpace(out.Data.TaskID) + if taskID == "" { + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + if file == nil { + return errors.New("file is nil") + } + if file.GetSize() < 0 { + return errors.New("invalid file size") + } + name := strings.TrimSpace(file.GetName()) + if name == "" { + return errors.New("file name is empty") + } + + parentID := dstDir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + token, code, err := d.getUploadToken(ctx, parentID, name, file.GetSize()) + if err != nil { + return err + } + taskID := strings.TrimSpace(token.TaskID) + if code == 156 { + if taskID == "" { + return errors.New("instant upload returns empty task id") + } + return d.waitUploadTaskInfo(ctx, taskID) + } + + if token.ObjectPath == "" || token.BucketName == "" || token.EndPoint == "" || token.AccessKeyID == "" || token.SecretAccessKey == "" { + return errors.New("upload token is incomplete") + } + + ossEndpoint := normalizeOSSEndpoint(token.EndPoint, token.BucketName) + client, err := oss.New(ossEndpoint, token.AccessKeyID, token.SecretAccessKey, oss.SecurityToken(token.SessionToken)) + if err != nil { + return fmt.Errorf("create oss client failed: %w", err) + } + bucket, err := client.Bucket(token.BucketName) + if err != nil { + return fmt.Errorf("create oss bucket failed: %w", err) + } + + if file.GetSize() == 0 { + if err := bucket.PutObject(token.ObjectPath, strings.NewReader("")); err != nil { + return err + } + } else { + if err := d.multipartUploadToOSS(ctx, bucket, token.ObjectPath, file, up); err != nil { + return err + } + } + + if taskID == "" { + return nil + } + return d.waitUploadTaskInfo(ctx, taskID) +} + +func (d *GuangYaPan) ensureAccessToken(ctx context.Context) error { + if strings.TrimSpace(d.AccessToken) != "" { + return nil + } + if strings.TrimSpace(d.RefreshToken) == "" { + if d.canSMSLogin() { + return d.loginBySMSCode(ctx) + } + if d.PhoneNumber != "" { + return errors.New("not logged in yet: please fill verify_code and save storage to finish SMS login") + } + return errors.New("access token is empty") + } + return d.refreshToken(ctx) +} + +func (d *GuangYaPan) validateToken(ctx context.Context) error { + var me userMeResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetResult(&me). + Get("/v1/user/me") + if err != nil { + return err + } + if resp.IsError() { + return fmt.Errorf("validate token failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + if strings.TrimSpace(me.Sub) == "" { + return errors.New("validate token failed: empty user sub") + } + return nil +} + +func (d *GuangYaPan) refreshToken(ctx context.Context) error { + if strings.TrimSpace(d.RefreshToken) == "" { + return errors.New("refresh_token is empty") + } + + var out tokenResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "client_id": d.ClientID, + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + }). + SetResult(&out). + Post("/v1/auth/token") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" { + errMsg := strings.TrimSpace(out.ErrorDesc) + if errMsg == "" { + errMsg = strings.TrimSpace(out.Error) + } + if errMsg == "" { + errMsg = strings.TrimSpace(resp.String()) + } + if errMsg == "" { + errMsg = fmt.Sprintf("status=%d", resp.StatusCode()) + } + return fmt.Errorf("refresh token failed: %s", errMsg) + } + + d.AccessToken = strings.TrimSpace(out.AccessToken) + if strings.TrimSpace(out.RefreshToken) != "" { + d.RefreshToken = strings.TrimSpace(out.RefreshToken) + } + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) canSMSLogin() bool { + return d.PhoneNumber != "" && d.VerifyCode != "" +} + +func (d *GuangYaPan) loginBySMSCode(ctx context.Context) error { + verificationID := strings.TrimSpace(d.VerificationID) + if verificationID == "" { + var err error + verificationID, err = d.requestVerificationID(ctx) + if err != nil { + return err + } + } + + var step2 verifyResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "verification_id": verificationID, + "verification_code": d.VerifyCode, + "client_id": d.ClientID, + }). + SetResult(&step2). + Post("/v1/auth/verification/verify") + if err != nil { + return err + } + if resp.IsError() || step2.Error != "" || strings.TrimSpace(step2.VerificationToken) == "" { + return fmt.Errorf("verify code failed: %s", d.accountErr(step2.ErrorDesc, step2.Error, resp)) + } + + var out tokenResp + resp, err = d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "verification_code": d.VerifyCode, + "verification_token": step2.VerificationToken, + "username": normalizePhoneE164(d.PhoneNumber), + "client_id": d.ClientID, + }). + SetResult(&out). + Post("/v1/auth/signin") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" { + return fmt.Errorf("signin failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp)) + } + + d.AccessToken = strings.TrimSpace(out.AccessToken) + d.RefreshToken = strings.TrimSpace(out.RefreshToken) + d.VerificationID = "" + // One-time SMS code should not be reused after successful login. + d.VerifyCode = "" + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) prepareSMSCode(ctx context.Context) error { + // Explicit send action should always refresh verification_id. + d.VerificationID = "" + if err := d.ensureCaptchaToken(ctx, false); err != nil { + return err + } + verificationID, err := d.requestVerificationID(ctx) + if err != nil { + return err + } + d.VerificationID = verificationID + d.SendCode = false + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) requestVerificationID(ctx context.Context) (string, error) { + if d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + } + + var step1 verificationResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "phone_number": normalizePhoneE164(d.PhoneNumber), + "target": "ANY", + "client_id": d.ClientID, + }). + SetResult(&step1). + Post("/v1/auth/verification") + if err != nil { + return "", err + } + if resp.IsError() || step1.Error != "" || strings.TrimSpace(step1.VerificationID) == "" { + // If captcha token is expired/invalid, refresh it once and retry. + if strings.Contains(step1.Error, "captcha_invalid") || strings.Contains(step1.ErrorDesc, "captcha_token expired") { + if err := d.ensureCaptchaToken(ctx, true); err == nil { + return d.requestVerificationID(ctx) + } + } + return "", fmt.Errorf("request verification failed: %s", d.accountErr(step1.ErrorDesc, step1.Error, resp)) + } + return strings.TrimSpace(step1.VerificationID), nil +} + +func (d *GuangYaPan) ensureCaptchaToken(ctx context.Context, force bool) error { + if !force && d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + return nil + } + + var out captchaInitResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "client_id": d.ClientID, + "action": "POST:/v1/auth/verification", + "device_id": d.DeviceID, + "meta": map[string]any{ + "username": normalizePhoneE164(d.PhoneNumber), + "phone_number": normalizePhoneE164(d.PhoneNumber), + "VERIFICATION_PHONE": normalizePhoneE164(d.PhoneNumber), + }, + }). + SetResult(&out). + Post("/v1/shield/captcha/init") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.CaptchaToken) == "" { + return fmt.Errorf("init captcha token failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp)) + } + d.CaptchaToken = strings.TrimSpace(out.CaptchaToken) + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + op.MustSaveDriverStorage(d) + return nil +} + +func normalizeCaptchaUsername(phone string) string { + p := strings.TrimSpace(phone) + p = strings.ReplaceAll(p, " ", "") + p = strings.TrimPrefix(p, "+") + // Keep only digits. + b := make([]rune, 0, len(p)) + for _, ch := range p { + if ch >= '0' && ch <= '9' { + b = append(b, ch) + } + } + digits := string(b) + // Mainland number normalization: +86xxxxxxxxxxx -> xxxxxxxxxxx + if strings.HasPrefix(digits, "86") && len(digits) > 11 { + digits = digits[2:] + } + return digits +} + +func normalizePhoneE164(phone string) string { + p := strings.TrimSpace(phone) + if p == "" { + return "" + } + p = strings.ReplaceAll(p, " ", "") + if strings.HasPrefix(p, "+") { + // Format as "+86 1xxxxxxxxxx" to match browser payload expectations. + if strings.HasPrefix(p, "+86") && len(p) > 3 { + rest := strings.TrimPrefix(p, "+86") + return "+86 " + rest + } + return p + } + // If raw mainland number is provided, normalize with +86 prefix. + digits := normalizeCaptchaUsername(p) + if len(digits) == 11 { + return "+86 " + digits + } + return p +} + +func (d *GuangYaPan) setTempStatus(status string) { + // initStorage sets status to WORK after Init returns, so we update it shortly after. + time.AfterFunc(200*time.Millisecond, func() { + d.GetStorage().SetStatus(status) + op.MustSaveDriverStorage(d) + }) +} + +func (d *GuangYaPan) accountErr(desc, short string, resp *resty.Response) string { + msg := strings.TrimSpace(desc) + if msg == "" { + msg = strings.TrimSpace(short) + } + if msg == "" && resp != nil { + msg = strings.TrimSpace(resp.String()) + } + if msg == "" && resp != nil { + msg = fmt.Sprintf("status=%d", resp.StatusCode()) + } + if msg == "" { + msg = "unknown error" + } + return msg +} + +func (d *GuangYaPan) postAPI(ctx context.Context, path string, body any, out any) error { + if strings.TrimSpace(d.AccessToken) == "" { + return errors.New("access token is empty") + } + resp, err := d.apiClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetBody(body). + SetResult(out). + Post(path) + if err != nil { + return err + } + if resp.StatusCode() == 401 || resp.StatusCode() == 403 { + if strings.TrimSpace(d.RefreshToken) == "" { + return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + if err := d.refreshToken(ctx); err != nil { + return err + } + resp, err = d.apiClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetBody(body). + SetResult(out). + Post(path) + if err != nil { + return err + } + } + if resp.IsError() { + return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + return nil +} + +func (d *GuangYaPan) waitTaskDone(ctx context.Context, taskID string) error { + const ( + maxTry = 30 + interval = 300 * time.Millisecond + ) + for i := 0; i < maxTry; i++ { + var out taskStatusResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_task_status", map[string]any{ + "taskId": taskID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("get task status failed: %s", strings.TrimSpace(out.Msg)) + } + switch out.Data.Status { + case 2: + return nil + case -1, 3: + return fmt.Errorf("task %s failed with status=%d", taskID, out.Data.Status) + } + if i == maxTry-1 { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } + return fmt.Errorf("task %s timeout", taskID) +} + +func (d *GuangYaPan) getUploadToken(ctx context.Context, parentID, name string, size int64) (*uploadTokenData, int, error) { + var out uploadTokenResp + err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_center_token", map[string]any{ + "capacity": 2, + "name": name, + "parentId": parentID, + "res": map[string]any{ + "fileSize": size, + }, + }, &out) + if err != nil { + return nil, 0, err + } + msg := strings.TrimSpace(out.Msg) + if msg != "" && !strings.EqualFold(msg, "success") { + return nil, out.Code, fmt.Errorf("get upload token failed: %s", msg) + } + if out.Data.TaskID == "" { + return nil, out.Code, errors.New("get upload token failed: empty task id") + } + if out.Data.AccessKeyID == "" { + out.Data.AccessKeyID = out.Data.Creds.AccessKeyID + } + if out.Data.SecretAccessKey == "" { + out.Data.SecretAccessKey = out.Data.Creds.SecretAccessKey + } + if out.Data.SessionToken == "" { + out.Data.SessionToken = out.Data.Creds.SessionToken + } + if strings.TrimSpace(out.Data.EndPoint) == "" { + out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint) + } + if strings.TrimSpace(out.Data.EndPoint) != "" && !strings.HasPrefix(out.Data.EndPoint, "http://") && !strings.HasPrefix(out.Data.EndPoint, "https://") { + if strings.TrimSpace(out.Data.FullEndPoint) != "" { + out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint) + } else if strings.TrimSpace(out.Data.BucketName) != "" { + host := strings.TrimSpace(out.Data.EndPoint) + prefix := strings.TrimSpace(out.Data.BucketName) + "." + if strings.HasPrefix(host, prefix) { + out.Data.EndPoint = "https://" + host + } else { + out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.BucketName) + "." + host + } + } else { + out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.EndPoint) + } + } + return &out.Data, out.Code, nil +} + +func (d *GuangYaPan) waitUploadTaskInfo(ctx context.Context, taskID string) error { + const ( + maxTry = 300 + interval = 1 * time.Second + ) + for i := 0; i < maxTry; i++ { + var out taskInfoResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_info_by_task_id", map[string]any{ + "taskId": taskID, + }, &out); err != nil { + return err + } + if out.Data.FileID != "" { + return nil + } + switch out.Code { + case 145, 146, 147, 155, 163, 0: + // uploading/verifying/processing + default: + if strings.TrimSpace(out.Msg) != "" { + return fmt.Errorf("upload task failed: code=%d msg=%s", out.Code, strings.TrimSpace(out.Msg)) + } + } + if i == maxTry-1 { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } + return fmt.Errorf("upload task %s timeout", taskID) +} + +func (d *GuangYaPan) multipartUploadToOSS(ctx context.Context, bucket *oss.Bucket, objectPath string, file model.FileStreamer, up driver.UpdateProgress) error { + partSize := calcUploadPartSize(file.GetSize()) + imur, err := bucket.InitiateMultipartUpload(objectPath, oss.Sequential()) + if err != nil { + return err + } + + total := file.GetSize() + partCount := int((total + partSize - 1) / partSize) + parts := make([]oss.UploadPart, 0, partCount) + var uploaded int64 + partNumber := 1 + + for uploaded < total { + if err := ctx.Err(); err != nil { + return err + } + curPartSize := partSize + left := total - uploaded + if left < curPartSize { + curPartSize = left + } + + reader := io.LimitReader(file, curPartSize) + part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, reader), curPartSize, partNumber) + if err != nil { + return err + } + parts = append(parts, part) + uploaded += curPartSize + partNumber++ + if total > 0 { + up(100 * float64(uploaded) / float64(total)) + } + } + + _, err = bucket.CompleteMultipartUpload(imur, parts) + return err +} + +func calcUploadPartSize(size int64) int64 { + const ( + mb = int64(1024 * 1024) + gb = int64(1024 * 1024 * 1024) + ) + switch { + case size <= 100*mb: + return 1 * mb + case size <= 16*gb: + return 2 * mb + case size <= 160*gb: + return 4 * mb + default: + return 8 * mb + } +} + +func normalizeOSSEndpoint(endpoint, bucket string) string { + ep := strings.TrimSpace(endpoint) + if ep == "" { + return ep + } + if !strings.HasPrefix(ep, "http://") && !strings.HasPrefix(ep, "https://") { + ep = "https://" + ep + } + u, err := url.Parse(ep) + if err != nil || u.Host == "" { + return ep + } + host := u.Host + prefix := strings.TrimSpace(bucket) + if prefix != "" && strings.HasPrefix(host, prefix+".") { + host = strings.TrimPrefix(host, prefix+".") + } + u.Host = host + return u.String() +} + +func normalizeDeviceID(v string) string { + v = strings.ToLower(strings.TrimSpace(v)) + v = strings.ReplaceAll(v, "-", "") + if len(v) != 32 { + return "" + } + for _, ch := range v { + if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { + return "" + } + } + return v +} + +func randomDeviceID() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "0123456789abcdef0123456789abcdef" + } + return hex.EncodeToString(b) +} + +var _ driver.Driver = (*GuangYaPan)(nil) diff --git a/drivers/guangyapan/meta.go b/drivers/guangyapan/meta.go new file mode 100644 index 00000000000..606d6aec8ee --- /dev/null +++ b/drivers/guangyapan/meta.go @@ -0,0 +1,42 @@ +package guangyapan + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + PhoneNumber string `json:"phone_number" type:"text" help:"Phone number for SMS login, e.g. +86 13800000000"` + CaptchaToken string `json:"captcha_token" type:"text" help:"Captcha token required by /v1/auth/verification"` + SendCode bool `json:"send_code" type:"bool" help:"Set true and save to send SMS code, it auto-resets to false after sending"` + VerifyCode string `json:"verify_code" type:"text" help:"SMS verification code used with phone_number; fill then save to finish login"` + VerificationID string `json:"verification_id" type:"text" help:"Auto-generated after sending SMS code; do not edit manually"` + AccessToken string `json:"access_token" type:"text" help:"Bearer access token (optional if refresh_token is provided)"` + RefreshToken string `json:"refresh_token" type:"text" help:"Refresh token for auto-login/auto-refresh"` + ClientID string `json:"client_id" default:"aMe-8VSlkrbQXpUR"` + DeviceID string `json:"device_id" help:"Optional custom device id (32 hex chars), auto-generated when empty"` + PageSize int `json:"page_size" type:"number" default:"100"` + OrderBy int `json:"order_by" type:"number" default:"3" help:"0:name,1:size,2:create_time,3:update_time"` + SortType int `json:"sort_type" type:"number" default:"1" help:"0:asc,1:desc"` +} + +var config = driver.Config{ + Name: "GuangYaPan", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: true, + Alert: "info|Two-stage SMS login: (1) fill phone_number (+ captcha_token if needed), set send_code=true and save; (2) fill verify_code and save to finish login and auto-save access_token/refresh_token.", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &GuangYaPan{} + }) +} diff --git a/drivers/guangyapan/types.go b/drivers/guangyapan/types.go new file mode 100644 index 00000000000..bd0094f3070 --- /dev/null +++ b/drivers/guangyapan/types.go @@ -0,0 +1,141 @@ +package guangyapan + +import "time" + +type tokenResp struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + TokenType string `json:"token_type"` + ExpiresIn int64 `json:"expires_in"` + Sub string `json:"sub"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type verificationResp struct { + VerificationID string `json:"verification_id"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type captchaInitResp struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type verifyResp struct { + VerificationToken string `json:"verification_token"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type userMeResp struct { + Sub string `json:"sub"` +} + +type listResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Total int `json:"total"` + List []fileItem `json:"list"` + } `json:"data"` +} + +type fileItem struct { + FileID string `json:"fileId"` + ParentID string `json:"parentId"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + ResType int `json:"resType"` + CTime int64 `json:"ctime"` + UTime int64 `json:"utime"` +} + +type downloadResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + SignedURL string `json:"signedURL"` + DownloadURL string `json:"downloadUrl"` + } `json:"data"` +} + +type createDirResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileID string `json:"fileId"` + FileName string `json:"fileName"` + ResType int `json:"resType"` + CTime int64 `json:"ctime"` + UTime int64 `json:"utime"` + } `json:"data"` +} + +type commonResp struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +type deleteResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID string `json:"taskId"` + } `json:"data"` +} + +type taskStatusResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Status int `json:"status"` + } `json:"data"` +} + +type uploadTokenResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data uploadTokenData `json:"data"` +} + +type uploadTokenData struct { + TaskID string `json:"taskId"` + ObjectPath string `json:"objectPath"` + Provider any `json:"provider"` + Region string `json:"region"` + BucketName string `json:"bucketName"` + EndPoint string `json:"endPoint"` + FullEndPoint string `json:"fullEndPoint"` + CallbackVar string `json:"callbackVar"` + AccessKeyID string `json:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` + Creds struct { + AccessKeyID string `json:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` + } `json:"creds"` +} + +type taskInfoResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileID string `json:"fileId"` + } `json:"data"` +} + +func unixOrZero(v int64) time.Time { + if v <= 0 { + return time.Time{} + } + return time.Unix(v, 0) +} From ee27244094a24b7fbff12837594ff68baeafb048 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 15 Apr 2026 20:32:42 +0800 Subject: [PATCH 113/133] feat(frp): add runtime log API and stop endpoint --- internal/bootstrap/frp.go | 19 +++ internal/frp/frp.go | 335 ++++++++++++++++++++++++++++++++++++++ server/handles/setting.go | 37 +++++ server/router.go | 3 + 4 files changed, 394 insertions(+) create mode 100644 internal/bootstrap/frp.go create mode 100644 internal/frp/frp.go diff --git a/internal/bootstrap/frp.go b/internal/bootstrap/frp.go new file mode 100644 index 00000000000..b8417843c30 --- /dev/null +++ b/internal/bootstrap/frp.go @@ -0,0 +1,19 @@ +package bootstrap + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func InitFRP() { + frp.Instance = frp.Init() + if setting.GetBool(conf.FRPEnabled) { + if err := frp.Instance.Start(); err != nil { + utils.Log.Warnf("failed to start frp client: %v", err) + } else { + utils.Log.Info("frp client started") + } + } +} diff --git a/internal/frp/frp.go b/internal/frp/frp.go new file mode 100644 index 00000000000..c244e18b870 --- /dev/null +++ b/internal/frp/frp.go @@ -0,0 +1,335 @@ +package frp + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/cmd/flags" + frpclient "github.com/fatedier/frp/client" + "github.com/fatedier/frp/pkg/config/source" + v1 "github.com/fatedier/frp/pkg/config/v1" + frplog "github.com/fatedier/frp/pkg/util/log" + log "github.com/sirupsen/logrus" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" +) + +// Instance is the global FRP manager. +var Instance *Manager + +// Manager controls the lifecycle of the embedded FRP client. +type Manager struct { + mu sync.Mutex + cancel context.CancelFunc + wg sync.WaitGroup + status string + logs []string + logPath string +} + +const maxLogEntries = 300 +const maxTailBytes = 512 * 1024 + +// RuntimeInfo contains FRP runtime status and recent logs. +type RuntimeInfo struct { + Status string `json:"status"` + Logs []string `json:"logs"` +} + +// Init creates and returns a new Manager. +func Init() *Manager { + m := &Manager{ + status: "stopped", + logPath: filepath.Join(flags.DataDir, "log", "frp.log"), + } + m.logs = append(m.logs, fmt.Sprintf("[%s] initialized", time.Now().Format(time.RFC3339))) + return m +} + +// Start builds the FRP config from settings and starts the client. +func (m *Manager) Start() error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.cancel != nil { + m.appendLogLocked("start skipped: already running") + return nil // already running + } + + cfg, proxyCfgs, err := buildConfig() + if err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("start failed: %s", err.Error()) + return err + } + cfg.Log.To = m.logPath + cfg.Log.Level = "info" + cfg.Log.MaxDays = 7 + if err := os.MkdirAll(filepath.Dir(m.logPath), 0o755); err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("init log dir failed: %s", err.Error()) + return err + } + frplog.InitLogger(cfg.Log.To, cfg.Log.Level, int(cfg.Log.MaxDays), true) + + configSource := source.NewConfigSource() + if err := configSource.ReplaceAll(proxyCfgs, nil); err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("replace config failed: %s", err.Error()) + return err + } + aggregator := source.NewAggregator(configSource) + + svr, err := frpclient.NewService(frpclient.ServiceOptions{ + Common: cfg, + ConfigSourceAggregator: aggregator, + }) + if err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("create service failed: %s", err.Error()) + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + m.cancel = cancel + m.status = "running" + m.appendLogLocked("service started") + m.wg.Add(1) + + go func() { + defer m.wg.Done() + if err := svr.Run(ctx); err != nil && ctx.Err() == nil { + // Context was not cancelled, so this is an unexpected error. + log.Warnf("frp client stopped unexpectedly: %v", err) + m.mu.Lock() + m.status = "error: " + err.Error() + m.appendLogLocked("service stopped unexpectedly: %s", err.Error()) + m.cancel = nil + m.mu.Unlock() + } + }() + + return nil +} + +// Stop gracefully shuts down the FRP client. +func (m *Manager) Stop() { + m.mu.Lock() + cancel := m.cancel + m.cancel = nil + m.mu.Unlock() + + if cancel != nil { + m.appendLog("stopping service") + cancel() + m.wg.Wait() + } + + m.mu.Lock() + m.status = "stopped" + m.appendLogLocked("service stopped") + m.mu.Unlock() +} + +// Restart stops any running client and starts a fresh one with current settings. +func (m *Manager) Restart() error { + m.appendLog("restarting service") + m.Stop() + return m.Start() +} + +// Status returns the current status string: "running", "stopped", or "error: ". +func (m *Manager) Status() string { + m.mu.Lock() + defer m.mu.Unlock() + return m.status +} + +// Runtime returns status and latest logs. +func (m *Manager) Runtime(limit int) RuntimeInfo { + m.mu.Lock() + status := m.status + logPath := m.logPath + m.mu.Unlock() + + logs, err := readLogTail(logPath, limit) + if err != nil { + m.mu.Lock() + logs = m.copyLogsLocked(limit) + m.mu.Unlock() + } + + return RuntimeInfo{ + Status: status, + Logs: logs, + } +} + +func (m *Manager) appendLog(format string, args ...interface{}) { + m.mu.Lock() + defer m.mu.Unlock() + m.appendLogLocked(format, args...) +} + +func (m *Manager) appendLogLocked(format string, args ...interface{}) { + line := fmt.Sprintf(format, args...) + entry := fmt.Sprintf("[%s] %s", time.Now().Format(time.RFC3339), line) + m.logs = append(m.logs, entry) + if len(m.logs) > maxLogEntries { + m.logs = m.logs[len(m.logs)-maxLogEntries:] + } +} + +func (m *Manager) copyLogsLocked(limit int) []string { + if limit <= 0 || limit > maxLogEntries { + limit = maxLogEntries + } + total := len(m.logs) + if total <= limit { + return append([]string(nil), m.logs...) + } + return append([]string(nil), m.logs[total-limit:]...) +} + +func readLogTail(path string, limit int) ([]string, error) { + if limit <= 0 || limit > maxLogEntries { + limit = maxLogEntries + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + size := info.Size() + start := int64(0) + if size > maxTailBytes { + start = size - maxTailBytes + } + if _, err = f.Seek(start, io.SeekStart); err != nil { + return nil, err + } + buf, err := io.ReadAll(f) + if err != nil { + return nil, err + } + + if start > 0 { + if idx := bytes.IndexByte(buf, '\n'); idx >= 0 && idx+1 < len(buf) { + buf = buf[idx+1:] + } + } + text := strings.TrimRight(string(buf), "\n") + if text == "" { + return []string{}, nil + } + lines := strings.Split(text, "\n") + if len(lines) <= limit { + return lines, nil + } + return lines[len(lines)-limit:], nil +} + +func buildConfig() (*v1.ClientCommonConfig, []v1.ProxyConfigurer, error) { + serverAddr := setting.GetStr(conf.FRPServerAddr) + if serverAddr == "" { + return nil, nil, fmt.Errorf("frp server address is required") + } + + serverPort := setting.GetInt(conf.FRPServerPort, 7000) + authToken := setting.GetStr(conf.FRPAuthToken) + proxyName := setting.GetStr(conf.FRPProxyName, "alist") + proxyType := setting.GetStr(conf.FRPProxyType, "http") + customDomain := setting.GetStr(conf.FRPCustomDomain) + subdomain := setting.GetStr(conf.FRPSubdomain) + remotePort := setting.GetInt(conf.FRPRemotePort, 0) + localPort := setting.GetInt(conf.FRPLocalPort, 5244) + tlsEnable := setting.GetBool(conf.FRPTLSEnable) + stcpSecretKey := setting.GetStr(conf.FRPSTCPSecretKey) + + cfg := &v1.ClientCommonConfig{ + ServerAddr: serverAddr, + ServerPort: serverPort, + Auth: v1.AuthClientConfig{ + Method: v1.AuthMethodToken, + Token: authToken, + }, + } + if tlsEnable { + enabled := true + cfg.Transport.TLS.Enable = &enabled + } + + backend := v1.ProxyBackend{ + LocalIP: "127.0.0.1", + LocalPort: localPort, + } + + var proxyCfgs []v1.ProxyConfigurer + + switch proxyType { + case "http": + p := &v1.HTTPProxyConfig{} + p.Name = proxyName + p.Type = "http" + p.ProxyBackend = backend + if customDomain != "" { + p.CustomDomains = []string{customDomain} + } + if subdomain != "" { + p.SubDomain = subdomain + } + proxyCfgs = append(proxyCfgs, p) + + case "https": + p := &v1.HTTPSProxyConfig{} + p.Name = proxyName + p.Type = "https" + p.ProxyBackend = backend + if customDomain != "" { + p.CustomDomains = []string{customDomain} + } + if subdomain != "" { + p.SubDomain = subdomain + } + proxyCfgs = append(proxyCfgs, p) + + case "tcp": + if remotePort <= 0 { + return nil, nil, fmt.Errorf("remote_port is required for tcp proxy type") + } + p := &v1.TCPProxyConfig{} + p.Name = proxyName + p.Type = "tcp" + p.ProxyBackend = backend + p.RemotePort = remotePort + proxyCfgs = append(proxyCfgs, p) + + case "stcp": + p := &v1.STCPProxyConfig{} + p.Name = proxyName + p.Type = "stcp" + p.ProxyBackend = backend + p.Secretkey = stcpSecretKey + p.AllowUsers = []string{"*"} + proxyCfgs = append(proxyCfgs, p) + + default: + return nil, nil, fmt.Errorf("unsupported proxy type: %s", proxyType) + } + + return cfg, proxyCfgs, nil +} diff --git a/server/handles/setting.go b/server/handles/setting.go index f209b7c5dc5..81f7dc61c24 100644 --- a/server/handles/setting.go +++ b/server/handles/setting.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/sign" @@ -183,6 +184,42 @@ func DeleteSetting(c *gin.Context) { common.SuccessResp(c) } +// SetFRP saves FRP settings and restarts the FRP client. +// Returns the current FRP connection status. +func SetFRP(c *gin.Context) { + var req []model.SettingItem + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := op.SaveSettingItems(req); err != nil { + common.ErrorResp(c, err, 500) + return + } + if err := frp.Instance.Restart(); err != nil { + common.SuccessResp(c, frp.Instance.Status()) + return + } + common.SuccessResp(c, frp.Instance.Status()) +} + +// StopFRP stops the FRP client and returns current status. +func StopFRP(c *gin.Context) { + frp.Instance.Stop() + common.SuccessResp(c, frp.Instance.Status()) +} + +// GetFRPRuntime returns current FRP status and recent runtime logs. +func GetFRPRuntime(c *gin.Context) { + limit := 200 + if limitStr := c.Query("limit"); limitStr != "" { + if parsed, err := strconv.Atoi(limitStr); err == nil { + limit = parsed + } + } + common.SuccessResp(c, frp.Instance.Runtime(limit)) +} + func PublicSettings(c *gin.Context) { common.SuccessResp(c, op.GetPublicSettingsMap()) } diff --git a/server/router.go b/server/router.go index 63503838af7..03a52841ed3 100644 --- a/server/router.go +++ b/server/router.go @@ -161,6 +161,9 @@ func admin(g *gin.RouterGroup) { setting.POST("/set_115", handles.Set115) setting.POST("/set_pikpak", handles.SetPikPak) setting.POST("/set_thunder", handles.SetThunder) + setting.POST("/set_frp", handles.SetFRP) + setting.POST("/stop_frp", handles.StopFRP) + setting.GET("/frp_runtime", handles.GetFRPRuntime) // retain /admin/task API to ensure compatibility with legacy automation scripts _task(g.Group("/task")) From 51114e4943f75f0c1dbd99fc6a3758b84eb5540b Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 15 Apr 2026 20:48:38 +0800 Subject: [PATCH 114/133] feat(frp): wire bootstrap settings and runtime dependencies --- cmd/server.go | 3 + go.mod | 51 +++++++++-- go.sum | 140 ++++++++++++++++++++++++----- internal/bootstrap/data/setting.go | 15 ++++ internal/conf/const.go | 15 ++++ internal/model/setting.go | 1 + 6 files changed, 195 insertions(+), 30 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 4263f02021d..abfbcb2c4c9 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -18,6 +18,7 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/bootstrap" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server" @@ -43,6 +44,7 @@ the address is defined in config file`, bootstrap.InitOfflineDownloadTools() bootstrap.LoadStorages() bootstrap.InitTaskManager() + bootstrap.InitFRP() if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) } @@ -167,6 +169,7 @@ the address is defined in config file`, <-quit utils.Log.Println("Shutdown server...") fs.ArchiveContentUploadTaskManager.RemoveAll() + frp.Instance.Stop() Release() ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() diff --git a/go.mod b/go.mod index bc2475c548d..f148b235562 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/alist-org/alist/v3 -go 1.23.4 +go 1.25.0 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 @@ -29,6 +29,7 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dlclark/regexp2 v1.11.4 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 + github.com/fatedier/frp v0.68.0 github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gin-contrib/cors v1.7.2 @@ -70,10 +71,10 @@ require ( github.com/xhofe/wopan-sdk-go v0.1.3 github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.36.0 + golang.org/x/crypto v0.41.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 - golang.org/x/net v0.38.0 + golang.org/x/net v0.43.0 golang.org/x/oauth2 v0.30.0 golang.org/x/time v0.8.0 google.golang.org/appengine v1.6.8 @@ -86,18 +87,50 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/ProtonMail/go-srp v0.0.7 // indirect github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect github.com/bradenaw/juniper v0.15.2 // indirect + github.com/coreos/go-oidc/v3 v3.14.1 // indirect github.com/cronokirby/saferith v0.33.0 // indirect github.com/emersion/go-message v0.18.0 // indirect github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect + github.com/fatedier/golib v0.5.1 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect + github.com/klauspost/reedsolomon v1.12.0 // indirect + github.com/pion/dtls/v2 v2.2.7 // indirect + github.com/pion/logging v0.2.2 // indirect + github.com/pion/stun/v2 v2.0.0 // indirect + github.com/pion/transport/v2 v2.2.1 // indirect + github.com/pion/transport/v3 v3.0.1 // indirect + github.com/pires/go-proxyproto v0.7.0 // indirect + github.com/quic-go/quic-go v0.55.0 // indirect github.com/relvacode/iso8601 v1.3.0 // indirect + github.com/samber/lo v1.47.0 // indirect + github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 // indirect + github.com/templexxx/cpu v0.1.1 // indirect + github.com/templexxx/xorsimd v0.4.3 // indirect + github.com/tjfoc/gmsm v1.4.1 // indirect + github.com/vishvananda/netlink v1.3.0 // indirect + github.com/vishvananda/netns v0.0.4 // indirect + github.com/xtaci/kcp-go/v5 v5.6.13 // indirect + golang.org/x/mod v0.27.0 // indirect + golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect + golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apimachinery v0.28.8 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) require ( @@ -266,15 +299,15 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.12.0 - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 - golang.org/x/tools v0.24.0 // indirect + golang.org/x/sync v0.16.0 + golang.org/x/sys v0.35.0 // indirect + golang.org/x/term v0.34.0 // indirect + golang.org/x/text v0.28.0 + golang.org/x/tools v0.36.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/grpc v1.66.0 - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index e6b04779ede..5ad4346b9fb 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -89,6 +91,8 @@ github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOL github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -195,8 +199,11 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk= +github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -229,10 +236,16 @@ github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwo github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA= github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fatedier/frp v0.68.0 h1:woKC31EpgCLQDirRAOAUgHP3IRxLu2mBHS6zGpcw7tw= +github.com/fatedier/frp v0.68.0/go.mod h1:qFdez6Z+RDqoqF1xJhh48+IW91SVQ+pNWnrmcl43Wjs= +github.com/fatedier/golib v0.5.1 h1:hcKAnaw5mdI/1KWRGejxR+i1Hn/NvbY5UsMKDr7o13M= +github.com/fatedier/golib v0.5.1/go.mod h1:W6kIYkIFxHsTzbgqg5piCxIiDo4LzwgTY6R5W8l9NFQ= github.com/fclairamb/go-log v0.5.0 h1:Gz9wSamEaA6lta4IU2cjJc2xSq5sV5VYSB5w/SUHhVc= github.com/fclairamb/go-log v0.5.0/go.mod h1:XoRO1dYezpsGmLLkZE9I+sHqpqY65p8JA+Vqblb7k40= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -258,13 +271,15 @@ github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -310,6 +325,12 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -318,6 +339,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -325,8 +348,9 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -348,6 +372,8 @@ github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUh github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -365,6 +391,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI0= github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= @@ -427,6 +455,8 @@ github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuV github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/reedsolomon v1.12.0 h1:I5FEp3xSwVCcEh3F5A7dofEfhXdF/bWhQWPH+XwBFno= +github.com/klauspost/reedsolomon v1.12.0/go.mod h1:EPLZJeh4l27pUGC3aXOjheaoh1I9yut7xTURiW3LQ9Y= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -535,6 +565,18 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0= +github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= +github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= +github.com/pires/go-proxyproto v0.7.0 h1:IukmRewDQFWC7kfnb66CSomk2q/seBuilHBYFwyq0Hs= +github.com/pires/go-proxyproto v0.7.0/go.mod h1:Vz/1JPY/OACxWGQNIRY2BeyDmpoaWmEP40O9LbuiFR4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -563,6 +605,8 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk= +github.com/quic-go/quic-go v0.55.0/go.mod h1:DR51ilwU1uE164KuWXhinFcKWGlEjzys2l8zUl5Ss1U= github.com/rclone/rclone v1.67.0 h1:yLRNgHEG2vQ60HCuzFqd0hYwKCRuWuvPUhvhMJ2jI5E= github.com/rclone/rclone v1.67.0/go.mod h1:Cb3Ar47M/SvwfhAjZTbVXdtrP/JLtPFCq2tkdtBVC6w= github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= @@ -581,25 +625,28 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/go-m1cpu v0.2.0 h1:t4GNqvPZ84Vjtpboo/kT3pIkbaK3vc+JIlD/Wz1zSFY= github.com/shoenig/go-m1cpu v0.2.0/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -624,6 +671,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -632,8 +680,14 @@ github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543/go.mod h1:jpwqYA8KUVEvSUJHkCXsnBRJCSKP1BMa81QZ6kvRpow= +github.com/templexxx/cpu v0.1.1 h1:isxHaxBXpYFWnk2DReuKkigaZyrjs2+9ypIdGP4h+HI= +github.com/templexxx/cpu v0.1.1/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= +github.com/templexxx/xorsimd v0.4.3 h1:9AQTFHd7Bhk3dIT7Al2XeBX5DWOvsUPZCuhyAtNbHjU= +github.com/templexxx/xorsimd v0.4.3/go.mod h1:oZQcD6RFDisW2Am58dSAGwwL6rHjbzrlu25VDqfWkQg= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= +github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= +github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= @@ -658,6 +712,10 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d h1:xS9QTPgKl9ewGsAOPc+xW7DeStJDqYPfisDmeSCcbco= github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk= +github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -670,6 +728,10 @@ github.com/xhofe/tache v0.1.5 h1:ezDcgim7tj7KNMXliQsmf8BJQbaZtitfyQA9Nt+B4WM= github.com/xhofe/tache v0.1.5/go.mod h1:PYt6I/XUKliSg1uHlgsk6ha+le/f6PAvjUtFZAVl3a8= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= +github.com/xtaci/kcp-go/v5 v5.6.13 h1:FEjtz9+D4p8t2x4WjciGt/jsIuhlWjjgPCCWjrVR4Hk= +github.com/xtaci/kcp-go/v5 v5.6.13/go.mod h1:75S1AKYYzNUSXIv30h+jPKJYZUwqpfvLshu63nCNSOM= +github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37 h1:EWU6Pktpas0n8lLQwDsRyZfmkPeRbdgPtW609es+/9E= +github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37/go.mod h1:HpMP7DB2CyokmAh4lp0EQnnWhmycP/TvwBGzvuie+H0= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 h1:K8gF0eekWPEX+57l30ixxzGhHH/qscI3JCnuhbN6V4M= @@ -701,6 +763,8 @@ go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -714,17 +778,20 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -761,6 +828,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -775,6 +844,7 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -789,12 +859,13 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -814,8 +885,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -851,14 +922,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -868,12 +940,13 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -886,12 +959,13 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -929,12 +1003,16 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -971,15 +1049,23 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -987,6 +1073,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= @@ -1010,11 +1098,17 @@ gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDa gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 h1:TbRPT0HtzFP3Cno1zZo7yPzEEnfu8EjLfl6IU9VfqkQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259/go.mod h1:AVgIgHMwK63XvmAzWG9vLQ41YnVHN0du0tEC46fI7yY= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/apimachinery v0.28.8 h1:hi/nrxHwk4QLV+W/SHve1bypTE59HCDorLY1stBIxKQ= +k8s.io/apimachinery v0.28.8/go.mod h1:cBnwIM3fXoRo28SqbV/Ihxf/iviw85KyXOrzxvZQ83U= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= @@ -1024,4 +1118,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 61ebe1fa207..265a6502b60 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -220,6 +220,21 @@ func InitialSettings() []model.SettingItem { {Key: conf.FTPTLSPrivateKeyPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPTLSPublicCertPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + // frp settings + {Key: conf.FRPEnabled, Value: "false", Type: conf.TypeBool, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPServerAddr, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPServerPort, Value: "7000", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPAuthToken, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPProxyName, Value: "alist", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPProxyType, Value: "http", Type: conf.TypeSelect, Options: "http,https,tcp,stcp", Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPCustomDomain, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPSubdomain, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPRemotePort, Value: "0", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPLocalPort, Value: "5244", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPTLSEnable, Value: "false", Type: conf.TypeBool, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPSTCPSecretKey, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE, Help: "Required for stcp proxy type"}, + {Key: conf.FRPStatus, Value: "stopped", Type: conf.TypeString, Group: model.FRP, Flag: model.READONLY}, + // traffic settings {Key: conf.TaskOfflineDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Download.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskOfflineDownloadTransferThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Transfer.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index db2c84dc4ae..5bcb6eb5f91 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -125,6 +125,21 @@ const ( FTPTLSPrivateKeyPath = "ftp_tls_private_key_path" FTPTLSPublicCertPath = "ftp_tls_public_cert_path" + // frp + FRPEnabled = "frp_enabled" + FRPServerAddr = "frp_server_addr" + FRPServerPort = "frp_server_port" + FRPAuthToken = "frp_auth_token" + FRPProxyName = "frp_proxy_name" + FRPProxyType = "frp_proxy_type" + FRPCustomDomain = "frp_custom_domain" + FRPSubdomain = "frp_subdomain" + FRPRemotePort = "frp_remote_port" + FRPLocalPort = "frp_local_port" + FRPTLSEnable = "frp_tls_enable" + FRPSTCPSecretKey = "frp_stcp_secret_key" + FRPStatus = "frp_status" + // traffic TaskOfflineDownloadThreadsNum = "offline_download_task_threads_num" TaskOfflineDownloadTransferThreadsNum = "offline_download_transfer_task_threads_num" diff --git a/internal/model/setting.go b/internal/model/setting.go index 93b81fe5941..9e23f9509e6 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -13,6 +13,7 @@ const ( S3 FTP TRAFFIC + FRP ) const ( From 2edfcaee7be9e57d10b9a8009e122586dde904be Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 15 Apr 2026 20:50:27 +0800 Subject: [PATCH 115/133] add mcp support --- cmd/mcp.go | 27 +++++ cmd/server.go | 23 ++++ go.mod | 4 + go.sum | 18 ++- internal/conf/config.go | 10 ++ internal/model/user.go | 10 ++ server/common/role_perm.go | 2 + server/mcp/auth.go | 182 +++++++++++++++++++++++++++++ server/mcp/convert.go | 56 +++++++++ server/mcp/errors.go | 40 +++++++ server/mcp/server.go | 64 +++++++++++ server/mcp/tools_manage.go | 228 +++++++++++++++++++++++++++++++++++++ server/mcp/tools_read.go | 207 +++++++++++++++++++++++++++++++++ server/mcp/tools_upload.go | 131 +++++++++++++++++++++ 14 files changed, 997 insertions(+), 5 deletions(-) create mode 100644 cmd/mcp.go create mode 100644 server/mcp/auth.go create mode 100644 server/mcp/convert.go create mode 100644 server/mcp/errors.go create mode 100644 server/mcp/server.go create mode 100644 server/mcp/tools_manage.go create mode 100644 server/mcp/tools_read.go create mode 100644 server/mcp/tools_upload.go diff --git a/cmd/mcp.go b/cmd/mcp.go new file mode 100644 index 00000000000..356b8bfc1b8 --- /dev/null +++ b/cmd/mcp.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "github.com/alist-org/alist/v3/internal/bootstrap" + mcpserver "github.com/alist-org/alist/v3/server/mcp" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/spf13/cobra" +) + +var MCPCmd = &cobra.Command{ + Use: "mcp", + Short: "Start MCP server in STDIO mode", + Long: `Start an MCP (Model Context Protocol) server that communicates via STDIO, suitable for integration with AI assistants like Claude Desktop.`, + Run: func(cmd *cobra.Command, args []string) { + Init() + bootstrap.LoadStorages() + username, _ := cmd.Flags().GetString("user") + if err := mcpserver.ServeStdio(username); err != nil { + utils.Log.Fatalf("MCP STDIO server error: %v", err) + } + }, +} + +func init() { + MCPCmd.Flags().String("user", "admin", "Username for MCP operations") + RootCmd.AddCommand(MCPCmd) +} diff --git a/cmd/server.go b/cmd/server.go index 4263f02021d..238b89296e0 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -21,6 +21,7 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server" + mcpserver "github.com/alist-org/alist/v3/server/mcp" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -157,6 +158,19 @@ the address is defined in config file`, }() } } + var mcpHttpSrv *http.Server + if conf.Conf.MCP.Port != -1 && conf.Conf.MCP.Enable { + mcpHandler := mcpserver.NewHTTPHandler() + mcpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.MCP.Port) + utils.Log.Infof("start MCP server @ %s", mcpBase) + mcpHttpSrv = &http.Server{Addr: mcpBase, Handler: mcpHandler} + go func() { + err := mcpHttpSrv.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + utils.Log.Fatalf("failed to start MCP server: %s", err.Error()) + } + }() + } // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) @@ -217,6 +231,15 @@ the address is defined in config file`, } }() } + if conf.Conf.MCP.Port != -1 && conf.Conf.MCP.Enable && mcpHttpSrv != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := mcpHttpSrv.Shutdown(ctx); err != nil { + utils.Log.Fatal("MCP server shutdown err: ", err) + } + }() + } wg.Wait() utils.Log.Println("Server exit") }, diff --git a/go.mod b/go.mod index 17c09201ab0..0807c49e1f1 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/kdomanski/iso9660 v0.4.0 github.com/larksuite/oapi-sdk-go/v3 v3.3.1 + github.com/mark3labs/mcp-go v0.48.0 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.27.2 github.com/mholt/archives v0.1.0 @@ -97,7 +98,10 @@ require ( github.com/emersion/go-message v0.18.0 // indirect github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect + github.com/google/jsonschema-go v0.4.2 // indirect github.com/relvacode/iso8601 v1.3.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect ) require ( diff --git a/go.sum b/go.sum index 370ac6d206f..3baf8885944 100644 --- a/go.sum +++ b/go.sum @@ -241,6 +241,8 @@ github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/sr github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= @@ -325,11 +327,14 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= +github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -456,6 +461,8 @@ github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIg github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mark3labs/mcp-go v0.48.0 h1:o+MXuGW/HCeR2ny5LcAcZQn2bo6I2xaZMEHnpRG+dtw= +github.com/mark3labs/mcp-go v0.48.0/go.mod h1:JKTC7R2LLVagkEWK7Kwu7DbmA6iIvnNAod6yrHiQMag= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= @@ -587,15 +594,12 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIG github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/go-m1cpu v0.2.0 h1:t4GNqvPZ84Vjtpboo/kT3pIkbaK3vc+JIlD/Wz1zSFY= -github.com/shoenig/go-m1cpu v0.2.0/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= @@ -609,6 +613,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -676,6 +682,8 @@ github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 h1:K8gF0eekWPEX+57l30ixxzGhHH/qscI3JCnuhbN6V4M= github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9/go.mod h1:9BnoKCcgJ/+SLhfAXj15352hTOuVmG5Gzo8xNRINfqI= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/internal/conf/config.go b/internal/conf/config.go index 15bd7feba0a..383a5a4e52f 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -94,6 +94,11 @@ type SFTP struct { Listen string `json:"listen" env:"LISTEN"` } +type MCP struct { + Enable bool `json:"enable" env:"ENABLE"` + Port int `json:"port" env:"PORT"` +} + type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` @@ -116,6 +121,7 @@ type Config struct { S3 S3 `json:"s3" envPrefix:"S3_"` FTP FTP `json:"ftp" envPrefix:"FTP_"` SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` + MCP MCP `json:"mcp" envPrefix:"MCP_"` LastLaunchedVersion string `json:"last_launched_version"` } @@ -218,6 +224,10 @@ func DefaultConfig() *Config { Enable: false, Listen: ":5222", }, + MCP: MCP{ + Enable: false, + Port: 5248, + }, LastLaunchedVersion: "", } } diff --git a/internal/model/user.go b/internal/model/user.go index 8ea1ef1aaff..f55b6a5a2a2 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -49,6 +49,8 @@ type User struct { // 12: can read archives // 13: can decompress archives // 14: check path limit + // 15: mcp read + // 16: mcp write Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -144,6 +146,14 @@ func (u *User) CheckPathLimit() bool { return (u.Permission>>14)&1 == 1 } +func (u *User) CanMCPAccess() bool { + return (u.Permission>>15)&1 == 1 +} + +func (u *User) CanMCPManage() bool { + return (u.Permission>>16)&1 == 1 +} + func (u *User) JoinPath(reqPath string) (string, error) { if reqPath == "/" { return utils.FixAndCleanPath(u.BasePath), nil diff --git a/server/common/role_perm.go b/server/common/role_perm.go index 36dedf98c5e..ec82d4d91a0 100644 --- a/server/common/role_perm.go +++ b/server/common/role_perm.go @@ -27,6 +27,8 @@ const ( PermReadArchives PermDecompress PermPathLimit + PermMCPAccess + PermMCPManage ) func HasPermission(perm int32, bit uint) bool { diff --git a/server/mcp/auth.go b/server/mcp/auth.go new file mode 100644 index 00000000000..6d1e670d14a --- /dev/null +++ b/server/mcp/auth.go @@ -0,0 +1,182 @@ +package mcp + +import ( + "context" + "crypto/subtle" + "fmt" + "net/http" + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/server/common" + log "github.com/sirupsen/logrus" +) + +type ctxKey string + +const userKey ctxKey = "user" + +// HTTPContextFunc extracts JWT/admin token from HTTP request and injects user into context. +// Used as WithHTTPContextFunc callback for Streamable HTTP transport. +func HTTPContextFunc(ctx context.Context, r *http.Request) context.Context { + token := r.Header.Get("Authorization") + if token == "" { + token = r.URL.Query().Get("token") + } + + user, err := authenticateToken(token) + if err != nil { + log.Debugf("MCP auth failed: %v", err) + return ctx + } + + return context.WithValue(ctx, userKey, user) +} + +func authenticateToken(token string) (*model.User, error) { + // Check admin static token + if token != "" && subtle.ConstantTimeCompare([]byte(token), []byte(setting.GetStr(conf.Token))) == 1 { + admin, err := op.GetAdmin() + if err != nil { + return nil, fmt.Errorf("failed to get admin: %w", err) + } + if err := loadRoles(admin); err != nil { + return nil, err + } + return admin, nil + } + + // No token: guest + if token == "" { + guest, err := op.GetGuest() + if err != nil { + return nil, fmt.Errorf("failed to get guest: %w", err) + } + if guest.Disabled { + return nil, fmt.Errorf("guest user is disabled") + } + if err := loadRoles(guest); err != nil { + return nil, err + } + return guest, nil + } + + // JWT token + claims, err := common.ParseToken(token) + if err != nil { + return nil, fmt.Errorf("invalid token: %w", err) + } + + user, err := op.GetUserByName(claims.Username) + if err != nil { + return nil, fmt.Errorf("user not found: %w", err) + } + + if claims.PwdTS != user.PwdTS { + return nil, fmt.Errorf("password has been changed") + } + if user.Disabled { + return nil, fmt.Errorf("user is disabled") + } + + if err := loadRoles(user); err != nil { + return nil, err + } + return user, nil +} + +func loadRoles(user *model.User) error { + if len(user.Role) > 0 { + roles, err := op.GetRolesByUserID(user.ID) + if err != nil { + return fmt.Errorf("failed to load roles: %w", err) + } + user.RolesDetail = roles + } + return nil +} + +// resolveUser extracts the authenticated user from context. +func resolveUser(ctx context.Context) (*model.User, error) { + user, ok := ctx.Value(userKey).(*model.User) + if !ok || user == nil { + return nil, fmt.Errorf("authentication required") + } + return user, nil +} + +// buildFsContext resolves path and sets meta in context for fs operations. +func buildFsContext(ctx context.Context, user *model.User, path string) (context.Context, string, error) { + reqPath, err := user.JoinPath(path) + if err != nil { + return ctx, "", err + } + meta, _ := op.GetNearestMeta(reqPath) + ctx = context.WithValue(ctx, "meta", meta) + ctx = context.WithValue(ctx, "user", user) + return ctx, reqPath, nil +} + +// checkAccess checks if user can access the path (read). +func checkAccess(user *model.User, reqPath string) error { + meta, _ := op.GetNearestMeta(reqPath) + if !common.CanAccessWithRoles(user, meta, reqPath, "") { + return fmt.Errorf("permission denied") + } + perm := common.MergeRolePermissions(user, reqPath) + if !user.IsAdmin() && !common.HasPermission(perm, common.PermMCPAccess) { + return fmt.Errorf("MCP access not permitted") + } + return nil +} + +// checkManage checks if user can perform write operations via MCP. +func checkManage(user *model.User, reqPath string, permBit uint) error { + if err := checkAccess(user, reqPath); err != nil { + return err + } + perm := common.MergeRolePermissions(user, reqPath) + if !user.IsAdmin() && !common.HasPermission(perm, common.PermMCPManage) { + return fmt.Errorf("MCP manage not permitted") + } + if !user.IsAdmin() && !common.HasPermission(perm, permBit) { + return fmt.Errorf("permission denied for this operation") + } + return nil +} + +// UserContextFunc returns an HTTPContextFunc that injects a specific user (for STDIO mode). +func userContextMiddleware(user *model.User) func(ctx context.Context) context.Context { + return func(ctx context.Context) context.Context { + return context.WithValue(ctx, userKey, user) + } +} + +// resolveUserForStdio resolves a user by username for STDIO mode. +func resolveUserForStdio(username string) (*model.User, error) { + username = strings.TrimSpace(username) + if username == "" || username == "admin" { + admin, err := op.GetAdmin() + if err != nil { + return nil, fmt.Errorf("failed to get admin user: %w", err) + } + if err := loadRoles(admin); err != nil { + return nil, err + } + return admin, nil + } + user, err := op.GetUserByName(username) + if err != nil { + return nil, fmt.Errorf("user %q not found: %w", username, err) + } + if user.Disabled { + return nil, fmt.Errorf("user %q is disabled", username) + } + if err := loadRoles(user); err != nil { + return nil, err + } + return user, nil +} diff --git a/server/mcp/convert.go b/server/mcp/convert.go new file mode 100644 index 00000000000..7eede8cbc87 --- /dev/null +++ b/server/mcp/convert.go @@ -0,0 +1,56 @@ +package mcp + +import ( + "encoding/json" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/mark3labs/mcp-go/mcp" +) + +type objJSON struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + HashInfo map[string]string `json:"hash_info,omitempty"` +} + +func objToJSON(obj model.Obj) objJSON { + j := objJSON{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + } + hi := obj.GetHash() + if hm := hashInfoToMap(hi); len(hm) > 0 { + j.HashInfo = hm + } + return j +} + +func hashInfoToMap(hi utils.HashInfo) map[string]string { + m := make(map[string]string) + for ht, v := range hi.All() { + if v != "" { + m[ht.Name] = v + } + } + return m +} + +func jsonResult(v interface{}) (*mcp.CallToolResult, error) { + data, err := json.Marshal(v) + if err != nil { + return nil, err + } + return mcp.NewToolResultText(string(data)), nil +} + +func textResult(msg string) (*mcp.CallToolResult, error) { + return mcp.NewToolResultText(msg), nil +} diff --git a/server/mcp/errors.go b/server/mcp/errors.go new file mode 100644 index 00000000000..c107633739f --- /dev/null +++ b/server/mcp/errors.go @@ -0,0 +1,40 @@ +package mcp + +import ( + "fmt" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/mark3labs/mcp-go/mcp" + pkgerr "github.com/pkg/errors" +) + +func toolError(msg string) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(msg), nil +} + +func toolErrorf(format string, args ...interface{}) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(fmt.Sprintf(format, args...)), nil +} + +func wrapError(err error) (*mcp.CallToolResult, error) { + if err == nil { + return nil, nil + } + cause := pkgerr.Cause(err) + switch { + case errs.IsObjectNotFound(err) || errs.IsNotFoundError(err): + return toolErrorf("not found: %s", err.Error()) + case cause == errs.PermissionDenied: + return toolError("permission denied") + case cause == errs.NotImplement: + return toolError("not supported by storage driver") + case cause == errs.NotSupport: + return toolError("operation not supported") + case cause == errs.UploadNotSupported: + return toolError("upload not supported by storage") + case cause == errs.MoveBetweenTwoStorages: + return toolError("can't move between two storages, use copy instead") + default: + return toolErrorf("error: %s", err.Error()) + } +} diff --git a/server/mcp/server.go b/server/mcp/server.go new file mode 100644 index 00000000000..b45a0113eb3 --- /dev/null +++ b/server/mcp/server.go @@ -0,0 +1,64 @@ +package mcp + +import ( + "context" + "net/http" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +// NewServer creates an MCP server with all alist tools registered. +func NewServer() *mcpserver.MCPServer { + s := mcpserver.NewMCPServer( + "alist", + conf.Version, + mcpserver.WithToolCapabilities(false), + mcpserver.WithRecovery(), + ) + registerReadTools(s) + registerManageTools(s) + registerUploadTools(s) + return s +} + +// NewHTTPHandler creates a Streamable HTTP handler for the MCP server. +func NewHTTPHandler() http.Handler { + s := NewServer() + return mcpserver.NewStreamableHTTPServer(s, + mcpserver.WithHTTPContextFunc(HTTPContextFunc), + ) +} + +// NewStdioServer creates an MCP server configured for STDIO mode with a fixed user. +func NewStdioServer(username string) (*mcpserver.MCPServer, *model.User, error) { + user, err := resolveUserForStdio(username) + if err != nil { + return nil, nil, err + } + s := NewServer() + return s, user, nil +} + +// ServeStdio starts the MCP server in STDIO mode. +func ServeStdio(username string) error { + s, user, err := NewStdioServer(username) + if err != nil { + return err + } + ctxFunc := userContextMiddleware(user) + return mcpserver.ServeStdio(s, mcpserver.WithStdioContextFunc(ctxFunc)) +} + +// toolHandlerWithAuth wraps a tool handler to require authentication. +func toolHandlerWithAuth(fn func(ctx context.Context, user *model.User, request mcp.CallToolRequest) (*mcp.CallToolResult, error)) mcpserver.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + user, err := resolveUser(ctx) + if err != nil { + return toolError("authentication required") + } + return fn(ctx, user, request) + } +} diff --git a/server/mcp/tools_manage.go b/server/mcp/tools_manage.go new file mode 100644 index 00000000000..d287bb6bed0 --- /dev/null +++ b/server/mcp/tools_manage.go @@ -0,0 +1,228 @@ +package mcp + +import ( + "context" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +func registerManageTools(s *mcpserver.MCPServer) { + // fs_mkdir + s.AddTool(mcp.NewTool("fs_mkdir", + mcp.WithDescription("Create a new directory"), + mcp.WithString("path", mcp.Required(), mcp.Description("Full path of directory to create")), + ), toolHandlerWithAuth(handleFsMkdir)) + + // fs_rename + s.AddTool(mcp.NewTool("fs_rename", + mcp.WithDescription("Rename a file or directory"), + mcp.WithString("path", mcp.Required(), mcp.Description("Current path of the file/directory")), + mcp.WithString("name", mcp.Required(), mcp.Description("New name (filename only, not a path)")), + ), toolHandlerWithAuth(handleFsRename)) + + // fs_move + s.AddTool(mcp.NewTool("fs_move", + mcp.WithDescription("Move files/directories to another location"), + mcp.WithString("src_dir", mcp.Required(), mcp.Description("Source directory")), + mcp.WithString("dst_dir", mcp.Required(), mcp.Description("Destination directory")), + mcp.WithArray("names", mcp.Description("Names of files/directories to move")), + ), toolHandlerWithAuth(handleFsMove)) + + // fs_copy + s.AddTool(mcp.NewTool("fs_copy", + mcp.WithDescription("Copy files/directories to another location"), + mcp.WithString("src_dir", mcp.Required(), mcp.Description("Source directory")), + mcp.WithString("dst_dir", mcp.Required(), mcp.Description("Destination directory")), + mcp.WithArray("names", mcp.Description("Names of files/directories to copy")), + ), toolHandlerWithAuth(handleFsCopy)) + + // fs_remove + s.AddTool(mcp.NewTool("fs_remove", + mcp.WithDescription("Delete files/directories"), + mcp.WithString("dir", mcp.Required(), mcp.Description("Directory containing items to remove")), + mcp.WithArray("names", mcp.Description("Names of files/directories to remove")), + ), toolHandlerWithAuth(handleFsRemove)) +} + +func handleFsMkdir(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqPath, common.PermWrite); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.MakeDir(ctx, reqPath); err != nil { + return wrapError(err) + } + return textResult("directory created successfully") +} + +func handleFsRename(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + name, err := req.RequireString("name") + if err != nil { + return toolError("name is required") + } + if err := utils.ValidateNameComponent(name); err != nil { + return toolErrorf("invalid name: %s", err.Error()) + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqPath, common.PermRename); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.Rename(ctx, reqPath, name); err != nil { + return wrapError(err) + } + return textResult("renamed successfully") +} + +func handleFsMove(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + srcDirStr, err := req.RequireString("src_dir") + if err != nil { + return toolError("src_dir is required") + } + dstDirStr, err := req.RequireString("dst_dir") + if err != nil { + return toolError("dst_dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + srcDir, err := user.JoinPath(srcDirStr) + if err != nil { + return wrapError(err) + } + dstDir, err := user.JoinPath(dstDirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, srcDir, common.PermMove); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for i, name := range names { + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if err := fs.Move(ctx, srcPath, dstDir, len(names) > i+1); err != nil { + return wrapError(err) + } + } + return textResult("moved successfully") +} + +func handleFsCopy(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + srcDirStr, err := req.RequireString("src_dir") + if err != nil { + return toolError("src_dir is required") + } + dstDirStr, err := req.RequireString("dst_dir") + if err != nil { + return toolError("dst_dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + srcDir, err := user.JoinPath(srcDirStr) + if err != nil { + return wrapError(err) + } + dstDir, err := user.JoinPath(dstDirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, srcDir, common.PermCopy); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for i, name := range names { + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if _, err := fs.Copy(ctx, srcPath, dstDir, len(names) > i+1); err != nil { + return wrapError(err) + } + } + return textResult("copied successfully") +} + +func handleFsRemove(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + dirStr, err := req.RequireString("dir") + if err != nil { + return toolError("dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + reqDir, err := user.JoinPath(dirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqDir, common.PermRemove); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for _, name := range names { + removePath, err := utils.JoinUnderBase(reqDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if err := fs.Remove(ctx, removePath); err != nil { + return wrapError(err) + } + } + return textResult("removed successfully") +} + +// getStringArray extracts a string array from tool request arguments. +func getStringArray(req mcp.CallToolRequest, name string) []string { + args := req.GetArguments() + val, ok := args[name] + if !ok { + return nil + } + arr, ok := val.([]interface{}) + if !ok { + return nil + } + result := make([]string, 0, len(arr)) + for _, v := range arr { + if s, ok := v.(string); ok { + result = append(result, s) + } + } + return result +} diff --git a/server/mcp/tools_read.go b/server/mcp/tools_read.go new file mode 100644 index 00000000000..d8af570b233 --- /dev/null +++ b/server/mcp/tools_read.go @@ -0,0 +1,207 @@ +package mcp + +import ( + "context" + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/search" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" + "github.com/pkg/errors" +) + +func registerReadTools(s *mcpserver.MCPServer) { + // fs_list + s.AddTool(mcp.NewTool("fs_list", + mcp.WithDescription("List files and directories at the given path"), + mcp.WithString("path", mcp.Required(), mcp.Description("Directory path to list")), + mcp.WithNumber("page", mcp.Description("Page number (default: 1)")), + mcp.WithNumber("per_page", mcp.Description("Items per page (default: 30, max: 500)")), + mcp.WithBoolean("refresh", mcp.Description("Force refresh from storage (default: false)")), + ), toolHandlerWithAuth(handleFsList)) + + // fs_get + s.AddTool(mcp.NewTool("fs_get", + mcp.WithDescription("Get file or directory metadata"), + mcp.WithString("path", mcp.Required(), mcp.Description("Path to file or directory")), + ), toolHandlerWithAuth(handleFsGet)) + + // fs_search + s.AddTool(mcp.NewTool("fs_search", + mcp.WithDescription("Search for files by keywords"), + mcp.WithString("path", mcp.Required(), mcp.Description("Parent directory to search within")), + mcp.WithString("keywords", mcp.Required(), mcp.Description("Search keywords")), + mcp.WithNumber("scope", mcp.Description("0=all, 1=dir only, 2=file only (default: 0)")), + mcp.WithNumber("page", mcp.Description("Page number (default: 1)")), + mcp.WithNumber("per_page", mcp.Description("Items per page (default: 20)")), + ), toolHandlerWithAuth(handleFsSearch)) + + // fs_download_url + s.AddTool(mcp.NewTool("fs_download_url", + mcp.WithDescription("Get download URL for a file"), + mcp.WithString("path", mcp.Required(), mcp.Description("Path to the file")), + ), toolHandlerWithAuth(handleFsDownloadURL)) +} + +func handleFsList(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + page := intParam(req, "page", 1) + perPage := intParam(req, "per_page", 30) + if perPage > 500 { + perPage = 500 + } + refresh := req.GetBool("refresh", false) + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + objs, err := fs.List(ctx, reqPath, &fs.ListArgs{Refresh: refresh}) + if err != nil { + return wrapError(err) + } + + // Paginate + total := len(objs) + start := (page - 1) * perPage + if start > total { + start = total + } + end := start + perPage + if end > total { + end = total + } + pageObjs := objs[start:end] + + items := make([]objJSON, 0, len(pageObjs)) + for _, obj := range pageObjs { + items = append(items, objToJSON(obj)) + } + + return jsonResult(map[string]interface{}{ + "content": items, + "total": total, + "page": page, + "per_page": perPage, + }) +} + +func handleFsGet(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + if err != nil { + return wrapError(err) + } + + return jsonResult(objToJSON(obj)) +} + +func handleFsSearch(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + keywords, err := req.RequireString("keywords") + if err != nil { + return toolError("keywords is required") + } + scope := intParam(req, "scope", 0) + page := intParam(req, "page", 1) + perPage := intParam(req, "per_page", 20) + + parent, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + + searchReq := model.SearchReq{ + Parent: parent, + Keywords: keywords, + Scope: scope, + PageReq: model.PageReq{Page: page, PerPage: perPage}, + } + if err := searchReq.Validate(); err != nil { + return toolErrorf("invalid search request: %s", err.Error()) + } + + nodes, total, err := search.Search(ctx, searchReq) + if err != nil { + return wrapError(err) + } + + // Filter by permission + filtered := make([]model.SearchNode, 0, len(nodes)) + for _, node := range nodes { + if !strings.HasPrefix(node.Parent, user.BasePath) { + continue + } + meta, err := op.GetNearestMeta(node.Parent) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + continue + } + if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), "") { + continue + } + filtered = append(filtered, node) + } + + return jsonResult(map[string]interface{}{ + "content": filtered, + "total": total, + }) +} + +func handleFsDownloadURL(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{}) + if err != nil { + return wrapError(err) + } + + return jsonResult(map[string]interface{}{ + "raw_url": link.URL, + }) +} + +// intParam extracts an integer parameter with a default value. +func intParam(req mcp.CallToolRequest, name string, defaultVal int) int { + v := req.GetFloat(name, float64(defaultVal)) + return int(v) +} diff --git a/server/mcp/tools_upload.go b/server/mcp/tools_upload.go new file mode 100644 index 00000000000..c84eb9623a9 --- /dev/null +++ b/server/mcp/tools_upload.go @@ -0,0 +1,131 @@ +package mcp + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + stdpath "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +func registerUploadTools(s *mcpserver.MCPServer) { + s.AddTool(mcp.NewTool("fs_upload", + mcp.WithDescription("Upload a local file to alist. Automatically uses direct internal upload (local deployment) or HTTP API upload (remote deployment)."), + mcp.WithString("path", mcp.Required(), mcp.Description("Destination path in alist including filename")), + mcp.WithString("local_path", mcp.Required(), mcp.Description("Absolute local file path to upload")), + ), toolHandlerWithAuth(handleFsUpload)) +} + +func handleFsUpload(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + localPath, err := req.RequireString("local_path") + if err != nil { + return toolError("local_path is required") + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + dir := stdpath.Dir(reqPath) + + if err := checkManage(user, dir, common.PermWrite); err != nil { + return toolError(err.Error()) + } + + if strings.Contains(conf.Conf.SiteURL, "://") { + return uploadViaHTTP(reqPath, localPath) + } + return uploadDirectly(ctx, user, reqPath, localPath) +} + +func uploadDirectly(ctx context.Context, user *model.User, reqPath, localPath string) (*mcp.CallToolResult, error) { + file, err := os.Open(localPath) + if err != nil { + return toolErrorf("failed to open local file: %s", err.Error()) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return toolErrorf("failed to stat local file: %s", err.Error()) + } + + dir := stdpath.Dir(reqPath) + name := stdpath.Base(reqPath) + + fileStream := &stream.FileStream{ + Ctx: ctx, + Obj: &model.Object{ + Name: name, + Size: info.Size(), + Modified: time.Now(), + IsFolder: false, + }, + Reader: io.NopCloser(file), + Closers: utils.EmptyClosers(), + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.PutDirectly(ctx, dir, fileStream); err != nil { + return wrapError(err) + } + return textResult("uploaded successfully") +} + +func uploadViaHTTP(reqPath, localPath string) (*mcp.CallToolResult, error) { + file, err := os.Open(localPath) + if err != nil { + return toolErrorf("failed to open local file: %s", err.Error()) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return toolErrorf("failed to stat local file: %s", err.Error()) + } + + name := stdpath.Base(reqPath) + apiURL := fmt.Sprintf("%s/api/fs/put", conf.Conf.SiteURL) + + httpReq, err := http.NewRequest(http.MethodPut, apiURL, file) + if err != nil { + return toolErrorf("failed to create request: %s", err.Error()) + } + + httpReq.Header.Set("File-Path", url.PathEscape(reqPath)) + httpReq.Header.Set("Content-Length", strconv.FormatInt(info.Size(), 10)) + httpReq.Header.Set("Content-Type", utils.GetMimeType(name)) + httpReq.Header.Set("Authorization", setting.GetStr(conf.Token)) + httpReq.ContentLength = info.Size() + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return toolErrorf("upload request failed: %s", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return toolErrorf("upload failed (HTTP %d): %s", resp.StatusCode, string(body)) + } + return textResult("uploaded successfully") +} From 135a0b433e54d4c7ea0e7202e710b7c6901ad769 Mon Sep 17 00:00:00 2001 From: kyle-meng <2078739489@qq.com> Date: Sun, 26 Apr 2026 14:41:44 +0800 Subject: [PATCH 116/133] =?UTF-8?q?feat(139-share):=20support=20mounting?= =?UTF-8?q?=20and=20HLS=20playback=20|=20=E6=94=AF=E6=8C=81=E5=88=86?= =?UTF-8?q?=E4=BA=AB=E9=93=BE=E6=8E=A5=E6=8C=82=E8=BD=BD=E4=B8=8E=E6=92=AD?= =?UTF-8?q?=E6=94=BE=20Root=20cause:=20139=20Cloud=20share=20links=20use?= =?UTF-8?q?=20relative=20TS=20paths=20in=20M3U8=20playlists=20which=20cann?= =?UTF-8?q?ot=20be=20resolved=20by=20proxied=20clients.=20Additionally,=20?= =?UTF-8?q?AList's=20downloader=20enforces=20strict=20metadata-to-stream?= =?UTF-8?q?=20size=20validation,=20leading=20to=20416=20(Range)=20or=20EOF?= =?UTF-8?q?=20errors=20when=20serving=20dynamic=20M3U8=20content.=20We=20i?= =?UTF-8?q?mplemented=20a=201MB=20padding=20technique=20to=20ensure=20comp?= =?UTF-8?q?atibility=20with=20AList's=20strict=20size=20checks;=201MB=20is?= =?UTF-8?q?=20sufficient=20for=20almost=20all=20M3U8=20files=20without=20i?= =?UTF-8?q?mpacting=20performance.=20|=20139=E4=BA=91=E7=9B=98=E5=88=86?= =?UTF-8?q?=E4=BA=AB=E9=93=BE=E6=8E=A5=E5=9C=A8M3U8=E4=B8=AD=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E7=9B=B8=E5=AF=B9TS=E8=B7=AF=E5=BE=84=EF=BC=8C?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E4=BB=A3=E7=90=86=E8=AF=B7=E6=B1=82=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E6=AD=A3=E5=B8=B8=E8=A7=A3=E6=9E=90=E3=80=82=E6=AD=A4?= =?UTF-8?q?=E5=A4=96=EF=BC=8CAList=E4=B8=8B=E8=BD=BD=E5=99=A8=E4=BC=9A?= =?UTF-8?q?=E4=B8=A5=E6=A0=BC=E6=A0=A1=E9=AA=8C=E6=96=87=E4=BB=B6=E5=85=83?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E4=B8=8E=E5=AE=9E=E9=99=85=E6=B5=81=E7=9A=84?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E4=B8=80=E8=87=B4=E6=80=A7=EF=BC=8C=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=8A=A8=E6=80=81=E7=94=9F=E6=88=90=E7=9A=84M3U8?= =?UTF-8?q?=E5=9B=A0=E9=95=BF=E5=BA=A6=E4=B8=8D=E5=8C=B9=E9=85=8D=E8=A7=A6?= =?UTF-8?q?=E5=8F=91416=E6=88=96EOF=E9=94=99=E8=AF=AF=E3=80=82=E6=88=91?= =?UTF-8?q?=E4=BB=AC=E9=87=87=E7=94=A8=E4=BA=861MB=E5=A1=AB=E5=85=85?= =?UTF-8?q?=E6=8A=80=E6=9C=AF=E4=BB=A5=E5=85=BC=E5=AE=B9AList=E7=9A=84?= =?UTF-8?q?=E4=B8=A5=E6=A0=BC=E6=A0=A1=E9=AA=8C=EF=BC=8C=E4=B8=941MB?= =?UTF-8?q?=E8=B6=B3=E4=BB=A5=E5=AE=B9=E7=BA=B3=E7=BB=9D=E5=A4=A7=E5=A4=9A?= =?UTF-8?q?=E6=95=B0M3U8=E6=96=87=E4=BB=B6=E8=80=8C=E4=B8=8D=E5=BD=B1?= =?UTF-8?q?=E5=93=8D=E6=80=A7=E8=83=BD=E3=80=82=20Changes:=20alist/drivers?= =?UTF-8?q?/139/types.go=20=20=20-=20Added=20ShareCatalog=20and=20ShareCon?= =?UTF-8?q?tent=20structs=20for=20API=20response=20mapping=20|=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=88=86=E4=BA=AB=E7=9B=AE=E5=BD=95=E4=B8=8E=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E7=9A=84API=E5=93=8D=E5=BA=94=E6=98=A0=E5=B0=84?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E4=BD=93=20alist/drivers/139/meta.go=20=20?= =?UTF-8?q?=20-=20Integrated=20'share'=20storage=20type=20and=20simplified?= =?UTF-8?q?=20struct=20tags=20for=20UI=20cleanliness=20|=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E5=88=86=E4=BA=AB=E5=AD=98=E5=82=A8=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=B9=B6=E7=B2=BE=E7=AE=80=E7=BB=93=E6=9E=84=E4=BD=93=E6=A0=87?= =?UTF-8?q?=E7=AD=BE=E4=BB=A5=E7=A1=AE=E4=BF=9D=E7=95=8C=E9=9D=A2=E6=95=B4?= =?UTF-8?q?=E6=B4=81=20alist/drivers/139/driver.go=20=20=20-=20Implemented?= =?UTF-8?q?=20share=20mode=20handling=20and=20forced=201MB=20file=20size?= =?UTF-8?q?=20for=20video=20listings=20|=20=E5=AE=9E=E7=8E=B0=E5=88=86?= =?UTF-8?q?=E4=BA=AB=E6=A8=A1=E5=BC=8F=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E5=B9=B6=E5=9C=A8=E5=88=97=E8=A1=A8=E6=97=B6=E5=B0=86=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E5=A4=A7=E5=B0=8F=E5=BC=BA=E5=88=B6=E5=A3=B0=E6=98=8E?= =?UTF-8?q?=E4=B8=BA1MB=20alist/drivers/139/util.go=20=20=20-=20Implemente?= =?UTF-8?q?d=20M3U8=20absolute=20URL=20rewriter=20and=20a=20padded=20Range?= =?UTF-8?q?ReadCloser=20to=20ensure=20proxy=20compatibility=20|=20?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0M3U8=E7=BB=9D=E5=AF=B9=E8=B7=AF=E5=BE=84?= =?UTF-8?q?=E9=87=8D=E5=86=99=E5=99=A8=E5=8F=8A=E5=B8=A6=E5=A1=AB=E5=85=85?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E7=9A=84=E8=AF=BB=E5=8F=96=E5=99=A8=E4=BB=A5?= =?UTF-8?q?=E9=80=82=E9=85=8D=E4=BB=A3=E7=90=86=E6=A0=A1=E9=AA=8C=20=20=20?= =?UTF-8?q?-=20Cleaned=20up=20all=20debug=20logging=20and=20temporary=20co?= =?UTF-8?q?de=20for=20production=20readiness=20|=20=E6=B8=85=E7=90=86?= =?UTF-8?q?=E4=BA=86=E6=89=80=E6=9C=89=E8=B0=83=E8=AF=95=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=92=8C=E4=B8=B4=E6=97=B6=E4=BB=A3=E7=A0=81=E4=BB=A5=E8=BE=BE?= =?UTF-8?q?=E5=88=B0=E5=8F=91=E5=B8=83=E6=A0=87=E5=87=86=20Verified:=20Suc?= =?UTF-8?q?cessfully=20mounted=20share=20links;=20shared=20videos=20play?= =?UTF-8?q?=20via=20HLS=20without=20416=20errors;=20padded=20content=20siz?= =?UTF-8?q?e=20matches=20the=201MB=20metadata.=20|=20=E6=88=90=E5=8A=9F?= =?UTF-8?q?=E6=8C=82=E8=BD=BD=E5=88=86=E4=BA=AB=E9=93=BE=E6=8E=A5=EF=BC=9B?= =?UTF-8?q?=E8=A7=86=E9=A2=91=E5=8F=AF=E9=80=9A=E8=BF=87HLS=E6=AD=A3?= =?UTF-8?q?=E5=B8=B8=E6=92=AD=E6=94=BE=E4=B8=94=E6=97=A0416=E9=94=99?= =?UTF-8?q?=E8=AF=AF=EF=BC=9B=E5=A1=AB=E5=85=85=E5=90=8E=E7=9A=84=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E5=A4=A7=E5=B0=8F=E4=B8=8E=E5=A3=B0=E6=98=8E=E7=9A=84?= =?UTF-8?q?1MB=E5=85=83=E6=95=B0=E6=8D=AE=E5=AE=8C=E7=BE=8E=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/139/driver.go | 9 ++ drivers/139/meta.go | 10 +- drivers/139/types.go | 36 +++++ drivers/139/util.go | 340 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 390 insertions(+), 5 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index c182dd24e93..10d9d3e9e03 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -76,6 +76,10 @@ func (d *Yun139) Init(ctx context.Context) error { d.RootFolderID = d.CloudID } case MetaFamily: + case "share": + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = "root" + } default: return errs.NotImplement } @@ -100,6 +104,7 @@ func (d *Yun139) Drop(ctx context.Context) error { } func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + log.Infof("[139Share-Debug] List called! Type: %s, DirID: %s", d.Addition.Type, dir.GetID()) switch d.Addition.Type { case MetaPersonalNew: return d.personalGetFiles(dir.GetID()) @@ -109,6 +114,8 @@ func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( return d.familyGetFiles(dir.GetID()) case MetaGroup: return d.groupGetFiles(dir.GetID()) + case "share": + return d.shareGetFiles(dir.GetID()) default: return nil, errs.NotImplement } @@ -126,6 +133,8 @@ func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) url, err = d.familyGetLink(file.GetID(), file.GetPath()) case MetaGroup: url, err = d.groupGetLink(file.GetID(), file.GetPath()) + case "share": + return d.shareGetLink(file.GetID()) default: return nil, errs.NotImplement } diff --git a/drivers/139/meta.go b/drivers/139/meta.go index c02b1347587..ea90df86cf9 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -6,14 +6,14 @@ import ( ) type Addition struct { - //Account string `json:"account" required:"true"` Authorization string `json:"authorization" type:"text" required:"true"` driver.RootID - Type string `json:"type" type:"select" options:"personal_new,family,group,personal" default:"personal_new"` + Type string `json:"type" type:"select" options:"personal_new,family,group,personal,share" default:"personal_new"` CloudID string `json:"cloud_id"` - CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` - ReportRealSize bool `json:"report_real_size" type:"bool" default:"true" help:"Enable to report the real file size during upload"` - UseLargeThumbnail bool `json:"use_large_thumbnail" type:"bool" default:"false" help:"Enable to use large thumbnail for images"` + LinkID string `json:"link_id"` + CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0"` + ReportRealSize bool `json:"report_real_size" type:"bool" default:"true"` + UseLargeThumbnail bool `json:"use_large_thumbnail" type:"bool" default:"false"` } var config = driver.Config{ diff --git a/drivers/139/types.go b/drivers/139/types.go index d5f025a1672..e3a2adc68b7 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -312,3 +312,39 @@ type RefreshTokenResp struct { AccessToken string `xml:"accessToken"` Desc string `xml:"desc"` } + +type ShareCatalog struct { + CaID string `json:"caID"` + CaName string `json:"caName"` +} + +type ShareContent struct { + CoID string `json:"coID"` + CoName string `json:"coName"` + CoSize int64 `json:"coSize"` + CoType int `json:"coType"` + CoSuffix string `json:"coSuffix"` +} + +type ShareListResp struct { + Data struct { + CaLst []ShareCatalog `json:"caLst"` + CoLst []ShareContent `json:"coLst"` + } `json:"data"` +} + +type ShareLinkResp struct { + DownloadURL string `json:"downloadURL"` +} +type ShareContentInfo struct { + ContentName string `json:"contentName"` + ContentSize int64 `json:"contentSize"` + PresentURL string `json:"presentURL"` // HLS 播放地址 + DownloadURL string `json:"cdnDownLoadUrl"` // 真正的下载直链 +} + +type ShareContentInfoResp struct { + Data struct { + ContentInfo ShareContentInfo `json:"contentInfo"` + } `json:"data"` +} \ No newline at end of file diff --git a/drivers/139/util.go b/drivers/139/util.go index 79c78842470..2bece66ba4a 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -1,9 +1,16 @@ package _139 import ( + "bytes" + "compress/gzip" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" "encoding/base64" "errors" "fmt" + "io" "net/http" "net/url" "path" @@ -12,11 +19,14 @@ import ( "strings" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/gin-gonic/gin" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" @@ -666,3 +676,333 @@ func (d *Yun139) getPersonalCloudHost() string { } return d.PersonalCloudHost } + +func (d *Yun139) sharePost(pathname string, data interface{}, resp interface{}) ([]byte, error) { + crypto := NewYunCrypto() + encryptedBody, err := crypto.Encrypt(data) + if err != nil { + return nil, err + } + + url := "https://share-kd-njs.yun.139.com" + pathname + req := base.RestyClient.R() + + auth := d.getAuthorization() + if !strings.HasPrefix(auth, "Basic ") { + auth = "Basic " + auth + } + // randStr := random.String(16) + // ts := time.Now().Format("2006-01-02 15:04:05") + // body, err := utils.Json.Marshal(req.Body) + // if err != nil { + // return nil, err + // } + // sign := calSign(string(body), ts, randStr) + // svcType := "1" + // if d.isFamily() { + // svcType = "2" + // } + req.SetHeaders(map[string]string{ + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0", + "Accept": "application/json, text/plain, */*", + "Content-Type": "application/json;charset=UTF-8", + "Authorization": auth, + "X-Deviceinfo": "||9|12.27.0|firefox|140.0|12b780037221ab547c682223327dc9cd||linux unknow|1920X526|zh-CN|||", + "hcy-cool-flag": "1", + "CMS-DEVICE": "default", + "x-m4c-caller": "PC", + "X-Yun-Api-Version": "v1", + "Origin": "https://yun.139.com", + "Referer": "https://yun.139.com/", + }) + req.SetBody(encryptedBody) + + res, err := req.Post(url) + if err != nil { + return nil, err + } + + decryptedText, err := crypto.Decrypt(res.String()) + if err != nil { + log.Errorf("[139Share] Decryption failed, raw response: %s", res.String()) + return nil, fmt.Errorf("decryption failed: %v, raw: %s", err, res.String()) + } + + if resp != nil { + err = utils.Json.Unmarshal([]byte(decryptedText), resp) + if err != nil { + return nil, err + } + } + return []byte(decryptedText), nil +} + +func (d *Yun139) shareGetFiles(pCaID string) ([]model.Obj, error) { + if pCaID == "" { + pCaID = "root" + } + data := base.Json{ + "getOutLinkInfoReq": base.Json{ + "account": d.getAccount(), + "linkID": d.LinkID, + "pCaID": pCaID, + }, + } + var resp ShareListResp + _, err := d.sharePost("/yun-share/richlifeApp/devapp/IOutLink/getOutLinkInfoV6", data, &resp) + if err != nil { + return nil, err + } + files := make([]model.Obj, 0) + // 直接从 Data 中读取 CaLst + for _, catalog := range resp.Data.CaLst { + f := model.Object{ + ID: catalog.CaID, + Name: catalog.CaName, + IsFolder: true, + } + files = append(files, &f) + } + for _, content := range resp.Data.CoLst { + name := content.CoName + size := content.CoSize + // 如果是视频，强行加 .m3u8 后缀，并声明为 1MB 以匹配 Padding 逻辑 + if content.CoType == 3 || strings.HasSuffix(strings.ToLower(name), ".mp4") { + if !strings.HasSuffix(name, ".m3u8") { + name += ".m3u8" + } + size = 1024 * 1024 // 关键：声明为 1MB + } + f := model.Object{ + ID: content.CoID, + Name: name, + Size: size, + } + files = append(files, &f) + } + + return files, nil +} + + + +type YunCrypto struct { + Key []byte + BlockSize int +} + +func NewYunCrypto() *YunCrypto { + return &YunCrypto{ + Key: []byte("PVGDwmcvfs1uV3d1"), + BlockSize: aes.BlockSize, + } +} + +func (y *YunCrypto) PKCS7Padding(ciphertext []byte, blockSize int) []byte { + padding := blockSize - len(ciphertext)%blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + return append(ciphertext, padtext...) +} + +func (y *YunCrypto) PKCS7UnPadding(origData []byte) ([]byte, error) { + length := len(origData) + if length == 0 { + return nil, errors.New("data is empty") + } + unpadding := int(origData[length-1]) + if length < unpadding { + return nil, errors.New("unpadding error") + } + return origData[:(length - unpadding)], nil +} + +func (y *YunCrypto) Encrypt(data interface{}) (string, error) { + jsonData, err := utils.Json.Marshal(data) + if err != nil { + return "", err + } + iv := make([]byte, y.BlockSize) + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return "", err + } + block, err := aes.NewCipher(y.Key) + if err != nil { + return "", err + } + content := y.PKCS7Padding(jsonData, y.BlockSize) + ciphertext := make([]byte, len(content)) + mode := cipher.NewCBCEncrypter(block, iv) + mode.CryptBlocks(ciphertext, content) + result := append(iv, ciphertext...) + return base64.StdEncoding.EncodeToString(result), nil +} + +func (y *YunCrypto) Decrypt(b64Data string) (string, error) { + b64Data = strings.Join(strings.Fields(b64Data), "") + raw, err := base64.StdEncoding.DecodeString(b64Data) + if err != nil { + return "", err + } + if len(raw) < y.BlockSize { + return "", errors.New("data too short") + } + iv := raw[:y.BlockSize] + ciphertext := raw[y.BlockSize:] + block, err := aes.NewCipher(y.Key) + if err != nil { + return "", err + } + decrypted := make([]byte, len(ciphertext)) + mode := cipher.NewCBCDecrypter(block, iv) + mode.CryptBlocks(decrypted, ciphertext) + if len(decrypted) > 2 && decrypted[0] == 0x1f && decrypted[1] == 0x8b { + reader, err := gzip.NewReader(bytes.NewReader(decrypted)) + if err == nil { + defer reader.Close() + unzipped, err := io.ReadAll(reader) + if err == nil { + return string(unzipped), nil + } + } + } + unpadded, err := y.PKCS7UnPadding(decrypted) + if err != nil { + return strings.TrimSpace(string(decrypted)), nil + } + return string(unpadded), nil +} + +func (d *Yun139) rewriteM3U8(masterURL string) (string, error) { + client := resty.New().SetTimeout(10 * time.Second) + headers := map[string]string{ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Referer": "https://yun.139.com/", + } + + // 1. 获取 Master M3U8 + resp, err := client.R().SetHeaders(headers).Get(masterURL) + if err != nil { + return "", err + } + masterContent := resp.String() + + // 2. 找到子播放列表路径 + var subRelPath string + lines := strings.Split(masterContent, "\n") + for i, line := range lines { + if strings.Contains(line, "RESOLUTION=") { + if i+1 < len(lines) { + subRelPath = strings.TrimSpace(lines[i+1]) + if strings.Contains(line, "1920x1080") { + break + } + } + } + } + if subRelPath == "" { + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if line != "" && !strings.HasPrefix(line, "#") { + subRelPath = line + break + } + } + } + if subRelPath == "" { + return "", fmt.Errorf("could not find sub-playlist") + } + + // 3. 获取子播放列表内容 + base, _ := url.Parse(masterURL) + if subRelPath == "" { + return "", fmt.Errorf("sub playlist not found in master m3u8") + } + ref, _ := url.Parse(subRelPath) + subURL := base.ResolveReference(ref).String() + + resp, err = client.R().SetHeaders(headers).Get(subURL) + if err != nil { + return "", err + } + subContent := resp.String() + + subBase, _ := url.Parse(subURL) + subLines := strings.Split(subContent, "\n") + var finalLines []string + for _, line := range subLines { + cleanLine := strings.TrimSpace(line) + if cleanLine != "" && !strings.HasPrefix(cleanLine, "#") { + if !strings.HasPrefix(cleanLine, "http") { + tsRef, _ := url.Parse(cleanLine) + finalLines = append(finalLines, subBase.ResolveReference(tsRef).String()) + } else { + finalLines = append(finalLines, cleanLine) + } + } else { + finalLines = append(finalLines, line) + } + } + + finalM3U8 := strings.Join(finalLines, "\n") + + return finalM3U8, nil +} + +func (d *Yun139) Proxy(c *gin.Context, obj model.Obj) error { + return nil +} + +func (d *Yun139) shareGetLink(coID string) (*model.Link, error) { + data := base.Json{ + "getContentInfoFromOutLinkReq": base.Json{ + "contentId": coID, + "linkID": d.LinkID, + "account": d.getAccount(), + }, + } + var resp ShareContentInfoResp + _, err := d.sharePost("/yun-share/richlifeApp/devapp/IOutLink/getContentInfoFromOutLink", data, &resp) + if err != nil { + return nil, err + } + + res := resp.Data.ContentInfo + if res.PresentURL != "" { + m3u8Content, err := d.rewriteM3U8(res.PresentURL) + if err != nil { + return nil, err + } + + // 核心逻辑：填充到 1MB，确保 AList 不报大小错误 + targetSize := int64(1024 * 1024) + contentBytes := []byte(m3u8Content) + if int64(len(contentBytes)) < targetSize { + padding := bytes.Repeat([]byte(" "), int(targetSize-int64(len(contentBytes)))) + contentBytes = append(contentBytes, padding...) + } else { + // 如果 M3U8 竟然超过了 1MB（极罕见），则按实际大小截断（或报错） + contentBytes = contentBytes[:targetSize] + } + + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{ + RangeReader: func(ctx context.Context, range_ http_range.Range) (io.ReadCloser, error) { + reader := bytes.NewReader(contentBytes) + // 处理 AList 的 Range 请求 + _, _ = reader.Seek(range_.Start, io.SeekStart) + // 包装成 ReadCloser + return io.NopCloser(reader), nil + }, + }, + Header: http.Header{ + "Content-Type": []string{"application/vnd.apple.mpegurl"}, + }, + }, nil + } + + if res.DownloadURL != "" { + return &model.Link{URL: res.DownloadURL}, nil + } + + return nil, fmt.Errorf("failed to get link") +} From 45daac9e04e4e3471758468b7d17e02516a553f5 Mon Sep 17 00:00:00 2001 From: kyle-meng <2078739489@qq.com> Date: Sun, 26 Apr 2026 16:02:23 +0800 Subject: [PATCH 117/133] =?UTF-8?q?fix(139-share):=20fix=20modification=20?= =?UTF-8?q?time=20parsing=20|=20=E4=BF=AE=E5=A4=8D=E5=88=86=E4=BA=AB?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E4=B8=8B=E7=9A=84=E4=BF=AE=E6=94=B9=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/139/types.go | 2 ++ drivers/139/util.go | 34 ++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/drivers/139/types.go b/drivers/139/types.go index e3a2adc68b7..03f88aa7aba 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -316,6 +316,7 @@ type RefreshTokenResp struct { type ShareCatalog struct { CaID string `json:"caID"` CaName string `json:"caName"` + UdTime string `json:"udTime"` } type ShareContent struct { @@ -323,6 +324,7 @@ type ShareContent struct { CoName string `json:"coName"` CoSize int64 `json:"coSize"` CoType int `json:"coType"` + UdTime string `json:"udTime"` CoSuffix string `json:"coSuffix"` } diff --git a/drivers/139/util.go b/drivers/139/util.go index 2bece66ba4a..3823569ad70 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -756,9 +756,11 @@ func (d *Yun139) shareGetFiles(pCaID string) ([]model.Obj, error) { files := make([]model.Obj, 0) // 直接从 Data 中读取 CaLst for _, catalog := range resp.Data.CaLst { + modTime, _ := time.ParseInLocation("20060102150405", catalog.UdTime, utils.CNLoc) f := model.Object{ ID: catalog.CaID, Name: catalog.CaName, + Modified: modTime, IsFolder: true, } files = append(files, &f) @@ -766,17 +768,19 @@ func (d *Yun139) shareGetFiles(pCaID string) ([]model.Obj, error) { for _, content := range resp.Data.CoLst { name := content.CoName size := content.CoSize - // 如果是视频，强行加 .m3u8 后缀，并声明为 1MB 以匹配 Padding 逻辑 + // Force .m3u8 suffix for videos and declare 1MB size for padding logic if content.CoType == 3 || strings.HasSuffix(strings.ToLower(name), ".mp4") { if !strings.HasSuffix(name, ".m3u8") { name += ".m3u8" } - size = 1024 * 1024 // 关键：声明为 1MB + size = 1024 * 1024 // Key: declare 1MB to match RangeReadCloser padding } + modTime, _ := time.ParseInLocation("20060102150405", content.UdTime, utils.CNLoc) f := model.Object{ - ID: content.CoID, - Name: name, - Size: size, + ID: content.CoID, + Name: name, + Size: size, + Modified: modTime, } files = append(files, &f) } @@ -879,14 +883,14 @@ func (d *Yun139) rewriteM3U8(masterURL string) (string, error) { "Referer": "https://yun.139.com/", } - // 1. 获取 Master M3U8 + // 1. Get Master M3U8 resp, err := client.R().SetHeaders(headers).Get(masterURL) if err != nil { return "", err } masterContent := resp.String() - // 2. 找到子播放列表路径 + // 2. Find sub-playlist path var subRelPath string lines := strings.Split(masterContent, "\n") for i, line := range lines { @@ -909,14 +913,11 @@ func (d *Yun139) rewriteM3U8(masterURL string) (string, error) { } } if subRelPath == "" { - return "", fmt.Errorf("could not find sub-playlist") + return "", fmt.Errorf("sub playlist not found in master m3u8") } - // 3. 获取子播放列表内容 + // 3. Get sub-playlist content base, _ := url.Parse(masterURL) - if subRelPath == "" { - return "", fmt.Errorf("sub playlist not found in master m3u8") - } ref, _ := url.Parse(subRelPath) subURL := base.ResolveReference(ref).String() @@ -926,6 +927,7 @@ func (d *Yun139) rewriteM3U8(masterURL string) (string, error) { } subContent := resp.String() + // 4. Resolve relative TS paths to absolute URLs subBase, _ := url.Parse(subURL) subLines := strings.Split(subContent, "\n") var finalLines []string @@ -973,14 +975,14 @@ func (d *Yun139) shareGetLink(coID string) (*model.Link, error) { return nil, err } - // 核心逻辑：填充到 1MB，确保 AList 不报大小错误 + // Core logic: pad to 1MB to ensure compatibility with AList's size validation targetSize := int64(1024 * 1024) contentBytes := []byte(m3u8Content) if int64(len(contentBytes)) < targetSize { padding := bytes.Repeat([]byte(" "), int(targetSize-int64(len(contentBytes)))) contentBytes = append(contentBytes, padding...) } else { - // 如果 M3U8 竟然超过了 1MB（极罕见），则按实际大小截断（或报错） + // Truncate if M3U8 exceeds 1MB (extremely rare) contentBytes = contentBytes[:targetSize] } @@ -988,9 +990,9 @@ func (d *Yun139) shareGetLink(coID string) (*model.Link, error) { RangeReadCloser: &model.RangeReadCloser{ RangeReader: func(ctx context.Context, range_ http_range.Range) (io.ReadCloser, error) { reader := bytes.NewReader(contentBytes) - // 处理 AList 的 Range 请求 + // Handle AList Range requests _, _ = reader.Seek(range_.Start, io.SeekStart) - // 包装成 ReadCloser + // Wrap as ReadCloser return io.NopCloser(reader), nil }, }, From dba5c279ca72faa322cde125cc0b62566a39153b Mon Sep 17 00:00:00 2001 From: abandonstudy <94209629+abandonstudy@users.noreply.github.com> Date: Thu, 30 Apr 2026 22:43:14 +0800 Subject: [PATCH 118/133] fix(guangyapan): allow user input folder path in driver root path fix #9493 allow users to mount a guangyapan subfolder --- drivers/guangyapan/driver.go | 137 ++++++++++++++++++++++++++++++----- drivers/guangyapan/meta.go | 4 +- 2 files changed, 121 insertions(+), 20 deletions(-) diff --git a/drivers/guangyapan/driver.go b/drivers/guangyapan/driver.go index ff83c9a88fa..8c80ebf3565 100644 --- a/drivers/guangyapan/driver.go +++ b/drivers/guangyapan/driver.go @@ -33,6 +33,9 @@ type GuangYaPan struct { accountClient *resty.Client apiClient *resty.Client + + resolvedRootFolderID string + rootFolderResolved bool } func (d *GuangYaPan) Config() driver.Config { @@ -62,12 +65,15 @@ func (d *GuangYaPan) Init(ctx context.Context) error { d.SortType = 1 } + d.RootPath = strings.TrimSpace(d.RootPath) d.AccessToken = strings.TrimSpace(d.AccessToken) d.RefreshToken = strings.TrimSpace(d.RefreshToken) d.PhoneNumber = strings.TrimSpace(d.PhoneNumber) d.VerifyCode = strings.TrimSpace(d.VerifyCode) d.CaptchaToken = strings.TrimSpace(d.CaptchaToken) d.VerificationID = strings.TrimSpace(d.VerificationID) + d.resolvedRootFolderID = "" + d.rootFolderResolved = false d.accountClient = base.NewRestyClient(). SetBaseURL(accountBaseURL). @@ -99,14 +105,14 @@ func (d *GuangYaPan) Init(ctx context.Context) error { // Priority: access_token -> refresh_token -> sms login. if d.AccessToken != "" { if err := d.validateToken(ctx); err == nil { - return nil + return d.prepareRootFolder(ctx) } d.AccessToken = "" } if d.RefreshToken != "" { if err := d.refreshToken(ctx); err == nil { if err2 := d.validateToken(ctx); err2 == nil { - return nil + return d.prepareRootFolder(ctx) } } } @@ -118,7 +124,10 @@ func (d *GuangYaPan) Init(ctx context.Context) error { if err := d.loginBySMSCode(ctx); err != nil { return err } - return d.validateToken(ctx) + if err := d.validateToken(ctx); err != nil { + return err + } + return d.prepareRootFolder(ctx) } if d.SendCode { d.setTempStatus("SMS sending in progress...") @@ -138,15 +147,27 @@ func (d *GuangYaPan) Drop(ctx context.Context) error { return nil } +func (d *GuangYaPan) GetRoot(ctx context.Context) (model.Obj, error) { + rootID, err := d.getRootFolderID(ctx) + if err != nil { + return nil, err + } + return &model.Object{ + ID: rootID, + Path: "/", + Name: "root", + Size: 0, + Modified: d.Modified, + IsFolder: true, + }, nil +} + func (d *GuangYaPan) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if err := d.ensureAccessToken(ctx); err != nil { return nil, err } parentID := dir.GetID() - if parentID == d.RootFolderID { - parentID = "" - } res := make([]model.Obj, 0, d.PageSize) for page := 0; ; page++ { @@ -219,9 +240,6 @@ func (d *GuangYaPan) MakeDir(ctx context.Context, parentDir model.Obj, dirName s } parentID := parentDir.GetID() - if parentID == d.RootFolderID { - parentID = "" - } var out createDirResp if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/create_dir", map[string]any{ @@ -301,9 +319,6 @@ func (d *GuangYaPan) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return errors.New("file id is empty") } parentID := dstDir.GetID() - if parentID == d.RootFolderID { - parentID = "" - } var out deleteResp if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/move_file", map[string]any{ @@ -332,9 +347,6 @@ func (d *GuangYaPan) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errors.New("file id is empty") } parentID := dstDir.GetID() - if parentID == d.RootFolderID { - parentID = "" - } var out deleteResp if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/copy_file", map[string]any{ @@ -369,9 +381,6 @@ func (d *GuangYaPan) Put(ctx context.Context, dstDir model.Obj, file model.FileS } parentID := dstDir.GetID() - if parentID == d.RootFolderID { - parentID = "" - } token, code, err := d.getUploadToken(ctx, parentID, name, file.GetSize()) if err != nil { @@ -415,6 +424,97 @@ func (d *GuangYaPan) Put(ctx context.Context, dstDir model.Obj, file model.FileS return d.waitUploadTaskInfo(ctx, taskID) } +func (d *GuangYaPan) getRootFolderID(ctx context.Context) (string, error) { + if d.rootFolderResolved { + return d.resolvedRootFolderID, nil + } + if err := d.ensureAccessToken(ctx); err != nil { + return "", err + } + if err := d.prepareRootFolder(ctx); err != nil { + return "", err + } + return d.resolvedRootFolderID, nil +} + +func (d *GuangYaPan) prepareRootFolder(ctx context.Context) error { + rootID, err := d.resolveConfiguredRootFolderID(ctx) + if err != nil { + return err + } + d.resolvedRootFolderID = rootID + d.rootFolderResolved = true + return nil +} + +func (d *GuangYaPan) resolveConfiguredRootFolderID(ctx context.Context) (string, error) { + root := strings.TrimSpace(d.RootPath) + if root == "" { + return "", nil + } + return d.resolveFolderPath(ctx, root) +} + +func (d *GuangYaPan) resolveFolderPath(ctx context.Context, rootPath string) (string, error) { + cleanPath := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rootPath), "\\", "/"), "/") + if cleanPath == "" { + return "", nil + } + + parentID := "" + for _, name := range strings.Split(cleanPath, "/") { + if name == "" { + continue + } + childID, err := d.findChildFolderID(ctx, parentID, name) + if err != nil { + return "", err + } + parentID = childID + } + return parentID, nil +} + +func (d *GuangYaPan) findChildFolderID(ctx context.Context, parentID, name string) (string, error) { + pageSize := d.PageSize + if pageSize <= 0 { + pageSize = 100 + } + + seen := 0 + for page := 0; ; page++ { + var resp listResp + body := map[string]any{ + "parentId": parentID, + "page": page, + "pageSize": pageSize, + "orderBy": d.OrderBy, + "sortType": d.SortType, + "fileTypes": []int{}, + } + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_file_list", body, &resp); err != nil { + return "", err + } + for _, item := range resp.Data.List { + seen++ + if item.ResType == 2 && item.FileName == name { + return item.FileID, nil + } + } + if len(resp.Data.List) < pageSize { + break + } + if resp.Data.Total > 0 && seen >= resp.Data.Total { + break + } + } + + if parentID == "" { + return "", fmt.Errorf("resolve root folder path failed: folder %q not found under /", name) + } + return "", fmt.Errorf("resolve root folder path failed: folder %q not found under parent %s", name, parentID) +} + func (d *GuangYaPan) ensureAccessToken(ctx context.Context) error { if strings.TrimSpace(d.AccessToken) != "" { return nil @@ -948,3 +1048,4 @@ func randomDeviceID() string { } var _ driver.Driver = (*GuangYaPan)(nil) +var _ driver.GetRooter = (*GuangYaPan)(nil) \ No newline at end of file diff --git a/drivers/guangyapan/meta.go b/drivers/guangyapan/meta.go index 606d6aec8ee..434a151655e 100644 --- a/drivers/guangyapan/meta.go +++ b/drivers/guangyapan/meta.go @@ -6,7 +6,7 @@ import ( ) type Addition struct { - driver.RootID + RootPath string `json:"root_path" help:"光鸭云盘中的完整路径"` PhoneNumber string `json:"phone_number" type:"text" help:"Phone number for SMS login, e.g. +86 13800000000"` CaptchaToken string `json:"captcha_token" type:"text" help:"Captcha token required by /v1/auth/verification"` SendCode bool `json:"send_code" type:"bool" help:"Set true and save to send SMS code, it auto-resets to false after sending"` @@ -39,4 +39,4 @@ func init() { op.RegisterDriver(func() driver.Driver { return &GuangYaPan{} }) -} +} \ No newline at end of file From ffbbe7a96fbe175f04f44b00f84acf24aff29224 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 7 May 2026 14:45:31 +0800 Subject: [PATCH 119/133] fix(storage): clear list cache after storage updates --- internal/op/storage.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/op/storage.go b/internal/op/storage.go index dfb305aaa43..3961e32ed9a 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -229,10 +229,13 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { if err != nil { return errors.WithMessage(err, "failed update storage in database") } + storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) + if err == nil { + ClearCache(storageDriver, "/") + } if storage.Disabled { return nil } - storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) From cbeb088d40fac6949f3e74871969ac1eb94f7f90 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 7 May 2026 14:45:38 +0800 Subject: [PATCH 120/133] feat(settings): add frontend sort memory switch --- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 43 +++++++++++++++--------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 265a6502b60..0d39a377915 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -108,6 +108,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.DefaultRole, Value: defaultRoleID, Type: conf.TypeSelect, Group: model.SITE}, // newui settings {Key: conf.UseNewui, Value: "false", Type: conf.TypeBool, Group: model.SITE}, + {Key: conf.FrontendRememberSort, Value: "false", Type: conf.TypeBool, Group: model.SITE, Help: "Persist frontend list sorting in the browser. When disabled, backend/driver order is used until the user sorts manually in the current session."}, // style settings {Key: conf.Logo, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeText, Group: model.STYLE}, {Key: conf.Favicon, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.STYLE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 5bcb6eb5f91..aaa2fac8349 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -10,15 +10,16 @@ const ( const ( // site - VERSION = "version" - SiteTitle = "site_title" - Announcement = "announcement" - AllowIndexed = "allow_indexed" - AllowMounted = "allow_mounted" - RobotsTxt = "robots_txt" - AllowRegister = "allow_register" - DefaultRole = "default_role" - UseNewui = "use_newui" + VERSION = "version" + SiteTitle = "site_title" + Announcement = "announcement" + AllowIndexed = "allow_indexed" + AllowMounted = "allow_mounted" + RobotsTxt = "robots_txt" + AllowRegister = "allow_register" + DefaultRole = "default_role" + UseNewui = "use_newui" + FrontendRememberSort = "frontend_remember_sort" Logo = "logo" Favicon = "favicon" @@ -126,19 +127,19 @@ const ( FTPTLSPublicCertPath = "ftp_tls_public_cert_path" // frp - FRPEnabled = "frp_enabled" - FRPServerAddr = "frp_server_addr" - FRPServerPort = "frp_server_port" - FRPAuthToken = "frp_auth_token" - FRPProxyName = "frp_proxy_name" - FRPProxyType = "frp_proxy_type" - FRPCustomDomain = "frp_custom_domain" - FRPSubdomain = "frp_subdomain" - FRPRemotePort = "frp_remote_port" - FRPLocalPort = "frp_local_port" - FRPTLSEnable = "frp_tls_enable" + FRPEnabled = "frp_enabled" + FRPServerAddr = "frp_server_addr" + FRPServerPort = "frp_server_port" + FRPAuthToken = "frp_auth_token" + FRPProxyName = "frp_proxy_name" + FRPProxyType = "frp_proxy_type" + FRPCustomDomain = "frp_custom_domain" + FRPSubdomain = "frp_subdomain" + FRPRemotePort = "frp_remote_port" + FRPLocalPort = "frp_local_port" + FRPTLSEnable = "frp_tls_enable" FRPSTCPSecretKey = "frp_stcp_secret_key" - FRPStatus = "frp_status" + FRPStatus = "frp_status" // traffic TaskOfflineDownloadThreadsNum = "offline_download_task_threads_num" From 4a23e6a506b191d702ebb56ac5cdae53e0378a4e Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 7 May 2026 14:45:47 +0800 Subject: [PATCH 121/133] fix(guangyapan): expose sorting options --- drivers/guangyapan/driver.go | 2 +- drivers/guangyapan/meta.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/guangyapan/driver.go b/drivers/guangyapan/driver.go index ff83c9a88fa..2cbcdecddc6 100644 --- a/drivers/guangyapan/driver.go +++ b/drivers/guangyapan/driver.go @@ -159,7 +159,7 @@ func (d *GuangYaPan) List(ctx context.Context, dir model.Obj, args model.ListArg "sortType": d.SortType, "fileTypes": []int{}, } - if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_file_list", body, &resp); err != nil { + if err := d.postAPI(ctx, "/userres/v1/file/get_file_list", body, &resp); err != nil { return nil, err } for _, item := range resp.Data.List { diff --git a/drivers/guangyapan/meta.go b/drivers/guangyapan/meta.go index 606d6aec8ee..506d078b206 100644 --- a/drivers/guangyapan/meta.go +++ b/drivers/guangyapan/meta.go @@ -17,8 +17,8 @@ type Addition struct { ClientID string `json:"client_id" default:"aMe-8VSlkrbQXpUR"` DeviceID string `json:"device_id" help:"Optional custom device id (32 hex chars), auto-generated when empty"` PageSize int `json:"page_size" type:"number" default:"100"` - OrderBy int `json:"order_by" type:"number" default:"3" help:"0:name,1:size,2:create_time,3:update_time"` - SortType int `json:"sort_type" type:"number" default:"1" help:"0:asc,1:desc"` + OrderBy int `json:"order_by" type:"number" options:"0,1,2,3,4" default:"3" help:"Sort field used by the file list"` + SortType int `json:"sort_type" type:"number" options:"0,1" default:"1" help:"Sort direction used by the file list"` } var config = driver.Config{ From 756cc65f41085e9f3bb053f8728b3a968791332a Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 7 May 2026 14:54:52 +0800 Subject: [PATCH 122/133] ci: merge companion frontend pull request --- .github/workflows/merge_frontend_pr.yml | 70 +++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/merge_frontend_pr.yml diff --git a/.github/workflows/merge_frontend_pr.yml b/.github/workflows/merge_frontend_pr.yml new file mode 100644 index 00000000000..f4e3434306b --- /dev/null +++ b/.github/workflows/merge_frontend_pr.yml @@ -0,0 +1,70 @@ +name: merge companion frontend pr + +on: + pull_request: + branches: + - 'main' + types: + - closed + workflow_dispatch: + inputs: + frontend_pr: + description: 'Frontend PR reference, e.g. AlistGo/alist-web#301 or https://github.com/AlistGo/alist-web/pull/301' + required: true + type: string + +permissions: + contents: read + +jobs: + merge_frontend_pr: + name: Merge companion frontend PR + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Find frontend PR + id: frontend + env: + INPUT_FRONTEND_PR: ${{ inputs.frontend_pr }} + PR_BODY: ${{ github.event.pull_request.body }} + run: | + set -euo pipefail + + text="${INPUT_FRONTEND_PR:-${PR_BODY:-}}" + ref="$(printf '%s\n' "$text" | grep -Eo '(https://github\.com/(AlistGo|alist-org)/alist-web/pull/[0-9]+)|((AlistGo|alist-org)/alist-web#[0-9]+)|(alist-web#[0-9]+)' | head -n1 || true)" + + if [[ -z "$ref" ]]; then + echo "found=false" >> "$GITHUB_OUTPUT" + echo "No companion frontend PR referenced." + exit 0 + fi + + if [[ "$ref" == *"/pull/"* ]]; then + number="${ref##*/}" + else + number="${ref##*#}" + fi + + echo "found=true" >> "$GITHUB_OUTPUT" + echo "url=https://github.com/AlistGo/alist-web/pull/$number" >> "$GITHUB_OUTPUT" + echo "Found companion frontend PR #$number." + + - name: Merge frontend PR + if: steps.frontend.outputs.found == 'true' + env: + GH_TOKEN: ${{ secrets.MY_TOKEN }} + FRONTEND_PR: ${{ steps.frontend.outputs.url }} + run: | + set -euo pipefail + + state="$(gh pr view "$FRONTEND_PR" --json state --jq .state)" + if [[ "$state" == "MERGED" ]]; then + echo "$FRONTEND_PR is already merged." + exit 0 + fi + if [[ "$state" != "OPEN" ]]; then + echo "$FRONTEND_PR is $state and cannot be merged." + exit 1 + fi + + gh pr merge "$FRONTEND_PR" --auto --merge || gh pr merge "$FRONTEND_PR" --merge From 1dbd503918da4d30fa795094bd000741e857f24b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 16:44:16 +0800 Subject: [PATCH 123/133] chore(deps): bump github.com/jackc/pgx/v5 from 5.5.5 to 5.9.0 (#9482) Bumps [github.com/jackc/pgx/v5](https://github.com/jackc/pgx) from 5.5.5 to 5.9.0. - [Changelog](https://github.com/jackc/pgx/blob/master/CHANGELOG.md) - [Commits](https://github.com/jackc/pgx/compare/v5.5.5...v5.9.0) --- updated-dependencies: - dependency-name: github.com/jackc/pgx/v5 dependency-version: 5.9.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 19 +++++++++---------- go.sum | 24 ++++++++++++------------ 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index a7accd494f0..bce6deae91e 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 @@ -104,6 +104,7 @@ require ( github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect github.com/fatedier/golib v0.5.1 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/google/jsonschema-go v0.4.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/klauspost/reedsolomon v1.12.0 // indirect @@ -117,12 +118,14 @@ require ( github.com/relvacode/iso8601 v1.3.0 // indirect github.com/samber/lo v1.47.0 // indirect github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 // indirect + github.com/spf13/cast v1.7.1 // indirect github.com/templexxx/cpu v0.1.1 // indirect github.com/templexxx/xorsimd v0.4.3 // indirect github.com/tjfoc/gmsm v1.4.1 // indirect github.com/vishvananda/netlink v1.3.0 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/xtaci/kcp-go/v5 v5.6.13 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect golang.org/x/mod v0.27.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect @@ -132,10 +135,6 @@ require ( k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/yaml v1.3.0 // indirect - github.com/google/jsonschema-go v0.4.2 // indirect - github.com/relvacode/iso8601 v1.3.0 // indirect - github.com/spf13/cast v1.7.1 // indirect - github.com/yosida95/uritemplate/v3 v3.0.2 // indirect ) require ( @@ -160,7 +159,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 @@ -233,8 +232,8 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -304,10 +303,10 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.16.0 + golang.org/x/sync v0.17.0 golang.org/x/sys v0.35.0 // indirect golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.28.0 + golang.org/x/text v0.29.0 golang.org/x/tools v0.36.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect diff --git a/go.sum b/go.sum index 34cc3338327..f8c574fd154 100644 --- a/go.sum +++ b/go.sum @@ -418,12 +418,12 @@ github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2 github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= -github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.0 h1:T/dI+2TvmI2H8s/KH1/lXIbz1CUFk3gn5oTjr0/mBsE= +github.com/jackc/pgx/v5 v5.9.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -682,8 +682,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= @@ -895,8 +895,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -974,8 +974,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= From 6bc8946741acd76a7a4f373cc8d576fe6228d90f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 16:44:31 +0800 Subject: [PATCH 124/133] chore(deps): bump github.com/golang-jwt/jwt/v4 from 4.5.0 to 4.5.2 (#9484) Bumps [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) from 4.5.0 to 4.5.2. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Commits](https://github.com/golang-jwt/jwt/compare/v4.5.0...v4.5.2) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v4 dependency-version: 4.5.2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index bce6deae91e..562296369cf 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/gin-gonic/gin v1.10.0 github.com/go-resty/resty/v2 v2.14.0 github.com/go-webauthn/webauthn v0.11.1 - github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/hekmon/transmissionrpc/v3 v3.0.0 diff --git a/go.sum b/go.sum index f8c574fd154..e3bb088e1ef 100644 --- a/go.sum +++ b/go.sum @@ -307,8 +307,9 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= From fb48ea49fb73eac94b3f1e4555103577e65d1c54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 08:46:35 +0000 Subject: [PATCH 125/133] chore(deps): bump google.golang.org/grpc from 1.66.0 to 1.79.3 Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.66.0 to 1.79.3. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.66.0...v1.79.3) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.79.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 26 +++++++++---------- go.sum | 80 ++++++++++++++++++++++++++++++++-------------------------- 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index 562296369cf..43098d33c83 100644 --- a/go.mod +++ b/go.mod @@ -72,11 +72,11 @@ require ( github.com/xhofe/wopan-sdk-go v0.1.3 github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.41.0 + golang.org/x/crypto v0.46.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 - golang.org/x/net v0.43.0 - golang.org/x/oauth2 v0.30.0 + golang.org/x/net v0.48.0 + golang.org/x/oauth2 v0.34.0 golang.org/x/time v0.8.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 @@ -103,7 +103,7 @@ require ( github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect github.com/fatedier/golib v0.5.1 // indirect - github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/google/jsonschema-go v0.4.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect @@ -126,7 +126,7 @@ require ( github.com/vishvananda/netns v0.0.4 // indirect github.com/xtaci/kcp-go/v5 v5.6.13 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect - golang.org/x/mod v0.27.0 // indirect + golang.org/x/mod v0.30.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect gopkg.in/ini.v1 v1.67.0 // indirect @@ -303,15 +303,15 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.17.0 - golang.org/x/sys v0.35.0 // indirect - golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.29.0 - golang.org/x/tools v0.36.0 // indirect + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 + golang.org/x/tools v0.39.0 // indirect google.golang.org/api v0.169.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/grpc v1.66.0 - google.golang.org/protobuf v1.36.5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/grpc v1.79.3 + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index e3bb088e1ef..ac30c8cc445 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,8 @@ cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= -cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= -cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -273,15 +273,15 @@ github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= -github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -764,14 +764,20 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= @@ -801,8 +807,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -839,8 +845,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -875,15 +881,15 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -896,8 +902,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -940,8 +946,8 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -956,8 +962,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -975,8 +981,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -1014,8 +1020,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1024,6 +1030,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4= golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1054,8 +1062,8 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1065,8 +1073,8 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= -google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1075,8 +1083,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From d509a8787e460d975e728fa2f51a16a0631d386d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 16:47:26 +0800 Subject: [PATCH 126/133] feat(lark): add export tools API (#9511) --- drivers/lark/driver.go | 396 ++++++++++++++++++++++++++++----- drivers/lark/meta.go | 12 +- drivers/lark/other.go | 433 +++++++++++++++++++++++++++++++++++++ drivers/lark/other_test.go | 176 +++++++++++++++ drivers/lark/util.go | 48 ++-- go.mod | 2 +- go.sum | 4 +- server/common/proxy.go | 3 + server/handles/fsread.go | 29 ++- server/handles/lark.go | 84 +++++++ server/router.go | 1 + 11 files changed, 1107 insertions(+), 81 deletions(-) create mode 100644 drivers/lark/other.go create mode 100644 drivers/lark/other_test.go create mode 100644 server/handles/lark.go diff --git a/drivers/lark/driver.go b/drivers/lark/driver.go index fbf7529afe3..ca704a2dbf3 100644 --- a/drivers/lark/driver.go +++ b/drivers/lark/driver.go @@ -8,14 +8,18 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" lark "github.com/larksuite/oapi-sdk-go/v3" larkcore "github.com/larksuite/oapi-sdk-go/v3/core" larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" + larkext "github.com/larksuite/oapi-sdk-go/v3/service/ext" + log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -25,8 +29,12 @@ type Lark struct { client *lark.Client rootFolderToken string + tokenMu sync.Mutex } +const larkListPageSize = 200 +const larkTokenRefreshSkew = 5 * time.Minute + func (c *Lark) Config() driver.Config { return config } @@ -41,34 +49,28 @@ func (c *Lark) Init(ctx context.Context) error { paths := strings.Split(c.RootFolderPath, "/") token := "" - var ok bool - var file *larkdrive.File for _, p := range paths { if p == "" { token = "" continue } - resp, err := c.client.Drive.File.ListByIterator(ctx, larkdrive.NewListFileReqBuilder().FolderToken(token).Build()) + files, err := c.listFiles(ctx, token) if err != nil { return err } - for { - ok, file, err = resp.Next() - if !ok { - return errs.ObjectNotFound - } - - if err != nil { - return err - } - + found := false + for _, file := range files { if *file.Type == "folder" && *file.Name == p { token = *file.Token + found = true break } } + if !found { + return errs.ObjectNotFound + } } c.rootFolderToken = token @@ -90,41 +92,262 @@ func (c *Lark) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] return nil, nil } - resp, err := c.client.Drive.File.ListByIterator(ctx, larkdrive.NewListFileReqBuilder().FolderToken(token).Build()) + files, err := c.listFiles(ctx, token) if err != nil { return nil, err } - ok = false - var file *larkdrive.File var res []model.Obj + for _, file := range files { + res = append(res, larkFileToObj(c.RootFolderPath, dir.GetPath(), file)) + } + + return res, nil +} + +func larkFileToObj(rootFolderPath, dirPath string, file *larkdrive.File) model.Obj { + name := larkString(file.Name) + fileType := larkString(file.Type) + modifiedUnix, _ := strconv.ParseInt(larkString(file.ModifiedTime), 10, 64) + createdUnix, _ := strconv.ParseInt(larkString(file.CreatedTime), 10, 64) + obj := model.Object{ + ID: larkString(file.Token), + Path: strings.Join([]string{rootFolderPath, dirPath, name}, "/"), + Name: larkDisplayName(name, fileType), + Size: 0, + Modified: time.Unix(modifiedUnix, 0), + Ctime: time.Unix(createdUnix, 0), + IsFolder: fileType == "folder", + } + if file.Url == nil || *file.Url == "" || obj.IsFolder || !isLarkNativeDocType(fileType) { + return &obj + } + return &model.ObjectURL{ + Object: obj, + Url: model.Url{Url: *file.Url}, + } +} + +func larkDisplayName(name, fileType string) string { + if isLarkCloudDocName(name) { + return name + } + switch fileType { + case "doc": + return name + ".lark-doc" + case "docx": + return name + ".lark-docx" + case "sheet": + return name + ".lark-sheet" + case "bitable": + return name + ".lark-bitable" + case "mindnote": + return name + ".lark-mindnote" + case "slides": + return name + ".lark-slides" + default: + return name + } +} + +func isLarkNativeDocType(fileType string) bool { + switch fileType { + case "doc", "docx", "sheet", "bitable", "mindnote", "slides": + return true + default: + return false + } +} + +func larkString(s *string) string { + if s == nil { + return "" + } + return *s +} + +func (c *Lark) requestOpts(ctx context.Context) ([]larkcore.RequestOptionFunc, error) { + userAccessToken, err := c.ensureUserAccessToken(ctx, false) + if err != nil { + return nil, err + } + if userAccessToken == "" { + return nil, nil + } + return []larkcore.RequestOptionFunc{larkcore.WithUserAccessToken(userAccessToken)}, nil +} + +func (c *Lark) ensureUserAccessToken(ctx context.Context, forceRefresh bool) (string, error) { + if strings.TrimSpace(c.RefreshToken) == "" { + return strings.TrimSpace(c.UserAccessToken), nil + } + if token := strings.TrimSpace(c.UserAccessToken); !forceRefresh && token != "" && !c.userAccessTokenExpired() { + return token, nil + } + + c.tokenMu.Lock() + defer c.tokenMu.Unlock() + + if token := strings.TrimSpace(c.UserAccessToken); !forceRefresh && token != "" && !c.userAccessTokenExpired() { + return token, nil + } + if c.RefreshTokenExpiresAt > 0 && time.Now().After(time.Unix(c.RefreshTokenExpiresAt, 0)) { + return "", errors.New("lark refresh token expired") + } + + resp, err := c.client.Ext.Authen.RefreshAuthenAccessToken(ctx, + larkext.NewRefreshAuthenAccessTokenReqBuilder(). + Body(larkext.NewRefreshAuthenAccessTokenReqBodyBuilder(). + GrantType(larkext.GrantTypeRefreshCode). + RefreshToken(strings.TrimSpace(c.RefreshToken)). + Build()). + Build()) + if err != nil { + return "", err + } + if !resp.Success() { + return "", errors.New(resp.Error()) + } + if resp.Data == nil || resp.Data.AccessToken == "" { + return "", errors.New("lark refresh token response missing access token") + } + + now := time.Now() + c.UserAccessToken = resp.Data.AccessToken + c.UserAccessTokenExpiresAt = now.Add(time.Duration(resp.Data.ExpiresIn) * time.Second).Unix() + if resp.Data.RefreshToken != "" { + c.RefreshToken = resp.Data.RefreshToken + } + if resp.Data.RefreshExpiresIn > 0 { + c.RefreshTokenExpiresAt = now.Add(time.Duration(resp.Data.RefreshExpiresIn) * time.Second).Unix() + } + op.MustSaveDriverStorage(c) + + return c.UserAccessToken, nil +} + +func (c *Lark) forceRefreshUserAccessToken(ctx context.Context) error { + if strings.TrimSpace(c.RefreshToken) == "" { + return nil + } + _, err := c.ensureUserAccessToken(ctx, true) + return err +} + +func (c *Lark) userAccessTokenExpired() bool { + if c.UserAccessTokenExpiresAt <= 0 { + return true + } + return time.Now().Add(larkTokenRefreshSkew).After(time.Unix(c.UserAccessTokenExpiresAt, 0)) +} + +func (c *Lark) listFiles(ctx context.Context, folderToken string) ([]*larkdrive.File, error) { + var files []*larkdrive.File + pageToken := "" + for { - ok, file, err = resp.Next() - if !ok { - break + builder := larkdrive.NewListFileReqBuilder(). + FolderToken(folderToken). + OrderBy("EditedTime"). + Direction("DESC") + if folderToken != "" { + builder.PageSize(larkListPageSize) + if pageToken != "" { + builder.PageToken(pageToken) + } } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.ListFileResp, error) { + return c.client.Drive.V1.File.List(ctx, builder.Build(), opts...) + }) if err != nil { return nil, err } + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + if resp.Data == nil { + return files, nil + } - modifiedUnix, _ := strconv.ParseInt(*file.ModifiedTime, 10, 64) - createdUnix, _ := strconv.ParseInt(*file.CreatedTime, 10, 64) - - f := model.Object{ - ID: *file.Token, - Path: strings.Join([]string{c.RootFolderPath, dir.GetPath(), *file.Name}, "/"), - Name: *file.Name, - Size: 0, - Modified: time.Unix(modifiedUnix, 0), - Ctime: time.Unix(createdUnix, 0), - IsFolder: *file.Type == "folder", + files = append(files, resp.Data.Files...) + if folderToken == "" || resp.Data.HasMore == nil || !*resp.Data.HasMore || + resp.Data.NextPageToken == nil || *resp.Data.NextPageToken == "" { + break } - res = append(res, &f) + pageToken = *resp.Data.NextPageToken } - return res, nil + return files, nil +} + +type larkResp interface { + Success() bool + Error() string +} + +func doDrive[T larkResp](ctx context.Context, c *Lark, call func(...larkcore.RequestOptionFunc) (T, error)) (T, error) { + opts, err := c.requestOpts(ctx) + if err != nil { + var zero T + return zero, err + } + + resp, err := call(opts...) + if err != nil { + var zero T + return zero, err + } + if !isLarkAuthFailed(resp) || strings.TrimSpace(c.RefreshToken) == "" { + return resp, nil + } + + log.WithField("mount_path", c.MountPath).Warn("lark user access token auth failed, refreshing and retrying once") + if err = c.forceRefreshUserAccessToken(ctx); err != nil { + return resp, nil + } + opts, err = c.requestOpts(ctx) + if err != nil { + var zero T + return zero, err + } + return call(opts...) +} + +func isLarkAuthFailed(resp larkResp) bool { + if resp == nil || resp.Success() { + return false + } + switch v := any(resp).(type) { + case *larkdrive.ListFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.CreateFolderFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.MoveFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.CopyFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.DeleteFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadPrepareFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadPartFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadFinishFileResp: + return isLarkAuthFailedCode(v.Code) + default: + return strings.Contains(resp.Error(), "1061005") || + strings.Contains(strings.ToLower(resp.Error()), "auth") + } +} + +func isLarkAuthFailedCode(code int) bool { + return code == 1061005 || code == 99991663 || code == 99991664 || code == 99991668 +} + +func isHTTPAuthFailed(statusCode int) bool { + return statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden } func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -133,17 +356,23 @@ func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* return nil, errs.ObjectNotFound } - resp, err := c.client.GetTenantAccessTokenBySelfBuiltApp(ctx, &larkcore.SelfBuiltTenantAccessTokenReq{ - AppID: c.AppId, - AppSecret: c.AppSecret, - }) + if isLarkCloudDocName(file.GetName()) { + return &model.Link{ + URL: c.filePreviewURL(token), + }, nil + } - if err != nil { - return nil, err + if !c.WebProxy || c.ExternalMode { + return &model.Link{ + URL: c.filePreviewURL(token), + }, nil } - if !c.ExternalMode { - accessToken := resp.TenantAccessToken + if c.WebProxy { + accessToken, err := c.downloadAccessToken(ctx, false) + if err != nil { + return nil, err + } url := fmt.Sprintf("https://open.feishu.cn/open-apis/drive/v1/files/%s/download", token) @@ -159,6 +388,20 @@ func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* if err != nil { return nil, err } + _ = ar.Body.Close() + + if isHTTPAuthFailed(ar.StatusCode) && strings.TrimSpace(c.RefreshToken) != "" { + accessToken, err = c.downloadAccessToken(ctx, true) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", accessToken)) + ar, err = http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + _ = ar.Body.Close() + } if ar.StatusCode != http.StatusPartialContent { return nil, errors.New("failed to get download link") @@ -170,13 +413,46 @@ func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* "Authorization": []string{fmt.Sprintf("Bearer %s", accessToken)}, }, }, nil - } else { - url := strings.Join([]string{c.TenantUrlPrefix, "file", token}, "/") + } - return &model.Link{ - URL: url, - }, nil + return nil, errors.New("lark download requires web proxy") +} + +func (c *Lark) filePreviewURL(token string) string { + prefix := strings.TrimRight(strings.TrimSpace(c.TenantUrlPrefix), "/") + if prefix == "" { + prefix = "https://www.feishu.cn" + } + return prefix + "/file/" + token +} + +func (c *Lark) downloadAccessToken(ctx context.Context, forceRefresh bool) (string, error) { + var accessToken string + var err error + if strings.TrimSpace(c.RefreshToken) != "" || strings.TrimSpace(c.UserAccessToken) != "" { + accessToken, err = c.ensureUserAccessToken(ctx, forceRefresh) + if err != nil { + return "", err + } } + if accessToken != "" { + return accessToken, nil + } + + resp, err := c.client.GetTenantAccessTokenBySelfBuiltApp(ctx, &larkcore.SelfBuiltTenantAccessTokenReq{ + AppID: c.AppId, + AppSecret: c.AppSecret, + }) + if err != nil { + return "", err + } + if !resp.Success() { + return "", errors.New(resp.Error()) + } + if resp.TenantAccessToken == "" { + return "", errors.New("lark tenant access token is empty") + } + return resp.TenantAccessToken, nil } func (c *Lark) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { @@ -190,8 +466,10 @@ func (c *Lark) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) return nil, err } - resp, err := c.client.Drive.File.CreateFolder(ctx, - larkdrive.NewCreateFolderFileReqBuilder().Body(body).Build()) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CreateFolderFileResp, error) { + return c.client.Drive.File.CreateFolder(ctx, + larkdrive.NewCreateFolderFileReqBuilder().Body(body).Build(), opts...) + }) if err != nil { return nil, err } @@ -228,7 +506,9 @@ func (c *Lark) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, e Build() // 发起请求 - resp, err := c.client.Drive.File.Move(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.MoveFileResp, error) { + return c.client.Drive.File.Move(ctx, req, opts...) + }) if err != nil { return nil, err } @@ -265,7 +545,9 @@ func (c *Lark) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, e Build() // 发起请求 - resp, err := c.client.Drive.File.Copy(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CopyFileResp, error) { + return c.client.Drive.File.Copy(ctx, req, opts...) + }) if err != nil { return nil, err } @@ -289,7 +571,9 @@ func (c *Lark) Remove(ctx context.Context, obj model.Obj) error { Build() // 发起请求 - resp, err := c.client.Drive.File.Delete(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.DeleteFileResp, error) { + return c.client.Drive.File.Delete(ctx, req, opts...) + }) if err != nil { return err } @@ -324,7 +608,9 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return nil, err } - resp, err := c.client.Drive.File.UploadPrepare(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadPrepareFileResp, error) { + return c.client.Drive.File.UploadPrepare(ctx, req, opts...) + }) if err != nil { return nil, err } @@ -360,7 +646,9 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return nil, err } - resp, err := c.client.Drive.File.UploadPart(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadPartFileResp, error) { + return c.client.Drive.File.UploadPart(ctx, req, opts...) + }) if err != nil { return nil, err @@ -382,7 +670,9 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea Build() // 发起请求 - closeResp, err := c.client.Drive.File.UploadFinish(ctx, closeReq) + closeResp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadFinishFileResp, error) { + return c.client.Drive.File.UploadFinish(ctx, closeReq, opts...) + }) if err != nil { return nil, err } diff --git a/drivers/lark/meta.go b/drivers/lark/meta.go index 221345e222c..fe6149d99a4 100644 --- a/drivers/lark/meta.go +++ b/drivers/lark/meta.go @@ -9,10 +9,14 @@ type Addition struct { // Usually one of two driver.RootPath // define other - AppId string `json:"app_id" type:"text" help:"app id"` - AppSecret string `json:"app_secret" type:"text" help:"app secret"` - ExternalMode bool `json:"external_mode" type:"bool" help:"external mode"` - TenantUrlPrefix string `json:"tenant_url_prefix" type:"text" help:"tenant url prefix"` + AppId string `json:"app_id" type:"text" help:"app id"` + AppSecret string `json:"app_secret" type:"text" help:"app secret"` + UserAccessToken string `json:"user_access_token" type:"text" help:"optional cached user access token for personal drive access"` + RefreshToken string `json:"refresh_token" type:"text" help:"optional refresh token for user access token auto refresh"` + UserAccessTokenExpiresAt int64 `json:"user_access_token_expires_at" type:"number" help:"user access token expires at unix timestamp"` + RefreshTokenExpiresAt int64 `json:"refresh_token_expires_at" type:"number" help:"refresh token expires at unix timestamp"` + ExternalMode bool `json:"external_mode" type:"bool" help:"external mode"` + TenantUrlPrefix string `json:"tenant_url_prefix" type:"text" help:"tenant url prefix"` } var config = driver.Config{ diff --git a/drivers/lark/other.go b/drivers/lark/other.go new file mode 100644 index 00000000000..f0217b72621 --- /dev/null +++ b/drivers/lark/other.go @@ -0,0 +1,433 @@ +package lark + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + larkcore "github.com/larksuite/oapi-sdk-go/v3/core" + larkbitable "github.com/larksuite/oapi-sdk-go/v3/service/bitable/v1" + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" + larksheets "github.com/larksuite/oapi-sdk-go/v3/service/sheets/v3" + "github.com/pkg/errors" +) + +const ( + larkExportOptionsMethod = "lark_export_options" + larkExportCreateMethod = "lark_export_create" + larkExportStatusMethod = "lark_export_status" + + larkExportStatusPending = "pending" + larkExportStatusProcessing = "processing" + larkExportStatusSuccess = "success" + larkExportStatusFailed = "failed" +) + +type larkExportCreateReq struct { + Format string `json:"format"` + SubID string `json:"sub_id"` +} + +type larkExportStatusReq struct { + Ticket string `json:"ticket"` +} + +type LarkExportCreateResp struct { + Ticket string `json:"ticket"` + Token string `json:"token"` + Type string `json:"type"` + Format string `json:"format"` + SubID string `json:"sub_id,omitempty"` +} + +type LarkExportOption struct { + Value string `json:"value"` + Label string `json:"label"` + RequiresSubID bool `json:"requires_sub_id"` +} + +type LarkExportSubResource struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` +} + +type LarkExportOptionsResp struct { + Type string `json:"type"` + Formats []LarkExportOption `json:"formats"` + SubResources []LarkExportSubResource `json:"sub_resources,omitempty"` + SubResourceError string `json:"sub_resource_error,omitempty"` +} + +type LarkExportStatusResp struct { + Status string `json:"status"` + FileToken string `json:"file_token,omitempty"` + FileSize int `json:"file_size,omitempty"` + JobStatus int `json:"job_status,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ErrorDetail string `json:"error_detail,omitempty"` +} + +type larkAPIErrorResp interface { + Error() string + ErrorResp() string +} + +func (c *Lark) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch strings.ToLower(strings.TrimSpace(args.Method)) { + case larkExportOptionsMethod: + return c.getExportOptions(ctx, args.Obj) + case larkExportCreateMethod: + var req larkExportCreateReq + if err := decodeOtherData(args.Data, &req); err != nil { + return nil, err + } + return c.createExportTask(ctx, args.Obj, req) + case larkExportStatusMethod: + var req larkExportStatusReq + if err := decodeOtherData(args.Data, &req); err != nil { + return nil, err + } + return c.getExportTask(ctx, args.Obj, req) + default: + return nil, fmt.Errorf("unsupported lark method: %s", args.Method) + } +} + +func decodeOtherData(data interface{}, v interface{}) error { + b, err := json.Marshal(data) + if err != nil { + return errors.WithMessage(err, "failed to encode request data") + } + if err = json.Unmarshal(b, v); err != nil { + return errors.WithMessage(err, "failed to decode request data") + } + return nil +} + +func (c *Lark) createExportTask(ctx context.Context, obj model.Obj, req larkExportCreateReq) (*LarkExportCreateResp, error) { + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + docType, err := larkExportType(obj.GetName()) + if err != nil { + return nil, err + } + format := strings.ToLower(strings.TrimSpace(req.Format)) + if !larkExportFormatAllowed(docType, format) { + return nil, fmt.Errorf("unsupported export format %q for lark type %q", format, docType) + } + subID := strings.TrimSpace(req.SubID) + if larkExportFormatRequiresSubID(docType, format) && subID == "" { + return nil, fmt.Errorf("sub_id is required when exporting lark type %q as %q", docType, format) + } + + builder := larkdrive.NewExportTaskBuilder(). + Token(token). + Type(docType). + FileExtension(format). + FileName(larkExportBaseName(obj.GetName())) + if subID != "" { + builder.SubId(subID) + } + exportTask := builder.Build() + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CreateExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Create(ctx, + larkdrive.NewCreateExportTaskReqBuilder().ExportTask(exportTask).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil || resp.Data.Ticket == nil || *resp.Data.Ticket == "" { + return nil, errors.New("lark export task response missing ticket") + } + return &LarkExportCreateResp{ + Ticket: *resp.Data.Ticket, + Token: token, + Type: docType, + Format: format, + SubID: subID, + }, nil +} + +func (c *Lark) getExportOptions(ctx context.Context, obj model.Obj) (*LarkExportOptionsResp, error) { + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + docType, err := larkExportType(obj.GetName()) + if err != nil { + return nil, err + } + out := &LarkExportOptionsResp{ + Type: docType, + Formats: larkExportOptions(docType), + } + switch docType { + case larkdrive.TypeSheet: + out.SubResources, err = c.listSheetSubResources(ctx, token) + case larkdrive.TypeBitable: + out.SubResources, err = c.listBitableSubResources(ctx, token) + } + if err != nil { + out.SubResourceError = err.Error() + } + return out, nil +} + +func (c *Lark) getExportTask(ctx context.Context, obj model.Obj, req larkExportStatusReq) (*LarkExportStatusResp, error) { + ticket := strings.TrimSpace(req.Ticket) + if ticket == "" { + return nil, errors.New("ticket is required") + } + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.GetExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Get(ctx, + larkdrive.NewGetExportTaskReqBuilder().Ticket(ticket).Token(token).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil || resp.Data.Result == nil { + return nil, errors.New("lark export task response missing result") + } + result := resp.Data.Result + out := &LarkExportStatusResp{ + Status: larkExportJobStatus(result), + } + if result.FileToken != nil { + out.FileToken = *result.FileToken + } + if result.FileSize != nil { + out.FileSize = *result.FileSize + } + if result.JobStatus != nil { + out.JobStatus = *result.JobStatus + } + if out.Status == larkExportStatusFailed { + out.ErrorMessage = larkExportTaskErrorMessage(result) + out.ErrorDetail = larkExportTaskErrorDetail(result) + } else if result.JobErrorMsg != nil { + out.ErrorMessage = strings.TrimSpace(*result.JobErrorMsg) + } + return out, nil +} + +func (c *Lark) DownloadExportFile(ctx context.Context, fileToken string) (io.Reader, string, error) { + fileToken = strings.TrimSpace(fileToken) + if fileToken == "" { + return nil, "", errors.New("file_token is required") + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.DownloadExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Download(ctx, + larkdrive.NewDownloadExportTaskReqBuilder().FileToken(fileToken).Build(), opts...) + }) + if err != nil { + return nil, "", err + } + if !resp.Success() { + return nil, "", larkAPIError(resp) + } + if resp.File == nil { + return nil, "", errors.New("lark export download response missing file") + } + return resp.File, resp.FileName, nil +} + +func larkExportType(name string) (string, error) { + switch { + case strings.HasSuffix(name, ".lark-doc"): + return larkdrive.TypeDoc, nil + case strings.HasSuffix(name, ".lark-docx"): + return larkdrive.TypeDocx, nil + case strings.HasSuffix(name, ".lark-sheet"): + return larkdrive.TypeSheet, nil + case strings.HasSuffix(name, ".lark-bitable"): + return larkdrive.TypeBitable, nil + default: + return "", fmt.Errorf("unsupported lark export file type: %s", name) + } +} + +func larkExportFormatAllowed(docType, format string) bool { + switch docType { + case larkdrive.TypeDoc, larkdrive.TypeDocx: + return format == larkdrive.FileExtensionPdf || format == larkdrive.FileExtensionDocx + case larkdrive.TypeSheet, larkdrive.TypeBitable: + return format == larkdrive.FileExtensionXlsx || format == larkdrive.FileExtensionCsv + default: + return false + } +} + +func larkExportFormatRequiresSubID(docType, format string) bool { + return (docType == larkdrive.TypeSheet || docType == larkdrive.TypeBitable) && format == larkdrive.FileExtensionCsv +} + +func larkExportOptions(docType string) []LarkExportOption { + switch docType { + case larkdrive.TypeDoc, larkdrive.TypeDocx: + return []LarkExportOption{ + {Value: larkdrive.FileExtensionPdf, Label: "PDF"}, + {Value: larkdrive.FileExtensionDocx, Label: "DOCX"}, + } + case larkdrive.TypeSheet, larkdrive.TypeBitable: + return []LarkExportOption{ + {Value: larkdrive.FileExtensionXlsx, Label: "XLSX"}, + {Value: larkdrive.FileExtensionCsv, Label: "CSV", RequiresSubID: true}, + } + default: + return nil + } +} + +func larkExportBaseName(name string) string { + return trimLarkDisplayExt(name) +} + +func larkExportJobStatus(result *larkdrive.ExportTask) string { + if result == nil { + return larkExportStatusProcessing + } + if result.FileToken != nil && *result.FileToken != "" { + return larkExportStatusSuccess + } + if result.JobStatus != nil && *result.JobStatus == 2 { + return larkExportStatusFailed + } + if result.JobErrorMsg != nil && *result.JobErrorMsg != "" && !strings.EqualFold(*result.JobErrorMsg, "success") { + return larkExportStatusFailed + } + return larkExportStatusProcessing +} + +func larkAPIError(resp larkAPIErrorResp) error { + msg := strings.TrimSpace(resp.ErrorResp()) + if msg == "" || msg == "{}" || msg == "null" { + msg = strings.TrimSpace(resp.Error()) + } + return errors.New(msg) +} + +func larkExportTaskErrorMessage(result *larkdrive.ExportTask) string { + if result == nil { + return "" + } + if result.JobErrorMsg != nil { + msg := strings.TrimSpace(*result.JobErrorMsg) + if msg != "" && !strings.EqualFold(msg, "success") { + return msg + } + } + if result.JobStatus != nil { + return fmt.Sprintf("job_status=%d", *result.JobStatus) + } + return "" +} + +func larkExportTaskErrorDetail(result *larkdrive.ExportTask) string { + if result == nil { + return "" + } + detail := map[string]interface{}{} + if result.JobStatus != nil { + detail["job_status"] = *result.JobStatus + } + if result.JobErrorMsg != nil { + detail["job_error_msg"] = strings.TrimSpace(*result.JobErrorMsg) + } + if len(detail) == 0 { + return "" + } + b, err := json.Marshal(detail) + if err != nil { + return "" + } + return string(b) +} + +func (c *Lark) listSheetSubResources(ctx context.Context, token string) ([]LarkExportSubResource, error) { + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larksheets.QuerySpreadsheetSheetResp, error) { + return c.client.Sheets.SpreadsheetSheet.Query(ctx, + larksheets.NewQuerySpreadsheetSheetReqBuilder().SpreadsheetToken(token).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil { + return nil, nil + } + var res []LarkExportSubResource + for _, sheet := range resp.Data.Sheets { + if sheet == nil || sheet.SheetId == nil || strings.TrimSpace(*sheet.SheetId) == "" { + continue + } + name := strings.TrimSpace(larkString(sheet.Title)) + if name == "" { + name = *sheet.SheetId + } + res = append(res, LarkExportSubResource{ + ID: *sheet.SheetId, + Name: name, + Type: strings.TrimSpace(larkString(sheet.ResourceType)), + }) + } + return res, nil +} + +func (c *Lark) listBitableSubResources(ctx context.Context, token string) ([]LarkExportSubResource, error) { + var res []LarkExportSubResource + pageToken := "" + for { + builder := larkbitable.NewListAppTableReqBuilder().AppToken(token).PageSize(100) + if pageToken != "" { + builder.PageToken(pageToken) + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkbitable.ListAppTableResp, error) { + return c.client.Bitable.AppTable.List(ctx, builder.Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil { + return res, nil + } + for _, table := range resp.Data.Items { + if table == nil || table.TableId == nil || strings.TrimSpace(*table.TableId) == "" { + continue + } + name := strings.TrimSpace(larkString(table.Name)) + if name == "" { + name = *table.TableId + } + res = append(res, LarkExportSubResource{ + ID: *table.TableId, + Name: name, + Type: "table", + }) + } + if resp.Data.HasMore == nil || !*resp.Data.HasMore || resp.Data.PageToken == nil || *resp.Data.PageToken == "" { + return res, nil + } + pageToken = *resp.Data.PageToken + } +} diff --git a/drivers/lark/other_test.go b/drivers/lark/other_test.go new file mode 100644 index 00000000000..6887c6bcd65 --- /dev/null +++ b/drivers/lark/other_test.go @@ -0,0 +1,176 @@ +package lark + +import ( + "testing" + + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" +) + +func TestLarkExportType(t *testing.T) { + tests := []struct { + name string + want string + wantErr bool + }{ + {name: "doc.lark-doc", want: larkdrive.TypeDoc}, + {name: "doc.lark-docx", want: larkdrive.TypeDocx}, + {name: "sheet.lark-sheet", want: larkdrive.TypeSheet}, + {name: "base.lark-bitable", want: larkdrive.TypeBitable}, + {name: "file.pdf", wantErr: true}, + } + for _, tt := range tests { + got, err := larkExportType(tt.name) + if tt.wantErr { + if err == nil { + t.Fatalf("larkExportType(%q) expected error", tt.name) + } + continue + } + if err != nil { + t.Fatalf("larkExportType(%q) unexpected error: %v", tt.name, err) + } + if got != tt.want { + t.Fatalf("larkExportType(%q) = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportFormatAllowed(t *testing.T) { + tests := []struct { + docType string + format string + want bool + }{ + {docType: larkdrive.TypeDoc, format: larkdrive.FileExtensionPdf, want: true}, + {docType: larkdrive.TypeDocx, format: larkdrive.FileExtensionDocx, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionXlsx, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionXlsx, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionPdf, want: false}, + } + for _, tt := range tests { + if got := larkExportFormatAllowed(tt.docType, tt.format); got != tt.want { + t.Fatalf("larkExportFormatAllowed(%q, %q) = %v, want %v", tt.docType, tt.format, got, tt.want) + } + } +} + +func TestLarkExportFormatRequiresSubID(t *testing.T) { + tests := []struct { + docType string + format string + want bool + }{ + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionXlsx, want: false}, + {docType: larkdrive.TypeDocx, format: larkdrive.FileExtensionDocx, want: false}, + } + for _, tt := range tests { + if got := larkExportFormatRequiresSubID(tt.docType, tt.format); got != tt.want { + t.Fatalf("larkExportFormatRequiresSubID(%q, %q) = %v, want %v", tt.docType, tt.format, got, tt.want) + } + } +} + +func TestLarkExportOptions(t *testing.T) { + tests := []struct { + docType string + want []LarkExportOption + }{ + {docType: larkdrive.TypeDocx, want: []LarkExportOption{ + {Value: larkdrive.FileExtensionPdf, Label: "PDF"}, + {Value: larkdrive.FileExtensionDocx, Label: "DOCX"}, + }}, + {docType: larkdrive.TypeSheet, want: []LarkExportOption{ + {Value: larkdrive.FileExtensionXlsx, Label: "XLSX"}, + {Value: larkdrive.FileExtensionCsv, Label: "CSV", RequiresSubID: true}, + }}, + } + for _, tt := range tests { + got := larkExportOptions(tt.docType) + if len(got) != len(tt.want) { + t.Fatalf("larkExportOptions(%q) len = %d, want %d", tt.docType, len(got), len(tt.want)) + } + for i := range got { + if got[i] != tt.want[i] { + t.Fatalf("larkExportOptions(%q)[%d] = %+v, want %+v", tt.docType, i, got[i], tt.want[i]) + } + } + } +} + +func TestLarkExportBaseName(t *testing.T) { + tests := map[string]string{ + "weekly.lark-docx": "weekly", + "weekly.report.lark-doc": "weekly.report", + "table.lark-sheet": "table", + "plain.pdf": "plain.pdf", + } + for name, want := range tests { + if got := larkExportBaseName(name); got != want { + t.Fatalf("larkExportBaseName(%q) = %q, want %q", name, got, want) + } + } +} + +func TestLarkExportJobStatus(t *testing.T) { + successToken := "box_token" + successCode := 0 + failCode := 2 + failMsg := "failed" + successMsg := "success" + + tests := []struct { + name string + result *larkdrive.ExportTask + want string + }{ + {name: "nil", result: nil, want: larkExportStatusProcessing}, + {name: "file token wins", result: &larkdrive.ExportTask{FileToken: &successToken, JobStatus: &failCode}, want: larkExportStatusSuccess}, + {name: "status failure", result: &larkdrive.ExportTask{JobStatus: &failCode}, want: larkExportStatusFailed}, + {name: "error message failure", result: &larkdrive.ExportTask{JobStatus: &successCode, JobErrorMsg: &failMsg}, want: larkExportStatusFailed}, + {name: "success message without token still processing", result: &larkdrive.ExportTask{JobStatus: &successCode, JobErrorMsg: &successMsg}, want: larkExportStatusProcessing}, + } + for _, tt := range tests { + if got := larkExportJobStatus(tt.result); got != tt.want { + t.Fatalf("%s: larkExportJobStatus() = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportTaskErrorMessage(t *testing.T) { + failCode := 2 + failMsg := "source document cannot be exported" + successMsg := "success" + + tests := []struct { + name string + result *larkdrive.ExportTask + want string + }{ + {name: "nil", result: nil, want: ""}, + {name: "job error message", result: &larkdrive.ExportTask{JobStatus: &failCode, JobErrorMsg: &failMsg}, want: failMsg}, + {name: "status fallback", result: &larkdrive.ExportTask{JobStatus: &failCode, JobErrorMsg: &successMsg}, want: "job_status=2"}, + } + for _, tt := range tests { + if got := larkExportTaskErrorMessage(tt.result); got != tt.want { + t.Fatalf("%s: larkExportTaskErrorMessage() = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportTaskErrorDetail(t *testing.T) { + failCode := 2 + failMsg := "source document cannot be exported" + + got := larkExportTaskErrorDetail(&larkdrive.ExportTask{ + JobStatus: &failCode, + JobErrorMsg: &failMsg, + }) + want := `{"job_error_msg":"source document cannot be exported","job_status":2}` + if got != want { + t.Fatalf("larkExportTaskErrorDetail() = %q, want %q", got, want) + } +} diff --git a/drivers/lark/util.go b/drivers/lark/util.go index 8c6828bd176..ebef4300c49 100644 --- a/drivers/lark/util.go +++ b/drivers/lark/util.go @@ -3,9 +3,9 @@ package lark import ( "context" "github.com/Xhofe/go-cache" - larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" log "github.com/sirupsen/logrus" "path" + "strings" "time" ) @@ -36,31 +36,47 @@ func (c *Lark) getObjToken(ctx context.Context, folderPath string) (string, bool return emptyFolderToken, false } - req := larkdrive.NewListFileReqBuilder().FolderToken(parentToken).Build() - resp, err := c.client.Drive.File.ListByIterator(ctx, req) - + files, err := c.listFiles(ctx, parentToken) if err != nil { log.WithError(err).Error("failed to list files") return emptyFolderToken, false } - var file *larkdrive.File - for { - found, file, err = resp.Next() - if !found { - break + for _, file := range files { + if *file.Name == name || *file.Name == trimLarkDisplayExt(name) { + objTokenCache.Set(folderPath, *file.Token, exOpts) + return *file.Token, true } + } + + return emptyFolderToken, false +} - if err != nil { - log.WithError(err).Error("failed to get next file") - break +func trimLarkDisplayExt(name string) string { + for _, suffix := range larkCloudDocSuffixes() { + if strings.HasSuffix(name, suffix) { + return strings.TrimSuffix(name, suffix) } + } + return name +} - if *file.Name == name { - objTokenCache.Set(folderPath, *file.Token, exOpts) - return *file.Token, true +func isLarkCloudDocName(name string) bool { + for _, suffix := range larkCloudDocSuffixes() { + if strings.HasSuffix(name, suffix) { + return true } } + return false +} - return emptyFolderToken, false +func larkCloudDocSuffixes() []string { + return []string{ + ".lark-doc", + ".lark-docx", + ".lark-sheet", + ".lark-bitable", + ".lark-mindnote", + ".lark-slides", + } } diff --git a/go.mod b/go.mod index 562296369cf..a550404e31d 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 github.com/kdomanski/iso9660 v0.4.0 - github.com/larksuite/oapi-sdk-go/v3 v3.3.1 + github.com/larksuite/oapi-sdk-go/v3 v3.6.1 github.com/mark3labs/mcp-go v0.48.0 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.27.2 diff --git a/go.sum b/go.sum index e3bb088e1ef..2dd61e11a2e 100644 --- a/go.sum +++ b/go.sum @@ -474,8 +474,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/larksuite/oapi-sdk-go/v3 v3.3.1 h1:DLQQEgHUAGZB6RVlceB1f6A94O206exxW2RIMH+gMUc= -github.com/larksuite/oapi-sdk-go/v3 v3.3.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/larksuite/oapi-sdk-go/v3 v3.6.1 h1:vAdu+sX9yXNkKnKnYQeIv6yBkjP37Q1JEJHmMa2eCjQ= +github.com/larksuite/oapi-sdk-go/v3 v3.6.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= diff --git a/server/common/proxy.go b/server/common/proxy.go index 97bf84efa12..dae97c9bf1c 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -80,6 +80,9 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. defer res.Body.Close() maps.Copy(w.Header(), res.Header) + if r.URL.Query().Get("type") == "preview" { + w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.PathEscape(file.GetName()))) + } w.WriteHeader(res.StatusCode) if r.Method == http.MethodHead { return nil diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 67f84e5d70a..9ca84edda52 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -333,6 +333,7 @@ type FsGetResp struct { Readme string `json:"readme"` Header string `json:"header"` Provider string `json:"provider"` + WebProxy bool `json:"web_proxy"` Related []ObjLabelResp `json:"related"` } @@ -367,14 +368,14 @@ func FsGet(c *gin.Context) { } var rawURL string - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + storage, storageErr := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) provider := "unknown" - if err == nil { + if storageErr == nil { provider = storage.Config().Name } if !obj.IsDir() { - if err != nil { - common.ErrorResp(c, err, 500) + if storageErr != nil { + common.ErrorResp(c, storageErr, 500) return } query := "" @@ -382,6 +383,7 @@ func FsGet(c *gin.Context) { query = "?sign=" + sign.Sign(reqPath) } forceRedirectRawURL := storage.GetStorage().Driver == "BaiduYouth" + forcePreviewRawURL := storage.GetStorage().Driver == "Lark" && isLarkCloudDocName(obj.GetName()) forceProxyRawURL := storage.GetStorage().Driver == "Quark" && utils.GetFileType(obj.GetName()) == conf.VIDEO if forceRedirectRawURL { // Baidu Youth direct links are minted per request and are not stable enough @@ -391,7 +393,7 @@ func FsGet(c *gin.Context) { common.GetApiUrl(c.Request), utils.EncodePath(reqPath, true), query) - } else if storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL { + } else if !forcePreviewRawURL && (storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL) { if storage.GetStorage().DownProxyUrl != "" { rawURL = common.BuildDownProxyURL( storage.GetStorage().DownProxyUrl, @@ -453,6 +455,7 @@ func FsGet(c *gin.Context) { Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), Provider: provider, + WebProxy: storageErr == nil && storage.GetStorage().WebProxy, Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)), }) } @@ -471,6 +474,22 @@ func filterRelated(objs []model.Obj, obj model.Obj) []model.Obj { return related } +func isLarkCloudDocName(name string) bool { + for _, suffix := range []string{ + ".lark-doc", + ".lark-docx", + ".lark-sheet", + ".lark-bitable", + ".lark-mindnote", + ".lark-slides", + } { + if strings.HasSuffix(name, suffix) { + return true + } + } + return false +} + type FsOtherReq struct { model.FsOtherArgs Password string `json:"password" form:"password"` diff --git a/server/handles/lark.go b/server/handles/lark.go new file mode 100644 index 00000000000..b8f6f7f9d60 --- /dev/null +++ b/server/handles/lark.go @@ -0,0 +1,84 @@ +package handles + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type larkExportDownloader interface { + DownloadExportFile(ctx context.Context, fileToken string) (io.Reader, string, error) +} + +func LarkExportDownload(c *gin.Context) { + rawPath := c.Query("path") + fileToken := c.Query("file_token") + password := c.Query("password") + filename := strings.TrimSpace(c.Query("filename")) + if rawPath == "" || fileToken == "" { + common.ErrorStrResp(c, "path and file_token are required", 400) + return + } + + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(rawPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500) + return + } + } + if !common.CanAccessWithRoles(user, meta, reqPath, password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + downloader, ok := storage.(larkExportDownloader) + if !ok || storage.GetStorage().Driver != "Lark" { + common.ErrorStrResp(c, "lark export download is not supported for this storage", 400) + return + } + + reader, respFilename, err := downloader.DownloadExportFile(c, fileToken) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if filename == "" { + filename = respFilename + } + if filename == "" { + filename = stdpath.Base(reqPath) + } + + c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, filename, url.PathEscape(filename))) + c.Header("Content-Type", utils.GetMimeType(filename)) + c.Status(http.StatusOK) + if _, err = io.Copy(c.Writer, reader); err != nil { + common.ErrorResp(c, err, 500) + return + } +} diff --git a/server/router.go b/server/router.go index 84c8a831e43..e930177910a 100644 --- a/server/router.go +++ b/server/router.go @@ -222,6 +222,7 @@ func _fs(g *gin.RouterGroup) { g.Any("/search", middlewares.SearchIndex, handles.Search) g.Any("/get", handles.FsGet) g.Any("/other", handles.FsOther) + g.GET("/lark/export/download", handles.LarkExportDownload) g.Any("/dirs", handles.FsDirs) g.POST("/mkdir", handles.FsMkdir) g.POST("/rename", handles.FsRename) From 1c0fc0b213fedff8d0fd5f3d4f8b3c754a65c714 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 08:47:35 +0000 Subject: [PATCH 127/133] chore(deps): bump github.com/golang-jwt/jwt/v5 from 5.2.1 to 5.2.2 Bumps [github.com/golang-jwt/jwt/v5](https://github.com/golang-jwt/jwt) from 5.2.1 to 5.2.2. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Commits](https://github.com/golang-jwt/jwt/compare/v5.2.1...v5.2.2) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v5 dependency-version: 5.2.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 562296369cf..6a3d69d5836 100644 --- a/go.mod +++ b/go.mod @@ -221,7 +221,7 @@ require ( github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/go-webauthn/x v0.1.12 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/go.sum b/go.sum index e3bb088e1ef..a5b6a8f53f3 100644 --- a/go.sum +++ b/go.sum @@ -310,8 +310,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= From 0fa863ed27da714832675567d7e6c444dde3fc92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 16:47:56 +0800 Subject: [PATCH 128/133] fix: support all pagination mode (#9512) --- server/handles/fsread.go | 17 +++++++++++++++-- server/handles/share_public.go | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 9ca84edda52..1d85dcae935 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -82,6 +82,7 @@ type ObjLabelResp struct { const ( DefaultPerPage = 200 MaxPerPage = 500 + AllPerPage = -1 ) func FsList(c *gin.Context) { @@ -136,7 +137,7 @@ func FsList(c *gin.Context) { total, pageObjs := pagination(filtered, &req.PageReq) respContent := toObjsResp(pageObjs, reqPath, isEncrypt(meta, reqPath)) pagesTotal := calcPagesTotal(total, req.PerPage) - hasMore := req.Page*req.PerPage < total + hasMore := req.PerPage != AllPerPage && req.Page*req.PerPage < total common.SuccessResp(c, FsListResp{ Content: respContent, @@ -253,7 +254,10 @@ func normalizeListPage(page, perPage int) (int, int) { effPage = 1 } effPerPage := perPage - if effPerPage <= 0 { + if effPerPage < 0 { + return effPage, AllPerPage + } + if effPerPage == 0 { effPerPage = DefaultPerPage } if effPerPage > MaxPerPage { @@ -263,6 +267,12 @@ func normalizeListPage(page, perPage int) (int, int) { } func calcPagesTotal(total, perPage int) int { + if perPage == AllPerPage { + if total > 0 { + return 1 + } + return 0 + } if total <= 0 || perPage <= 0 { return 0 } @@ -272,6 +282,9 @@ func calcPagesTotal(total, perPage int) int { func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { pageIndex, pageSize := req.Page, req.PerPage total := len(objs) + if pageSize == AllPerPage { + return total, objs + } start := (pageIndex - 1) * pageSize if start > total { return total, []model.Obj{} diff --git a/server/handles/share_public.go b/server/handles/share_public.go index 83ca79f3591..f88d5b338c5 100644 --- a/server/handles/share_public.go +++ b/server/handles/share_public.go @@ -147,7 +147,7 @@ func ListPublicShare(c *gin.Context) { Total: int64(total), Page: req.Page, PerPage: req.PerPage, - HasMore: req.Page*req.PerPage < total, + HasMore: req.PerPage != AllPerPage && req.Page*req.PerPage < total, PagesTotal: calcPagesTotal(total, req.PerPage), }) } From fc26c3d8b2825093e3dafd286db1e77d07d04e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 16:53:35 +0800 Subject: [PATCH 129/133] feat: add GuangYaPan offline download (#9505) --- drivers/guangyapan/offline.go | 165 ++++++++++++++++++ drivers/guangyapan/types.go | 73 ++++++++ internal/conf/const.go | 27 +-- internal/offline_download/all.go | 1 + .../offline_download/guangyapan/guangyapan.go | 131 ++++++++++++++ internal/offline_download/guangyapan/util.go | 77 ++++++++ internal/offline_download/tool/add.go | 11 ++ internal/offline_download/tool/download.go | 5 +- server/handles/offline_download.go | 45 +++++ server/router.go | 1 + 10 files changed, 523 insertions(+), 13 deletions(-) create mode 100644 drivers/guangyapan/offline.go create mode 100644 internal/offline_download/guangyapan/guangyapan.go create mode 100644 internal/offline_download/guangyapan/util.go diff --git a/drivers/guangyapan/offline.go b/drivers/guangyapan/offline.go new file mode 100644 index 00000000000..d523aaa677c --- /dev/null +++ b/drivers/guangyapan/offline.go @@ -0,0 +1,165 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/model" +) + +func (d *GuangYaPan) ResolveOfflineResource(ctx context.Context, fileURL string) (*OfflineResolveData, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + fileURL = strings.TrimSpace(fileURL) + if fileURL == "" { + return nil, errors.New("offline url is empty") + } + + var resp offlineResolveResp + if err := d.postAPI(ctx, "/cloudcollection/v1/resolve_res", map[string]any{ + "url": fileURL, + }, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("resolve offline resource failed: %s", strings.TrimSpace(resp.Msg)) + } + return &resp.Data, nil +} + +func (d *GuangYaPan) OfflineDownload(ctx context.Context, fileURL string, parentDir model.Obj, fileName string) (*OfflineTask, error) { + resolved, err := d.ResolveOfflineResource(ctx, fileURL) + if err != nil { + return nil, err + } + + parentID := parentDir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + taskURL := strings.TrimSpace(resolved.URL) + if taskURL == "" { + taskURL = strings.TrimSpace(fileURL) + } + name := strings.TrimSpace(fileName) + if name == "" { + name = resolved.defaultName(taskURL) + } + + body := map[string]any{ + "url": taskURL, + "parentId": parentID, + "newName": name, + } + if indexes := resolved.fileIndexes(); len(indexes) > 0 { + body["fileIndexes"] = indexes + } + + var resp offlineCreateResp + if err := d.postAPI(ctx, "/cloudcollection/v1/create_task", body, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("create offline task failed: %s", strings.TrimSpace(resp.Msg)) + } + taskID := strings.TrimSpace(resp.Data.TaskID) + if taskID == "" { + return nil, errors.New("create offline task failed: empty task id") + } + return &OfflineTask{ + TaskID: taskID, + FileName: name, + Res: taskURL, + }, nil +} + +func (d *GuangYaPan) OfflineList(ctx context.Context, taskIDs []string, statuses []int, cursor string, pageSize int) ([]OfflineTask, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + body := map[string]any{} + if len(taskIDs) > 0 { + body["taskIds"] = taskIDs + } + if len(statuses) > 0 { + body["status"] = statuses + } + if cursor = strings.TrimSpace(cursor); cursor != "" { + body["cursor"] = cursor + } + if pageSize > 0 { + body["pageSize"] = pageSize + } + + var resp offlineListResp + if err := d.postAPI(ctx, "/cloudcollection/v1/list_task", body, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("list offline tasks failed: %s", strings.TrimSpace(resp.Msg)) + } + return resp.Data.List, nil +} + +func (d *GuangYaPan) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + if len(taskIDs) == 0 { + return nil + } + + var resp offlineDeleteResp + if err := d.postAPI(ctx, "/cloudcollection/v2/delete_task", map[string]any{ + "taskIds": taskIDs, + }, &resp); err != nil { + return err + } + if !isSuccessMsg(resp.Msg) { + return fmt.Errorf("delete offline tasks failed: %s", strings.TrimSpace(resp.Msg)) + } + return nil +} + +func (d OfflineResolveData) defaultName(fileURL string) string { + if d.BTResInfo != nil && strings.TrimSpace(d.BTResInfo.FileName) != "" { + return strings.TrimSpace(d.BTResInfo.FileName) + } + u, err := url.Parse(fileURL) + if err == nil { + name := strings.TrimSpace(stdpath.Base(u.Path)) + if name != "" && name != "." && name != "/" { + if decoded, err := url.PathUnescape(name); err == nil { + name = decoded + } + return name + } + } + return "offline_download" +} + +func (d OfflineResolveData) fileIndexes() []int { + if d.BTResInfo == nil || len(d.BTResInfo.Subfiles) == 0 { + return nil + } + indexes := make([]int, 0, len(d.BTResInfo.Subfiles)) + for i, file := range d.BTResInfo.Subfiles { + if file.FileIndex != nil { + indexes = append(indexes, *file.FileIndex) + continue + } + indexes = append(indexes, i) + } + return indexes +} + +func isSuccessMsg(msg string) bool { + msg = strings.TrimSpace(msg) + return msg == "" || strings.EqualFold(msg, "success") +} diff --git a/drivers/guangyapan/types.go b/drivers/guangyapan/types.go index bd0094f3070..1a1c129c06a 100644 --- a/drivers/guangyapan/types.go +++ b/drivers/guangyapan/types.go @@ -133,6 +133,79 @@ type taskInfoResp struct { } `json:"data"` } +type offlineResolveResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data OfflineResolveData `json:"data"` +} + +type OfflineResolveData struct { + ResType int `json:"resType"` + BTResInfo *OfflineBTResInfo `json:"btResInfo"` + URL string `json:"url"` +} + +type OfflineBTResInfo struct { + InfoHash string `json:"infoHash"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + SubfilesNum int `json:"subfilesNum"` + Subfiles []OfflineSubfile `json:"subfiles"` + CreateTime int64 `json:"createTime"` + ExcludeIndices []int `json:"excludeIndices"` +} + +type OfflineSubfile struct { + FileName string `json:"fileName"` + FileIndex *int `json:"fileIndex"` + FileSize int64 `json:"fileSize"` +} + +type offlineCreateResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID string `json:"taskId"` + URL string `json:"url"` + } `json:"data"` +} + +type offlineDeleteResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskIDs []string `json:"taskIds"` + } `json:"data"` +} + +type offlineListResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + StatusCounts []struct { + Status int `json:"status"` + Count int `json:"count"` + } `json:"statusCounts"` + Cursor string `json:"cursor"` + List []OfflineTask `json:"list"` + Total int `json:"total"` + } `json:"data"` +} + +type OfflineTask struct { + TaskID string `json:"taskId"` + FileName string `json:"fileName"` + TotalSize int64 `json:"totalSize"` + Status int `json:"status"` + CreateTime int64 `json:"createTime"` + Res string `json:"res"` + ResType int `json:"resType"` + Progress int `json:"progress"` + FileID string `json:"fileId"` + IsDir bool `json:"isDir"` + Exist bool `json:"exist"` +} + func unixOrZero(v int64) time.Time { if v <= 0 { return time.Time{} diff --git a/internal/conf/const.go b/internal/conf/const.go index 5bcb6eb5f91..4a3bbfa0b8f 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -76,6 +76,9 @@ const ( // thunder ThunderTempDir = "thunder_temp_dir" + // guangyapan + GuangYaPanTempDir = "guangyapan_temp_dir" + // single Token = "token" IndexProgress = "index_progress" @@ -126,19 +129,19 @@ const ( FTPTLSPublicCertPath = "ftp_tls_public_cert_path" // frp - FRPEnabled = "frp_enabled" - FRPServerAddr = "frp_server_addr" - FRPServerPort = "frp_server_port" - FRPAuthToken = "frp_auth_token" - FRPProxyName = "frp_proxy_name" - FRPProxyType = "frp_proxy_type" - FRPCustomDomain = "frp_custom_domain" - FRPSubdomain = "frp_subdomain" - FRPRemotePort = "frp_remote_port" - FRPLocalPort = "frp_local_port" - FRPTLSEnable = "frp_tls_enable" + FRPEnabled = "frp_enabled" + FRPServerAddr = "frp_server_addr" + FRPServerPort = "frp_server_port" + FRPAuthToken = "frp_auth_token" + FRPProxyName = "frp_proxy_name" + FRPProxyType = "frp_proxy_type" + FRPCustomDomain = "frp_custom_domain" + FRPSubdomain = "frp_subdomain" + FRPRemotePort = "frp_remote_port" + FRPLocalPort = "frp_local_port" + FRPTLSEnable = "frp_tls_enable" FRPSTCPSecretKey = "frp_stcp_secret_key" - FRPStatus = "frp_status" + FRPStatus = "frp_status" // traffic TaskOfflineDownloadThreadsNum = "offline_download_task_threads_num" diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index 3d0c7c73a0b..1ba191e47e8 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -3,6 +3,7 @@ package offline_download import ( _ "github.com/alist-org/alist/v3/internal/offline_download/115" _ "github.com/alist-org/alist/v3/internal/offline_download/aria2" + _ "github.com/alist-org/alist/v3/internal/offline_download/guangyapan" _ "github.com/alist-org/alist/v3/internal/offline_download/http" _ "github.com/alist-org/alist/v3/internal/offline_download/pikpak" _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" diff --git a/internal/offline_download/guangyapan/guangyapan.go b/internal/offline_download/guangyapan/guangyapan.go new file mode 100644 index 00000000000..eb58af0bd4a --- /dev/null +++ b/internal/offline_download/guangyapan/guangyapan.go @@ -0,0 +1,131 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" +) + +type GuangYaPan struct { + refreshTaskCache bool +} + +func (g *GuangYaPan) Name() string { + return "GuangYaPan" +} + +func (g *GuangYaPan) Items() []model.SettingItem { + return nil +} + +func (g *GuangYaPan) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (g *GuangYaPan) Init() (string, error) { + g.refreshTaskCache = false + return "ok", nil +} + +func (g *GuangYaPan) IsReady() bool { + tempDir := setting.GetStr(conf.GuangYaPanTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*guangyapandriver.GuangYaPan); !ok { + return false + } + return true +} + +func (g *GuangYaPan) AddURL(args *tool.AddUrlArgs) (string, error) { + g.refreshTaskCache = true + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return "", errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + + ctx := context.Background() + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + task, err := driver.OfflineDownload(ctx, args.Url, parentDir, "") + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + return task.TaskID, nil +} + +func (g *GuangYaPan) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + ctx := context.Background() + if err := driver.DeleteOfflineTasks(ctx, []string{task.GID}, false); err != nil { + return err + } + g.DelTaskCache(driver, task.GID) + return nil +} + +func (g *GuangYaPan) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return nil, errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + tasks, err := g.GetTasks(driver, task.GID) + if err != nil { + return nil, err + } + status := &tool.Status{ + Status: "the task has been deleted", + } + for _, t := range tasks { + if t.TaskID != task.GID { + continue + } + status.Progress = float64(t.Progress) + status.TotalBytes = t.TotalSize + status.Completed = t.Status == offlineStatusCompleted || t.Status == offlineStatusPartiallyCompleted + status.Status = taskStatusText(t) + if t.Status == offlineStatusFailed || t.Status == offlineStatusCanceled { + status.Err = errors.New(status.Status) + } + return status, nil + } + status.Err = errors.New("the task has been deleted") + return status, nil +} + +func init() { + tool.Tools.Add(&GuangYaPan{}) +} diff --git a/internal/offline_download/guangyapan/util.go b/internal/offline_download/guangyapan/util.go new file mode 100644 index 00000000000..cb7bb926ead --- /dev/null +++ b/internal/offline_download/guangyapan/util.go @@ -0,0 +1,77 @@ +package guangyapan + +import ( + "context" + "fmt" + "time" + + "github.com/Xhofe/go-cache" + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/singleflight" +) + +const ( + offlineStatusQueued = 0 + offlineStatusRunning = 1 + offlineStatusCompleted = 2 + offlineStatusFailed = 3 + offlineStatusCanceled = 4 + offlineStatusPartiallyCompleted = 5 +) + +var taskCache = cache.NewMemCache(cache.WithShards[[]guangyapandriver.OfflineTask](16)) +var taskG singleflight.Group[[]guangyapandriver.OfflineTask] + +func (g *GuangYaPan) GetTasks(driver *guangyapandriver.GuangYaPan, taskID string) ([]guangyapandriver.OfflineTask, error) { + key := op.Key(driver, "/cloudcollection/v1/list_task/"+taskID) + if !g.refreshTaskCache { + if tasks, ok := taskCache.Get(key); ok { + return tasks, nil + } + } + g.refreshTaskCache = false + tasks, err, _ := taskG.Do(key, func() ([]guangyapandriver.OfflineTask, error) { + ctx := context.Background() + tasks, err := driver.OfflineList(ctx, []string{taskID}, nil, "", 0) + if err != nil { + return nil, err + } + if len(tasks) > 0 { + taskCache.Set(key, tasks, cache.WithEx[[]guangyapandriver.OfflineTask](time.Second*10)) + } else { + taskCache.Del(key) + } + return tasks, nil + }) + if err != nil { + return nil, err + } + return tasks, nil +} + +func (g *GuangYaPan) DelTaskCache(driver *guangyapandriver.GuangYaPan, taskID string) { + taskCache.Del(op.Key(driver, "/cloudcollection/v1/list_task/"+taskID)) +} + +func taskStatusText(task guangyapandriver.OfflineTask) string { + switch task.Status { + case offlineStatusQueued: + return "queued" + case offlineStatusRunning: + if task.Progress > 0 { + return fmt.Sprintf("running (%d%%)", task.Progress) + } + return "running" + case offlineStatusCompleted: + return "completed" + case offlineStatusFailed: + return "failed" + case offlineStatusCanceled: + return "canceled" + case offlineStatusPartiallyCompleted: + return "partially completed" + default: + return fmt.Sprintf("unknown status %d", task.Status) + } +} diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index d64e43e8615..92e7a3e1dfb 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -7,6 +7,7 @@ import ( "path/filepath" _115 "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/drivers/guangyapan" "github.com/alist-org/alist/v3/drivers/pikpak" "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" @@ -103,6 +104,16 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } else { tempDir = filepath.Join(setting.GetStr(conf.ThunderTempDir), uid) } + case "GuangYaPan": + if _, ok := storage.(*guangyapan.GuangYaPan); ok { + tempDir = args.DstDirPath + } else { + tempBase := setting.GetStr(conf.GuangYaPanTempDir) + if tempBase == "" { + return nil, errors.New("GuangYaPan temp dir is not set") + } + tempDir = filepath.Join(tempBase, uid) + } } taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 42b2dbfb2cb..c6ad09947e1 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -87,6 +87,9 @@ outer: if t.tool.Name() == "Thunder" { return nil } + if t.tool.Name() == "GuangYaPan" { + return nil + } if t.tool.Name() == "115 Cloud" { // hack for 115 <-time.After(time.Second * 1) @@ -159,7 +162,7 @@ func (t *DownloadTask) Update() (bool, error) { func (t *DownloadTask) Transfer() error { toolName := t.tool.Name() - if toolName == "115 Cloud" || toolName == "PikPak" || toolName == "Thunder" { + if toolName == "115 Cloud" || toolName == "PikPak" || toolName == "Thunder" || toolName == "GuangYaPan" { // 如果不是直接下载到目标路径，则进行转存 if t.TempDir != t.DstDirPath { return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 8aade9eae57..68a922efda7 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -2,6 +2,7 @@ package handles import ( _115 "github.com/alist-org/alist/v3/drivers/115" + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" "github.com/alist-org/alist/v3/drivers/pikpak" "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" @@ -240,6 +241,50 @@ func SetThunder(c *gin.Context) { common.SuccessResp(c, "ok") } +type SetGuangYaPanReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetGuangYaPan(c *gin.Context) { + var req SetGuangYaPanReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*guangyapandriver.GuangYaPan); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only GuangYaPan is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.GuangYaPanTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("GuangYaPan") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + func OfflineDownloadTools(c *gin.Context) { tools := tool.Tools.Names() common.SuccessResp(c, tools) diff --git a/server/router.go b/server/router.go index e930177910a..e42d09472d1 100644 --- a/server/router.go +++ b/server/router.go @@ -181,6 +181,7 @@ func admin(g *gin.RouterGroup) { setting.POST("/set_115", handles.Set115) setting.POST("/set_pikpak", handles.SetPikPak) setting.POST("/set_thunder", handles.SetThunder) + setting.POST("/set_guangyapan", handles.SetGuangYaPan) setting.POST("/set_frp", handles.SetFRP) setting.POST("/stop_frp", handles.StopFRP) setting.GET("/frp_runtime", handles.GetFRPRuntime) From f4459fcbadbb6d1c7006ff8f1c65646859220e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 16:54:35 +0800 Subject: [PATCH 130/133] fix(meta): expire missing meta cache (#9504) --- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 1 + internal/op/meta.go | 35 ++++++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 265a6502b60..2faba91ea31 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -169,6 +169,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.MaxDevices, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL}, {Key: conf.DeviceEvictPolicy, Value: "deny", Type: conf.TypeSelect, Options: "deny,evict_oldest", Group: model.GLOBAL}, {Key: conf.DeviceSessionTTL, Value: "86400", Type: conf.TypeNumber, Group: model.GLOBAL}, + {Key: conf.MetaNotFoundCacheExpire, Value: "60", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE, Help: "Negative cache expiration for missing meta records, in seconds. Set 0 to disable."}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 4a3bbfa0b8f..9218e28f74e 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -52,6 +52,7 @@ const ( MaxDevices = "max_devices" DeviceEvictPolicy = "device_evict_policy" DeviceSessionTTL = "device_session_ttl" + MetaNotFoundCacheExpire = "meta_not_found_cache_expire" // index SearchIndex = "search_index" diff --git a/internal/op/meta.go b/internal/op/meta.go index 930f49634c3..29146fcc72c 100644 --- a/internal/op/meta.go +++ b/internal/op/meta.go @@ -2,9 +2,11 @@ package op import ( stdpath "path" + "strconv" "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -19,6 +21,17 @@ var metaCache = cache.NewMemCache(cache.WithShards[*model.Meta](2)) // metaG maybe not needed var metaG singleflight.Group[*model.Meta] +const ( + metaCacheExpiration = time.Hour + defaultMetaNotFoundCacheSec = 60 +) + +func init() { + RegisterSettingChangingCallback(func() { + metaCache.Clear() + }) +} + func GetNearestMeta(path string) (*model.Meta, error) { return getNearestMeta(utils.FixAndCleanPath(path)) } @@ -51,17 +64,34 @@ func getMetaByPath(path string) (*model.Meta, error) { _meta, err := db.GetMetaByPath(path) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { - metaCache.Set(path, nil) + if ex := metaNotFoundCacheExpiration(); ex > 0 { + metaCache.Set(path, nil, cache.WithEx[*model.Meta](ex)) + } return nil, errs.MetaNotFound } return nil, err } - metaCache.Set(path, _meta, cache.WithEx[*model.Meta](time.Hour)) + metaCache.Set(path, _meta, cache.WithEx[*model.Meta](metaCacheExpiration)) return _meta, nil }) return meta, err } +func metaNotFoundCacheExpiration() time.Duration { + item, err := GetSettingItemByKey(conf.MetaNotFoundCacheExpire) + if err != nil || item == nil { + return time.Second * defaultMetaNotFoundCacheSec + } + seconds, err := strconv.Atoi(item.Value) + if err != nil { + return time.Second * defaultMetaNotFoundCacheSec + } + if seconds <= 0 { + return 0 + } + return time.Second * time.Duration(seconds) +} + func DeleteMetaById(id uint) error { old, err := db.GetMetaById(id) if err != nil { @@ -78,6 +108,7 @@ func UpdateMeta(u *model.Meta) error { return err } metaCache.Del(old.Path) + metaCache.Del(u.Path) return db.UpdateMeta(u) } From de56968a2d59d1cdd67cdafd68cfa1ee7024d6ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 21:08:50 +0800 Subject: [PATCH 131/133] fix(guangyapan): resolve offline root folder lookup (#9516) --- drivers/guangyapan/offline.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/guangyapan/offline.go b/drivers/guangyapan/offline.go index d523aaa677c..a53b8adc33f 100644 --- a/drivers/guangyapan/offline.go +++ b/drivers/guangyapan/offline.go @@ -39,7 +39,11 @@ func (d *GuangYaPan) OfflineDownload(ctx context.Context, fileURL string, parent } parentID := parentDir.GetID() - if parentID == d.RootFolderID { + rootID, err := d.getRootFolderID(ctx) + if err != nil { + return nil, err + } + if parentID == rootID { parentID = "" } From 5cf59e48df2a498b1ee9f7952a840efb74b2454b Mon Sep 17 00:00:00 2001 From: Premiermoney Date: Mon, 25 May 2026 01:40:07 -0400 Subject: [PATCH 132/133] Create go.yml --- .github/workflows/go.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 00000000000..0b443f376a6 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,28 @@ +# This workflow will build a golang project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go + +name: Go + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.20' + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... From 575b0d2b29a4bcaaedfa8132f63a0657b32e4261 Mon Sep 17 00:00:00 2001 From: Premiermoney Date: Mon, 25 May 2026 21:30:55 -0400 Subject: [PATCH 133/133] Create docker-publish.yml --- .github/workflows/docker-publish.yml | 98 ++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 00000000000..f876760e54d --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,98 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + schedule: + - cron: '41 23 * * *' + push: + branches: [ "main" ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + pull_request: + branches: [ "main" ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: 'v2.2.4' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}