diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index a336406b355..f9e80a5aada 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -10,4 +10,4 @@ liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry -custom: ['https://alist.nn.ci/guide/sponsor.html'] +custom: ['https://alistgo.com/guide/sponsor.html'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 07a8338e5ef..f5cfaedacf8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -16,14 +16,14 @@ body: 您必须勾选以下所有内容，否则您的issue可能会被直接关闭。或者您可以去[讨论区](https://github.com/alist-org/alist/discussions) options: - label: | - I have read the [documentation](https://alist.nn.ci). - 我已经阅读了[文档](https://alist.nn.ci)。 + I have read the [documentation](https://alistgo.com). + 我已经阅读了[文档](https://alistgo.com)。 - label: | I'm sure there are no duplicate issues or discussions. 我确定没有重复的issue或讨论。 - label: | - I'm sure it's due to `AList` and not something else(such as [Network](https://alist.nn.ci/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host) ,`Dependencies` or `Operational`). - 我确定是`AList`的问题，而不是其他原因（例如[网络](https://alist.nn.ci/zh/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host)，`依赖`或`操作`）。 + I'm sure it's due to `AList` and not something else(such as [Network](https://alistgo.com/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host) ,`Dependencies` or `Operational`). + 我确定是`AList`的问题，而不是其他原因（例如[网络](https://alistgo.com/zh/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host)，`依赖`或`操作`）。 - label: | I'm sure this issue is not fixed in the latest version. 我确定这个问题在最新版本中没有被修复。 diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index a16c8f98d24..a118992ce0b 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -7,7 +7,7 @@ body: label: Please make sure of the following things description: You may select more than one, even select all. options: - - label: I have read the [documentation](https://alist.nn.ci). + - label: I have read the [documentation](https://alistgo.com). - label: I'm sure there are no duplicate issues or discussions. - label: I'm sure this feature is not implemented. - label: I'm sure it's a reasonable and popular requirement. diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 485942c4a9b..27e0142ea3f 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -119,7 +119,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 with: - repository: alist-org/desktop-release + repository: AlistGo/desktop-release ref: main persist-credentials: false fetch-depth: 0 @@ -135,4 +135,4 @@ jobs: with: github_token: ${{ secrets.MY_TOKEN }} branch: main - repository: alist-org/desktop-release \ No newline at end of file + repository: AlistGo/desktop-release \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2c934e7a5e..cf6eff39e48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,8 @@ jobs: - android-arm64 name: Build runs-on: ${{ matrix.platform }} + env: + GOPROXY: https://proxy.golang.org,direct steps: - name: Checkout diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 00000000000..f876760e54d --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,98 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + schedule: + - cron: '41 23 * * *' + push: + branches: [ "main" ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + pull_request: + branches: [ "main" ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: 'v2.2.4' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 00000000000..0b443f376a6 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,28 @@ +# This workflow will build a golang project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go + +name: Go + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.20' + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... diff --git a/.github/workflows/merge_frontend_pr.yml b/.github/workflows/merge_frontend_pr.yml new file mode 100644 index 00000000000..f4e3434306b --- /dev/null +++ b/.github/workflows/merge_frontend_pr.yml @@ -0,0 +1,70 @@ +name: merge companion frontend pr + +on: + pull_request: + branches: + - 'main' + types: + - closed + workflow_dispatch: + inputs: + frontend_pr: + description: 'Frontend PR reference, e.g. AlistGo/alist-web#301 or https://github.com/AlistGo/alist-web/pull/301' + required: true + type: string + +permissions: + contents: read + +jobs: + merge_frontend_pr: + name: Merge companion frontend PR + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Find frontend PR + id: frontend + env: + INPUT_FRONTEND_PR: ${{ inputs.frontend_pr }} + PR_BODY: ${{ github.event.pull_request.body }} + run: | + set -euo pipefail + + text="${INPUT_FRONTEND_PR:-${PR_BODY:-}}" + ref="$(printf '%s\n' "$text" | grep -Eo '(https://github\.com/(AlistGo|alist-org)/alist-web/pull/[0-9]+)|((AlistGo|alist-org)/alist-web#[0-9]+)|(alist-web#[0-9]+)' | head -n1 || true)" + + if [[ -z "$ref" ]]; then + echo "found=false" >> "$GITHUB_OUTPUT" + echo "No companion frontend PR referenced." + exit 0 + fi + + if [[ "$ref" == *"/pull/"* ]]; then + number="${ref##*/}" + else + number="${ref##*#}" + fi + + echo "found=true" >> "$GITHUB_OUTPUT" + echo "url=https://github.com/AlistGo/alist-web/pull/$number" >> "$GITHUB_OUTPUT" + echo "Found companion frontend PR #$number." + + - name: Merge frontend PR + if: steps.frontend.outputs.found == 'true' + env: + GH_TOKEN: ${{ secrets.MY_TOKEN }} + FRONTEND_PR: ${{ steps.frontend.outputs.url }} + run: | + set -euo pipefail + + state="$(gh pr view "$FRONTEND_PR" --json state --jq .state)" + if [[ "$state" == "MERGED" ]]; then + echo "$FRONTEND_PR is already merged." + exit 0 + fi + if [[ "$state" != "OPEN" ]]; then + echo "$FRONTEND_PR is $state and cannot be merged." + exit 1 + fi + + gh pr merge "$FRONTEND_PR" --auto --merge || gh pr merge "$FRONTEND_PR" --merge diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d42019ad06..2257826b064 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,7 +72,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 with: - repository: alist-org/desktop-release + repository: AlistGo/desktop-release ref: main persist-credentials: false fetch-depth: 0 @@ -89,4 +89,4 @@ jobs: with: github_token: ${{ secrets.MY_TOKEN }} branch: main - repository: alist-org/desktop-release \ No newline at end of file + repository: AlistGo/desktop-release \ No newline at end of file diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 7cd05549f18..1c31b2fd20f 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -18,6 +18,7 @@ env: REGISTRY: 'xhofe/alist' REGISTRY_USERNAME: 'xhofe' REGISTRY_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + GITHUB_CR_REPO: ghcr.io/${{ github.repository }} ARTIFACT_NAME: 'binaries_docker_release' RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64' IMAGE_PUSH: ${{ github.event_name == 'push' }} @@ -114,11 +115,21 @@ jobs: username: ${{ env.REGISTRY_USERNAME }} password: ${{ env.REGISTRY_PASSWORD }} + - name: Login to GHCR + uses: docker/login-action@v3 + with: + logout: true + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }} + images: | + ${{ env.REGISTRY }} + ${{ env.GITHUB_CR_REPO }} tags: ${{ env.IMAGE_IS_PROD == 'true' && '' || env.IMAGE_TAGS_BETA }} flavor: | ${{ env.IMAGE_IS_PROD == 'true' && 'latest=true' || '' }} diff --git a/Dockerfile.ci b/Dockerfile.ci index a17aae9fcfd..6075acc639a 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:edge +FROM alpine:3.20.7 ARG TARGETPLATFORM ARG INSTALL_FFMPEG=false @@ -31,4 +31,4 @@ RUN /entrypoint.sh version ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} VOLUME /opt/alist/data/ EXPOSE 5244 5245 -CMD [ "/entrypoint.sh" ] \ No newline at end of file +CMD [ "/entrypoint.sh" ] diff --git a/README.md b/README.md index 1261839e429..032c2d17362 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
- logo + logo

🗂️A file list program that supports multiple storages, powered by Gin and Solidjs.

@@ -31,7 +31,7 @@ Downloads - + sponsor
@@ -57,7 +57,9 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] WebDav(Support OneDrive/SharePoint without API) - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) + - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) - [x] [BaiduNetdisk](http://pan.baidu.com/) @@ -88,7 +90,7 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing - [x] Dark mode - [x] I18n - [x] Protected routes (password protection and authentication) -- [x] WebDav (see https://alist.nn.ci/guide/webdav.html for details) +- [x] WebDav (see https://alistgo.com/guide/webdav.html for details) - [x] [Docker Deploy](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare Workers proxy - [x] File/Folder package download @@ -101,6 +103,10 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing +## API Documentation (via Apifox): + + + ## Demo @@ -112,13 +118,11 @@ Please go to our [discussion forum](https://github.com/alist-org/alist/discussio ## Sponsor AList is an open-source software, if you happen to like this project and want me to keep going, please consider sponsoring me or providing a single donation! Thanks for all the love and support: -https://alist.nn.ci/guide/sponsor.html +https://alistgo.com/guide/sponsor.html ### Special sponsors - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. -- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) -- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## Contributors diff --git a/README_cn.md b/README_cn.md index 5c71ccce4c3..cf1b1e1c29a 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@
- logo + logo

🗂一个支持多存储的文件列表程序，使用 Gin 和 Solidjs。

@@ -57,7 +57,9 @@ - [x] [又拍云对象存储](https://www.upyun.com/products/file-storage) - [x] WebDav(支持无API的OneDrive/SharePoint) - [x] Teambition（[中国](https://www.teambition.com/ )，[国际](https://us.teambition.com/ )） + - [x] [MediaFire](https://www.mediafire.com) - [x] [分秒帧](https://www.mediatrack.cn/) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [和彩云](https://yun.139.com/) (个人云, 家庭云，共享群组) - [x] [Yandex.Disk](https://disk.yandex.com/) - [x] [百度网盘](http://pan.baidu.com/) @@ -86,7 +88,7 @@ - [x] 黑暗模式 - [x] 国际化 - [x] 受保护的路由（密码保护和身份验证） -- [x] WebDav (具体见 https://alist.nn.ci/zh/guide/webdav.html) +- [x] WebDav (具体见 https://alistgo.com/zh/guide/webdav.html) - [x] [Docker 部署](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare workers 中转 - [x] 文件/文件夹打包下载 @@ -97,7 +99,11 @@ ## 文档 - + + +## API 文档（通过 Apifox 提供） + + ## Demo @@ -109,13 +115,11 @@ ## 赞助 -AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我继续下去，请考虑赞助我或提供一个单一的捐款！感谢所有的爱和支持：https://alist.nn.ci/zh/guide/sponsor.html +AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我继续下去，请考虑赞助我或提供一个单一的捐款！感谢所有的爱和支持：https://alistgo.com/zh/guide/sponsor.html ### 特别赞助 - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - 苹果生态下优雅的网盘视频播放器，iPhone，iPad，Mac，Apple TV全平台支持。 -- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (国内API服务器赞助) -- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## 贡献者 diff --git a/README_ja.md b/README_ja.md index cd4446fab8e..a1a21253068 100644 --- a/README_ja.md +++ b/README_ja.md @@ -1,5 +1,5 @@
- logo + logo

🗂️Gin と Solidjs による、複数のストレージをサポートするファイルリストプログラム。

@@ -57,7 +57,9 @@ - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] WebDav(Support OneDrive/SharePoint without API) - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) + - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) - [x] [BaiduNetdisk](http://pan.baidu.com/) @@ -87,7 +89,7 @@ - [x] ダークモード - [x] 国際化 - [x] 保護されたルート (パスワード保護と認証) -- [x] WebDav (詳細は https://alist.nn.ci/guide/webdav.html を参照) +- [x] WebDav (詳細は https://alistgo.com/guide/webdav.html を参照) - [x] [Docker デプロイ](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare ワーカープロキシ - [x] ファイル/フォルダパッケージのダウンロード @@ -98,7 +100,11 @@ ## ドキュメント - + + +## APIドキュメント（Apifox 提供） + + ## デモ @@ -111,13 +117,11 @@ ## スポンサー AList はオープンソースのソフトウェアです。もしあなたがこのプロジェクトを気に入ってくださり、続けて欲しいと思ってくださるなら、ぜひスポンサーになってくださるか、1口でも寄付をしてくださるようご検討ください！すべての愛とサポートに感謝します: -https://alist.nn.ci/guide/sponsor.html +https://alistgo.com/guide/sponsor.html ### スペシャルスポンサー - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. -- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) -- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## コントリビューター diff --git a/build.sh b/build.sh index 2dee8e20773..4045820adfd 100644 --- a/build.sh +++ b/build.sh @@ -93,7 +93,7 @@ BuildDocker() { PrepareBuildDockerMusl() { mkdir -p build/musl-libs - BASE="https://musl.cc/" + BASE="https://github.com/go-cross/musl-toolchain-archive/releases/latest/download/" FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross s390x-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross riscv64-linux-musl-cross powerpc64le-linux-musl-cross) for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" @@ -245,7 +245,7 @@ BuildReleaseFreeBSD() { cgo_cc="clang --target=${CGO_ARGS[$i]} --sysroot=/opt/freebsd/${os_arch}" echo building for freebsd-${os_arch} sudo mkdir -p "/opt/freebsd/${os_arch}" - wget -q https://download.freebsd.org/releases/${os_arch}/14.1-RELEASE/base.txz + wget -q https://download.freebsd.org/releases/${os_arch}/14.3-RELEASE/base.txz sudo tar -xf ./base.txz -C /opt/freebsd/${os_arch} rm base.txz export GOOS=freebsd diff --git a/cmd/common.go b/cmd/common.go index 8a73f9b0582..d88a86eb09d 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_46_0" "os" "path/filepath" "strconv" @@ -16,6 +17,12 @@ func Init() { bootstrap.InitConfig() bootstrap.Log() bootstrap.InitDB() + + if v3_46_0.IsLegacyRoleDetected() { + utils.Log.Warnf("Detected legacy role format, executing ConvertLegacyRoles patch early...") + v3_46_0.ConvertLegacyRoles() + } + data.InitData() bootstrap.InitStreamLimit() bootstrap.InitIndex() diff --git a/cmd/mcp.go b/cmd/mcp.go new file mode 100644 index 00000000000..356b8bfc1b8 --- /dev/null +++ b/cmd/mcp.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "github.com/alist-org/alist/v3/internal/bootstrap" + mcpserver "github.com/alist-org/alist/v3/server/mcp" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/spf13/cobra" +) + +var MCPCmd = &cobra.Command{ + Use: "mcp", + Short: "Start MCP server in STDIO mode", + Long: `Start an MCP (Model Context Protocol) server that communicates via STDIO, suitable for integration with AI assistants like Claude Desktop.`, + Run: func(cmd *cobra.Command, args []string) { + Init() + bootstrap.LoadStorages() + username, _ := cmd.Flags().GetString("user") + if err := mcpserver.ServeStdio(username); err != nil { + utils.Log.Fatalf("MCP STDIO server error: %v", err) + } + }, +} + +func init() { + MCPCmd.Flags().String("user", "admin", "Username for MCP operations") + RootCmd.AddCommand(MCPCmd) +} diff --git a/cmd/root.go b/cmd/root.go index 59eb989c3a0..cd50529728b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ var RootCmd = &cobra.Command{ Short: "A file list program that supports multiple storage.", Long: `A file list program that supports multiple storage, built with love by Xhofe and friends in Go/Solid.js. -Complete documentation is available at https://alist.nn.ci/`, +Complete documentation is available at https://alistgo.com/`, } func Execute() { diff --git a/cmd/server.go b/cmd/server.go index 4263f02021d..9e76333a627 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -18,9 +18,11 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/bootstrap" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server" + mcpserver "github.com/alist-org/alist/v3/server/mcp" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -43,6 +45,7 @@ the address is defined in config file`, bootstrap.InitOfflineDownloadTools() bootstrap.LoadStorages() bootstrap.InitTaskManager() + bootstrap.InitFRP() if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) } @@ -157,6 +160,19 @@ the address is defined in config file`, }() } } + var mcpHttpSrv *http.Server + if conf.Conf.MCP.Port != -1 && conf.Conf.MCP.Enable { + mcpHandler := mcpserver.NewHTTPHandler() + mcpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.MCP.Port) + utils.Log.Infof("start MCP server @ %s", mcpBase) + mcpHttpSrv = &http.Server{Addr: mcpBase, Handler: mcpHandler} + go func() { + err := mcpHttpSrv.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + utils.Log.Fatalf("failed to start MCP server: %s", err.Error()) + } + }() + } // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) @@ -167,6 +183,7 @@ the address is defined in config file`, <-quit utils.Log.Println("Shutdown server...") fs.ArchiveContentUploadTaskManager.RemoveAll() + frp.Instance.Stop() Release() ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() @@ -217,6 +234,15 @@ the address is defined in config file`, } }() } + if conf.Conf.MCP.Port != -1 && conf.Conf.MCP.Enable && mcpHttpSrv != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := mcpHttpSrv.Shutdown(ctx); err != nil { + utils.Log.Fatal("MCP server shutdown err: ", err) + } + }() + } wg.Wait() utils.Log.Println("Server exit") }, diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 0dcb64d8284..60fe60e68e7 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -66,9 +66,11 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err := d.WaitLimit(ctx); err != nil { return nil, err } - userAgent := args.Header.Get("User-Agent") - downloadInfo, err := d. - DownloadWithUA(file.(*FileObj).PickCode, userAgent) + userAgent := "" + if args.Header != nil { + userAgent = args.Header.Get("User-Agent") + } + downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err } diff --git a/drivers/115/types.go b/drivers/115/types.go index 40b951d80ce..7a80e3ef047 100644 --- a/drivers/115/types.go +++ b/drivers/115/types.go @@ -12,6 +12,7 @@ var _ model.Obj = (*FileObj)(nil) type FileObj struct { driver.File + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -22,6 +23,10 @@ func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + type UploadResult struct { driver.BasicResp Data struct { diff --git a/drivers/115/util.go b/drivers/115/util.go index fc17fe3cebf..79d869178cb 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "net/url" "strconv" "strings" @@ -25,11 +24,28 @@ import ( "github.com/aliyun/aliyun-oss-go-sdk/oss" cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" - crypto "github.com/SheltonZhu/115driver/pkg/crypto/m115" driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/pkg/errors" ) +type fileInfoWithThumb struct { + driver115.FileInfo + ThumbURL string `json:"u"` +} + +type fileListRespWithThumb struct { + driver115.BasicResp + CategoryID driver115.IntString `json:"cid"` + Count int `json:"count"` + Offset int `json:"offset"` + Files []fileInfoWithThumb `json:"data"` +} + +type getFileInfoResponseWithThumb struct { + driver115.BasicResp + Files []*fileInfoWithThumb `json:"data"` +} + // var UserAgent = driver115.UA115Browser func (d *Pan115) login() error { var err error @@ -66,100 +82,113 @@ func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { if d.PageSize <= 0 { d.PageSize = driver115.FileListLimit } - files, err := d.client.ListWithLimit(fileId, d.PageSize, driver115.WithMultiUrls()) - if err != nil { - return nil, err + limit := d.PageSize + if limit > driver115.MaxDirPageLimit { + limit = driver115.MaxDirPageLimit } - for _, file := range *files { - res = append(res, FileObj{file}) + + opts := driver115.DefaultListOptions() + driver115.WithMultiUrls()(opts) + if len(opts.ApiURLs) == 0 { + opts.ApiURLs = []string{driver115.ApiFileList} } + + offset := int64(0) + for i := 0; ; i++ { + result, err := d.getFilesPageWithThumb(fileId, opts.ApiURLs[i%len(opts.ApiURLs)], limit, offset) + if err != nil { + return nil, err + } + for _, fileInfo := range result.Files { + res = append(res, fileObjFromInfo(&fileInfo)) + } + offset = int64(result.Offset) + limit + if offset >= int64(result.Count) { + break + } + } + return res, nil } func (d *Pan115) getNewFile(fileId string) (*FileObj, error) { - file, err := d.client.GetFile(fileId) + fileInfo, err := d.getFileInfoWithThumb("file_id", fileId) if err != nil { return nil, err } - return &FileObj{*file}, nil + file := fileObjFromInfo(fileInfo) + return &file, nil } func (d *Pan115) getNewFileByPickCode(pickCode string) (*FileObj, error) { - result := driver115.GetFileInfoResponse{} - req := d.client.NewRequest(). - SetQueryParam("pick_code", pickCode). - ForceContentType("application/json;charset=UTF-8"). - SetResult(&result) - resp, err := req.Get(driver115.ApiFileInfo) - if err := driver115.CheckErr(err, &result, resp); err != nil { + fileInfo, err := d.getFileInfoWithThumb("pick_code", pickCode) + if err != nil { return nil, err } - if len(result.Files) == 0 { - return nil, errors.New("not get file info") - } - fileInfo := result.Files[0] - - f := &FileObj{} - f.From(fileInfo) - return f, nil + file := fileObjFromInfo(fileInfo) + return &file, nil } func (d *Pan115) getUA() string { return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } -func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { - key := crypto.GenerateKey() - result := driver115.DownloadResp{} - params, err := utils.Json.Marshal(map[string]string{"pick_code": pickCode}) - if err != nil { - return nil, err - } - - data := crypto.Encode(params, key) - - bodyReader := strings.NewReader(url.Values{"data": []string{data}}.Encode()) - reqUrl := fmt.Sprintf("%s?t=%s", driver115.AndroidApiDownloadGetUrl, driver115.Now().String()) - req, _ := http.NewRequest(http.MethodPost, reqUrl, bodyReader) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Cookie", d.Cookie) - req.Header.Set("User-Agent", ua) - - resp, err := d.client.Client.GetClient().Do(req) - if err != nil { - return nil, err +func fileObjFromInfo(fileInfo *fileInfoWithThumb) FileObj { + file := &driver115.File{} + file.From(&fileInfo.FileInfo) + return FileObj{ + File: *file, + ThumbURL: fileInfo.ThumbURL, } - defer resp.Body.Close() +} - body, err := io.ReadAll(resp.Body) - if err != nil { +func (d *Pan115) getFileInfoWithThumb(queryKey, queryVal string) (*fileInfoWithThumb, error) { + result := getFileInfoResponseWithThumb{} + req := d.client.NewRequest(). + SetQueryParam(queryKey, queryVal). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&result) + resp, err := req.Get(driver115.ApiFileInfo) + if err := driver115.CheckErr(err, &result, resp); err != nil { return nil, err } - if err := utils.Json.Unmarshal(body, &result); err != nil { - return nil, err + if len(result.Files) == 0 { + return nil, errors.New("not get file info") } + return result.Files[0], nil +} - if err = result.Err(string(body)); err != nil { - return nil, err +func (d *Pan115) getFilesPageWithThumb(dirID, apiURL string, limit, offset int64) (*fileListRespWithThumb, error) { + if dirID == "" { + dirID = "0" + } + result := fileListRespWithThumb{} + params := map[string]string{ + "aid": "1", + "cid": dirID, + "o": driver115.FileOrderByTime, + "asc": "1", + "offset": strconv.FormatInt(offset, 10), + "show_dir": "1", + "limit": strconv.FormatInt(limit, 10), + "snap": "0", + "natsort": "0", + "record_open_time": "1", + "format": "json", + "fc_mix": "0", } - - b, err := crypto.Decode(string(result.EncodedData), key) - if err != nil { + req := d.client.NewRequest(). + ForceContentType("application/json;charset=UTF-8"). + SetQueryParams(params). + SetResult(&result) + resp, err := req.Get(apiURL) + if err := driver115.CheckErr(err, &result, resp); err != nil { return nil, err } - - downloadInfo := struct { - Url string `json:"url"` - }{} - if err := utils.Json.Unmarshal(b, &downloadInfo); err != nil { - return nil, err + if dirID != string(result.CategoryID) { + return nil, driver115.ErrUnexpected } - - info := &driver115.DownloadInfo{} - info.PickCode = pickCode - info.Header = resp.Request.Header - info.Url.Url = downloadInfo.Url - return info, nil + return &result, nil } func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index 886a369c1b8..322b64afd45 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -4,6 +4,7 @@ import ( "context" driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -50,8 +51,9 @@ func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListAr return nil, err } - files := make([]driver115.ShareFile, 0) - fileResp, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) + ua := base.UserAgent + files := make([]shareFile, 0) + fileResp, err := d.getShareSnapWithUA(ua, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) if err != nil { return nil, err } @@ -59,10 +61,7 @@ func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListAr total := fileResp.Data.Count count := len(fileResp.Data.List) for total > count { - fileResp, err := d.client.GetShareSnap( - d.ShareCode, d.ReceiveCode, dir.GetID(), - driver115.QueryLimit(int(d.PageSize)), driver115.QueryOffset(count), - ) + fileResp, err := d.getShareSnapWithUA(ua, dir.GetID(), driver115.QueryLimit(int(d.PageSize)), driver115.QueryOffset(count)) if err != nil { return nil, err } @@ -77,7 +76,14 @@ func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkA if err := d.WaitLimit(ctx); err != nil { return nil, err } - downloadInfo, err := d.client.DownloadByShareCode(d.ShareCode, d.ReceiveCode, file.GetID()) + ua := "" + if args.Header != nil { + ua = args.Header.Get("User-Agent") + } + if ua == "" { + ua = base.UserAgent + } + downloadInfo, err := d.downloadByShareCodeWithUA(ua, file.GetID()) if err != nil { return nil, err } diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go index 1f9e112deef..e36a5ef8ccb 100644 --- a/drivers/115_share/utils.go +++ b/drivers/115_share/utils.go @@ -6,6 +6,7 @@ import ( "time" driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" @@ -20,6 +21,7 @@ type FileObj struct { FileName string isDir bool FileID string + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -54,7 +56,39 @@ func (f *FileObj) GetPath() string { return "" } -func transFunc(sf driver115.ShareFile) (model.Obj, error) { +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + +type shareFile struct { + FileID string `json:"fid"` + UID int `json:"uid"` + CategoryID driver115.IntString `json:"cid"` + FileName string `json:"n"` + Type string `json:"ico"` + Sha1 string `json:"sha"` + Size driver115.StringInt64 `json:"s"` + Labels []*driver115.LabelInfo `json:"fl"` + UpdateTime string `json:"t"` + IsFile int `json:"fc"` + ParentID string `json:"pid"` + ThumbURL string `json:"u"` +} + +type shareSnapResp struct { + driver115.BasicResp + Data struct { + Count int `json:"count"` + List []shareFile `json:"list"` + } `json:"data"` +} + +type downloadShareResp struct { + driver115.BasicResp + Data driver115.SharedDownloadInfo `json:"data"` +} + +func transFunc(sf shareFile) (model.Obj, error) { timeInt, err := strconv.ParseInt(sf.UpdateTime, 10, 64) if err != nil { return nil, err @@ -74,18 +108,77 @@ func transFunc(sf driver115.ShareFile) (model.Obj, error) { FileName: string(sf.FileName), isDir: isDir, FileID: fileID, + ThumbURL: sf.ThumbURL, }, nil } -var UserAgent = driver115.UA115Browser +func buildShareReferer(shareCode, receiveCode string) string { + return fmt.Sprintf("https://115cdn.com/s/%s?password=%s&", shareCode, receiveCode) +} + +func (d *Pan115Share) getShareSnapWithUA(ua, dirID string, queries ...driver115.Query) (*shareSnapResp, error) { + result := shareSnapResp{} + query := map[string]string{ + "share_code": d.ShareCode, + "receive_code": d.ReceiveCode, + "cid": dirID, + "limit": "20", + "asc": "0", + "offset": "0", + "format": "json", + } + for _, q := range queries { + q(&query) + } + + req := d.client.NewRequest(). + SetQueryParams(query). + SetHeader("referer", buildShareReferer(d.ShareCode, d.ReceiveCode)). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&result) + if ua != "" { + req = req.SetHeader("User-Agent", ua) + } + + resp, err := req.Get(driver115.ApiShareSnap) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + return &result, nil +} + +func (d *Pan115Share) downloadByShareCodeWithUA(ua, fileID string) (*driver115.SharedDownloadInfo, error) { + result := downloadShareResp{} + params := map[string]string{ + "share_code": d.ShareCode, + "receive_code": d.ReceiveCode, + "file_id": fileID, + "dl": "1", + } + + req := d.client.NewRequest(). + SetQueryParams(params). + ForceContentType("application/json"). + SetHeader("referer", buildShareReferer(d.ShareCode, d.ReceiveCode)). + SetResult(&result) + if ua != "" { + req = req.SetHeader("User-Agent", ua) + } + + resp, err := req.Get(driver115.ApiDownloadGetShareUrl) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + return &result.Data, nil +} func (d *Pan115Share) login() error { var err error opts := []driver115.Option{ - driver115.UA(UserAgent), + driver115.UA(base.UserAgent), } d.client = driver115.New(opts...) - if _, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, ""); err != nil { + if _, err = d.getShareSnapWithUA(base.UserAgent, ""); err != nil { return errors.Wrap(err, "failed to get share snap") } cr := &driver115.Credential{} diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 32c053e22ab..cf221fee6d8 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -6,6 +6,8 @@ import ( "fmt" "net/http" "net/url" + "strconv" + "strings" "sync" "time" @@ -28,7 +30,8 @@ import ( type Pan123 struct { model.Storage Addition - apiRateLimit sync.Map + apiRateLimit sync.Map + safeBoxUnlocked sync.Map } func (d *Pan123) Config() driver.Config { @@ -52,9 +55,26 @@ func (d *Pan123) Drop(ctx context.Context) error { } func (d *Pan123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if f, ok := dir.(File); ok && f.IsLock { + if err := d.unlockSafeBox(f.FileId); err != nil { + return nil, err + } + } files, err := d.getFiles(ctx, dir.GetID(), dir.GetName()) if err != nil { - return nil, err + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "safe box") || strings.Contains(err.Error(), "保险箱") { + if id, e := strconv.ParseInt(dir.GetID(), 10, 64); e == nil { + if e = d.unlockSafeBox(id); e == nil { + files, err = d.getFiles(ctx, dir.GetID(), dir.GetName()) + } else { + return nil, e + } + } + } + if err != nil { + return nil, err + } } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return src, nil diff --git a/drivers/123/meta.go b/drivers/123/meta.go index cb2cbc15ba0..6c5f013ad4a 100644 --- a/drivers/123/meta.go +++ b/drivers/123/meta.go @@ -6,8 +6,9 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + SafePassword string `json:"safe_password"` driver.RootID //OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"` //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` diff --git a/drivers/123/types.go b/drivers/123/types.go index a8682c52fc9..962e5fbdac0 100644 --- a/drivers/123/types.go +++ b/drivers/123/types.go @@ -20,6 +20,7 @@ type File struct { Etag string `json:"Etag"` S3KeyFlag string `json:"S3KeyFlag"` DownloadUrl string `json:"DownloadUrl"` + IsLock bool `json:"IsLock"` } func (f File) CreateTime() time.Time { diff --git a/drivers/123/util.go b/drivers/123/util.go index 7e5a23970c6..bca54b599f4 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -43,6 +43,7 @@ const ( S3Auth = MainApi + "/file/s3_upload_object/auth" UploadCompleteV2 = MainApi + "/file/upload_complete/v2" S3Complete = MainApi + "/file/s3_complete_multipart_upload" + SafeBoxUnlock = MainApi + "/restful/goapi/v1/file/safe_box/auth/unlockbox" //AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) @@ -161,12 +162,12 @@ func (d *Pan123) login() error { } res, err := base.RestyClient.R(). SetHeaders(map[string]string{ - "origin": "https://www.123pan.com", - "referer": "https://www.123pan.com/", - "user-agent": "Dart/2.19(dart:io)-alist", + "origin": "https://www.123pan.com", + "referer": "https://www.123pan.com/", + //"user-agent": "Dart/2.19(dart:io)-alist", "platform": "web", "app-version": "3", - //"user-agent": base.UserAgent, + "user-agent": base.UserAgent, }). SetBody(body).Post(SignIn) if err != nil { @@ -202,7 +203,7 @@ do: "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "authorization": "Bearer " + d.AccessToken, - "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) alist-client", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", "platform": "web", "app-version": "3", //"user-agent": base.UserAgent, @@ -238,6 +239,22 @@ do: return body, nil } +func (d *Pan123) unlockSafeBox(fileId int64) error { + if _, ok := d.safeBoxUnlocked.Load(fileId); ok { + return nil + } + data := base.Json{"password": d.SafePassword} + url := fmt.Sprintf("%s?fileId=%d", SafeBoxUnlock, fileId) + _, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + if err != nil { + return err + } + d.safeBoxUnlocked.Store(fileId, true) + return nil +} + func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([]File, error) { page := 1 total := 0 @@ -267,6 +284,15 @@ func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([] req.SetQueryParams(query) }, &resp) if err != nil { + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "safe box") || strings.Contains(err.Error(), "保险箱") { + if fid, e := strconv.ParseInt(parentId, 10, 64); e == nil { + if e = d.unlockSafeBox(fid); e == nil { + return d.getFiles(ctx, parentId, name) + } + return nil, e + } + } return nil, err } log.Debug(string(_res)) diff --git a/drivers/123_open/api.go b/drivers/123_open/api.go new file mode 100644 index 00000000000..1d2a6f164ff --- /dev/null +++ b/drivers/123_open/api.go @@ -0,0 +1,191 @@ +package _123Open + +import ( + "fmt" + "github.com/go-resty/resty/v2" + "net/http" +) + +const ( + // baseurl + ApiBaseURL = "https://open-api.123pan.com" + + // auth + ApiToken = "/api/v1/access_token" + + // file list + ApiFileList = "/api/v2/file/list" + + // direct link + ApiGetDirectLink = "/api/v1/direct-link/url" + + // mkdir + ApiMakeDir = "/upload/v1/file/mkdir" + + // remove + ApiRemove = "/api/v1/file/trash" + + // upload + ApiUploadDomainURL = "/upload/v2/file/domain" + ApiSingleUploadURL = "/upload/v2/file/single/create" + ApiCreateUploadURL = "/upload/v2/file/create" + ApiUploadSliceURL = "/upload/v2/file/slice" + ApiUploadCompleteURL = "/upload/v2/file/upload_complete" + + // move + ApiMove = "/api/v1/file/move" + + // rename + ApiRename = "/api/v1/file/name" +) + +type Response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type TokenResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data TokenData `json:"data"` +} + +type TokenData struct { + AccessToken string `json:"accessToken"` + ExpiredAt string `json:"expiredAt"` +} + +type FileListResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data FileListData `json:"data"` +} + +type FileListData struct { + LastFileId int64 `json:"lastFileId"` + FileList []File `json:"fileList"` +} + +type DirectLinkResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data DirectLinkData `json:"data"` +} + +type DirectLinkData struct { + URL string `json:"url"` +} + +type MakeDirRequest struct { + Name string `json:"name"` + ParentID int64 `json:"parentID"` +} + +type MakeDirResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data MakeDirData `json:"data"` +} + +type MakeDirData struct { + DirID int64 `json:"dirID"` +} + +type RemoveRequest struct { + FileIDs []int64 `json:"fileIDs"` +} + +type UploadCreateResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadCreateData `json:"data"` +} + +type UploadCreateData struct { + FileID int64 `json:"fileId"` + Reuse bool `json:"reuse"` + PreuploadID string `json:"preuploadId"` + SliceSize int64 `json:"sliceSize"` + Servers []string `json:"servers"` +} + +type UploadUrlResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadUrlData `json:"data"` +} + +type UploadUrlData struct { + PresignedURL string `json:"presignedUrl"` +} + +type UploadCompleteResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadCompleteData `json:"data"` +} + +type UploadCompleteData struct { + FileID int `json:"fileID"` + Completed bool `json:"completed"` +} + +func (d *Open123) Request(endpoint string, method string, setup func(*resty.Request), result any) (*resty.Response, error) { + client := resty.New() + token, err := d.tm.getToken() + if err != nil { + return nil, err + } + + req := client.R(). + SetHeader("Authorization", "Bearer "+token). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", "application/json"). + SetResult(result) + + if setup != nil { + setup(req) + } + + switch method { + case http.MethodGet: + return req.Get(ApiBaseURL + endpoint) + case http.MethodPost: + return req.Post(ApiBaseURL + endpoint) + case http.MethodPut: + return req.Put(ApiBaseURL + endpoint) + default: + return nil, fmt.Errorf("unsupported method: %s", method) + } +} + +func (d *Open123) RequestTo(fullURL string, method string, setup func(*resty.Request), result any) (*resty.Response, error) { + client := resty.New() + + token, err := d.tm.getToken() + if err != nil { + return nil, err + } + + req := client.R(). + SetHeader("Authorization", "Bearer "+token). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", "application/json"). + SetResult(result) + + if setup != nil { + setup(req) + } + + switch method { + case http.MethodGet: + return req.Get(fullURL) + case http.MethodPost: + return req.Post(fullURL) + case http.MethodPut: + return req.Put(fullURL) + default: + return nil, fmt.Errorf("unsupported method: %s", method) + } +} diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go new file mode 100644 index 00000000000..ace86bb981a --- /dev/null +++ b/drivers/123_open/driver.go @@ -0,0 +1,294 @@ +package _123Open + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "net/http" + "strconv" + "time" +) + +type Open123 struct { + model.Storage + Addition + + UploadThread int + tm *tokenManager +} + +func (d *Open123) Config() driver.Config { + return config +} + +func (d *Open123) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Open123) Init(ctx context.Context) error { + d.tm = newTokenManager(d.ClientID, d.ClientSecret) + + if _, err := d.tm.getToken(); err != nil { + return fmt.Errorf("token 初始化失败: %w", err) + } + + return nil +} + +func (d *Open123) Drop(ctx context.Context) error { + return nil +} + +func (d *Open123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + parentFileId, err := strconv.ParseInt(dir.GetID(), 10, 64) + if err != nil { + return nil, err + } + + fileLastId := int64(0) + var results []File + + for fileLastId != -1 { + files, err := d.getFiles(parentFileId, 100, fileLastId) + if err != nil { + return nil, err + } + for _, f := range files.Data.FileList { + if f.Trashed == 0 { + results = append(results, f) + } + } + fileLastId = files.Data.LastFileId + } + + objs := make([]model.Obj, 0, len(results)) + for _, f := range results { + objs = append(objs, f) + } + return objs, nil +} + +func (d *Open123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.LinkIsDir + } + + fileID := file.GetID() + + var result DirectLinkResp + url := fmt.Sprintf("%s?fileID=%s", ApiGetDirectLink, fileID) + _, err := d.Request(url, http.MethodGet, nil, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("get link failed: %s", result.Message) + } + + linkURL := result.Data.URL + if d.PrivateKey != "" { + if d.UID == 0 { + return nil, fmt.Errorf("uid is required when private key is set") + } + duration := time.Duration(d.ValidDuration) + if duration <= 0 { + duration = 30 + } + signedURL, err := SignURL(linkURL, d.PrivateKey, d.UID, duration*time.Minute) + if err != nil { + return nil, err + } + linkURL = signedURL + } + + return &model.Link{ + URL: linkURL, + }, nil +} + +func (d *Open123) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + parentID, err := strconv.ParseInt(parentDir.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid parent ID: %w", err) + } + + var result MakeDirResp + reqBody := MakeDirRequest{ + Name: dirName, + ParentID: parentID, + } + + _, err = d.Request(ApiMakeDir, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("mkdir failed: %s", result.Message) + } + + newDir := File{ + FileId: result.Data.DirID, + FileName: dirName, + Type: 1, + ParentFileId: int(parentID), + Size: 0, + Trashed: 0, + } + return newDir, nil +} + +func (d *Open123) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcID, err := strconv.ParseInt(srcObj.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid src file ID: %w", err) + } + dstID, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid dest dir ID: %w", err) + } + + var result Response[any] + reqBody := map[string]interface{}{ + "fileIDs": []int64{srcID}, + "toParentFileID": dstID, + } + + _, err = d.Request(ApiMove, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("move failed: %s", result.Message) + } + + files, err := d.getFiles(dstID, 100, 0) + if err != nil { + return nil, fmt.Errorf("move succeed but failed to get target dir: %w", err) + } + for _, f := range files.Data.FileList { + if f.FileId == srcID { + return f, nil + } + } + return nil, fmt.Errorf("move succeed but file not found in target dir") +} + +func (d *Open123) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + srcID, err := strconv.ParseInt(srcObj.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid file ID: %w", err) + } + + var result Response[any] + reqBody := map[string]interface{}{ + "fileId": srcID, + "fileName": newName, + } + + _, err = d.Request(ApiRename, http.MethodPut, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("rename failed: %s", result.Message) + } + + parentID := 0 + if file, ok := srcObj.(File); ok { + parentID = file.ParentFileId + } + files, err := d.getFiles(int64(parentID), 100, 0) + if err != nil { + return nil, fmt.Errorf("rename succeed but failed to get parent dir: %w", err) + } + for _, f := range files.Data.FileList { + if f.FileId == srcID { + return f, nil + } + } + return nil, fmt.Errorf("rename succeed but file not found in parent dir") +} + +func (d *Open123) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { + idStr := obj.GetID() + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + return fmt.Errorf("invalid file ID: %w", err) + } + + var result Response[any] + reqBody := RemoveRequest{ + FileIDs: []int64{id}, + } + + _, err = d.Request(ApiRemove, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return err + } + if result.Code != 0 { + return fmt.Errorf("remove failed: %s", result.Message) + } + + return nil +} + +func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + etag := file.GetHash().GetHash(utils.MD5) + + if len(etag) < utils.MD5.Width { + up = model.UpdateProgressWithRange(up, 50, 100) + _, etag, err = stream.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return err + } + } + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + if err != nil { + return err + } + if createResp.Data.Reuse { + return nil + } + + return d.Upload(ctx, file, parentFileId, createResp, up) +} + +func (d *Open123) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotSupport +} + +func (d *Open123) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Open123) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotSupport +} + +func (d *Open123) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +//func (d *Open123) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Open123)(nil) diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go new file mode 100644 index 00000000000..d0b117aa7c0 --- /dev/null +++ b/drivers/123_open/meta.go @@ -0,0 +1,36 @@ +package _123Open + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + + ClientID string `json:"client_id" required:"true" label:"Client ID"` + ClientSecret string `json:"client_secret" required:"true" label:"Client Secret"` + PrivateKey string `json:"private_key"` + UID uint64 `json:"uid" type:"number"` + ValidDuration int64 `json:"valid_duration" type:"number" default:"30" help:"minutes"` +} + +var config = driver.Config{ + Name: "123 Open", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Open123{} + }) +} diff --git a/drivers/123_open/sign.go b/drivers/123_open/sign.go new file mode 100644 index 00000000000..549d7ad8fdf --- /dev/null +++ b/drivers/123_open/sign.go @@ -0,0 +1,27 @@ +package _123Open + +import ( + "crypto/md5" + "fmt" + "math/rand" + "net/url" + "time" +) + +func SignURL(originURL, privateKey string, uid uint64, validDuration time.Duration) (string, error) { + if privateKey == "" { + return originURL, nil + } + parsed, err := url.Parse(originURL) + if err != nil { + return "", err + } + ts := time.Now().Add(validDuration).Unix() + randInt := rand.Int() + signature := fmt.Sprintf("%d-%d-%d-%x", ts, randInt, uid, md5.Sum([]byte(fmt.Sprintf("%s-%d-%d-%d-%s", + parsed.Path, ts, randInt, uid, privateKey)))) + query := parsed.Query() + query.Add("auth_key", signature) + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} diff --git a/drivers/123_open/token.go b/drivers/123_open/token.go new file mode 100644 index 00000000000..435c0b0de67 --- /dev/null +++ b/drivers/123_open/token.go @@ -0,0 +1,85 @@ +package _123Open + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" +) + +const tokenURL = ApiBaseURL + ApiToken + +type tokenManager struct { + clientID string + clientSecret string + + mu sync.Mutex + accessToken string + expireTime time.Time +} + +func newTokenManager(clientID, clientSecret string) *tokenManager { + return &tokenManager{ + clientID: clientID, + clientSecret: clientSecret, + } +} + +func (tm *tokenManager) getToken() (string, error) { + tm.mu.Lock() + defer tm.mu.Unlock() + + if tm.accessToken != "" && time.Now().Before(tm.expireTime.Add(-5*time.Minute)) { + return tm.accessToken, nil + } + + reqBody := map[string]string{ + "clientID": tm.clientID, + "clientSecret": tm.clientSecret, + } + body, _ := json.Marshal(reqBody) + req, err := http.NewRequest("POST", tokenURL, bytes.NewBuffer(body)) + if err != nil { + return "", err + } + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var result TokenResp + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", err + } + + if result.Code != 0 { + return "", fmt.Errorf("get token failed: %s", result.Message) + } + + tm.accessToken = result.Data.AccessToken + expireAt, err := time.Parse(time.RFC3339, result.Data.ExpiredAt) + if err != nil { + return "", fmt.Errorf("parse expire time failed: %w", err) + } + tm.expireTime = expireAt + + return tm.accessToken, nil +} + +func (tm *tokenManager) buildHeaders() (http.Header, error) { + token, err := tm.getToken() + if err != nil { + return nil, err + } + header := http.Header{} + header.Set("Authorization", "Bearer "+token) + header.Set("Platform", "open_platform") + header.Set("Content-Type", "application/json") + return header, nil +} diff --git a/drivers/123_open/types.go b/drivers/123_open/types.go new file mode 100644 index 00000000000..afece279e60 --- /dev/null +++ b/drivers/123_open/types.go @@ -0,0 +1,70 @@ +package _123Open + +import ( + "fmt" + "github.com/alist-org/alist/v3/pkg/utils" + "time" +) + +type File struct { + FileName string `json:"filename"` + Size int64 `json:"size"` + CreateAt string `json:"createAt"` + UpdateAt string `json:"updateAt"` + FileId int64 `json:"fileId"` + Type int `json:"type"` + Etag string `json:"etag"` + S3KeyFlag string `json:"s3KeyFlag"` + ParentFileId int `json:"parentFileId"` + Category int `json:"category"` + Status int `json:"status"` + Trashed int `json:"trashed"` +} + +func (f File) GetID() string { + return fmt.Sprint(f.FileId) +} + +func (f File) GetName() string { + return f.FileName +} + +func (f File) GetSize() int64 { + return f.Size +} + +func (f File) IsDir() bool { + return f.Type == 1 +} + +func (f File) GetModified() string { + return f.UpdateAt +} + +func (f File) GetThumb() string { + return "" +} + +func (f File) ModTime() time.Time { + t, err := time.Parse("2006-01-02 15:04:05", f.UpdateAt) + if err != nil { + return time.Time{} + } + return t +} + +func (f File) CreateTime() time.Time { + t, err := time.Parse("2006-01-02 15:04:05", f.CreateAt) + if err != nil { + return time.Time{} + } + return t +} + +func (f File) GetHash() utils.HashInfo { + return utils.NewHashInfo(utils.MD5, f.Etag) +} + +func (f File) GetPath() string { + return "" +} diff --git a/drivers/123_open/upload.go b/drivers/123_open/upload.go new file mode 100644 index 00000000000..76e8ead46f8 --- /dev/null +++ b/drivers/123_open/upload.go @@ -0,0 +1,282 @@ +package _123Open + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "golang.org/x/sync/errgroup" + "io" + "mime/multipart" + "net/http" + "runtime" + "strconv" + "time" +) + +func (d *Open123) create(parentFileID int64, filename, etag string, size int64, duplicate int, containDir bool) (*UploadCreateResp, error) { + var resp UploadCreateResp + + _, err := d.Request(ApiCreateUploadURL, http.MethodPost, func(req *resty.Request) { + body := base.Json{ + "parentFileID": parentFileID, + "filename": filename, + "etag": etag, + "size": size, + } + if duplicate > 0 { + body["duplicate"] = duplicate + } + if containDir { + body["containDir"] = true + } + req.SetBody(body) + }, &resp) + + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Open123) GetUploadDomains() ([]string, error) { + var resp struct { + Code int `json:"code"` + Message string `json:"message"` + Data []string `json:"data"` + } + + _, err := d.Request(ApiUploadDomainURL, http.MethodGet, nil, &resp) + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("get upload domain failed: %s", resp.Message) + } + return resp.Data, nil +} + +func (d *Open123) UploadSingle(ctx context.Context, createResp *UploadCreateResp, file model.FileStreamer, parentID int64) error { + domain := createResp.Data.Servers[0] + + etag := file.GetHash().GetHash(utils.MD5) + if len(etag) < utils.MD5.Width { + _, _, err := stream.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return err + } + } + + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) + if err != nil { + return err + } + reader = driver.NewLimitedUploadStream(ctx, reader) + + var b bytes.Buffer + mw := multipart.NewWriter(&b) + mw.WriteField("parentFileID", fmt.Sprint(parentID)) + mw.WriteField("filename", file.GetName()) + mw.WriteField("etag", etag) + mw.WriteField("size", fmt.Sprint(file.GetSize())) + fw, _ := mw.CreateFormFile("file", file.GetName()) + _, err = io.Copy(fw, reader) + mw.Close() + + req, err := http.NewRequestWithContext(ctx, "POST", domain+ApiSingleUploadURL, &b) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+d.tm.accessToken) + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", mw.FormDataContentType()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + var result struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + FileID int64 `json:"fileID"` + Completed bool `json:"completed"` + } `json:"data"` + } + body, _ := io.ReadAll(resp.Body) + if err := json.Unmarshal(body, &result); err != nil { + return fmt.Errorf("unmarshal response error: %v, body: %s", err, string(body)) + } + if result.Code != 0 { + return fmt.Errorf("upload failed: %s", result.Message) + } + if !result.Data.Completed || result.Data.FileID == 0 { + return fmt.Errorf("upload incomplete or missing fileID") + } + return nil +} + +func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, parentID int64, createResp *UploadCreateResp, up driver.UpdateProgress) error { + if cacher, ok := file.(interface{ CacheFullInTempFile() (model.File, error) }); ok { + if _, err := cacher.CacheFullInTempFile(); err != nil { + return err + } + } + + size := file.GetSize() + chunkSize := createResp.Data.SliceSize + uploadNums := (size + chunkSize - 1) / chunkSize + uploadDomain := createResp.Data.Servers[0] + + if d.UploadThread <= 0 { + cpuCores := runtime.NumCPU() + threads := cpuCores * 2 + if threads < 4 { + threads = 4 + } + if threads > 16 { + threads = 16 + } + d.UploadThread = threads + fmt.Printf("[Upload] Auto set upload concurrency: %d (CPU cores=%d)\n", d.UploadThread, cpuCores) + } + + fmt.Printf("[Upload] File size: %d bytes, chunk size: %d bytes, total slices: %d, concurrency: %d\n", + size, chunkSize, uploadNums, d.UploadThread) + + if size <= 1<<30 { + return d.UploadSingle(ctx, createResp, file, parentID) + } + + if createResp.Data.Reuse { + up(100) + return nil + } + + client := resty.New() + semaphore := make(chan struct{}, d.UploadThread) + threadG, _ := errgroup.WithContext(ctx) + + var progressArr = make([]int64, uploadNums) + + for partIndex := int64(0); partIndex < uploadNums; partIndex++ { + partIndex := partIndex + semaphore <- struct{}{} + + threadG.Go(func() error { + defer func() { <-semaphore }() + offset := partIndex * chunkSize + length := min(chunkSize, size-offset) + partNumber := partIndex + 1 + + fmt.Printf("[Slice %d] Starting read from offset %d, length %d\n", partNumber, offset, length) + reader, err := file.RangeRead(http_range.Range{Start: offset, Length: length}) + if err != nil { + return fmt.Errorf("[Slice %d] RangeRead error: %v", partNumber, err) + } + + buf := make([]byte, length) + n, err := io.ReadFull(reader, buf) + if err != nil && err != io.EOF { + return fmt.Errorf("[Slice %d] Read error: %v", partNumber, err) + } + buf = buf[:n] + hash := md5.Sum(buf) + sliceMD5Str := hex.EncodeToString(hash[:]) + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + writer.WriteField("preuploadID", createResp.Data.PreuploadID) + writer.WriteField("sliceNo", strconv.FormatInt(partNumber, 10)) + writer.WriteField("sliceMD5", sliceMD5Str) + partName := fmt.Sprintf("%s.part%d", file.GetName(), partNumber) + fw, _ := writer.CreateFormFile("slice", partName) + fw.Write(buf) + writer.Close() + + resp, err := client.R(). + SetHeader("Authorization", "Bearer "+d.tm.accessToken). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", writer.FormDataContentType()). + SetBody(body.Bytes()). + Post(uploadDomain + ApiUploadSliceURL) + + if err != nil { + return fmt.Errorf("[Slice %d] Upload HTTP error: %v", partNumber, err) + } + if resp.StatusCode() != 200 { + return fmt.Errorf("[Slice %d] Upload failed with status: %s, resp: %s", partNumber, resp.Status(), resp.String()) + } + + progressArr[partIndex] = length + var totalUploaded int64 = 0 + for _, v := range progressArr { + totalUploaded += v + } + if up != nil { + percent := float64(totalUploaded) / float64(size) * 100 + up(percent) + } + + fmt.Printf("[Slice %d] MD5: %s\n", partNumber, sliceMD5Str) + fmt.Printf("[Slice %d] Upload finished\n", partNumber) + return nil + }) + } + + if err := threadG.Wait(); err != nil { + return err + } + + var completeResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Completed bool `json:"completed"` + FileID int64 `json:"fileID"` + } `json:"data"` + } + + for { + reqBody := fmt.Sprintf(`{"preuploadID":"%s"}`, createResp.Data.PreuploadID) + req, err := http.NewRequestWithContext(ctx, "POST", uploadDomain+ApiUploadCompleteURL, bytes.NewBufferString(reqBody)) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+d.tm.accessToken) + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + if err := json.Unmarshal(body, &completeResp); err != nil { + return fmt.Errorf("completion response unmarshal error: %v, body: %s", err, string(body)) + } + if completeResp.Code != 0 { + return fmt.Errorf("completion API returned error code %d: %s", completeResp.Code, completeResp.Message) + } + if completeResp.Data.Completed && completeResp.Data.FileID != 0 { + fmt.Printf("[Upload] Upload completed successfully. FileID: %d\n", completeResp.Data.FileID) + break + } + time.Sleep(time.Second) + } + up(100) + return nil +} diff --git a/drivers/123_open/util.go b/drivers/123_open/util.go new file mode 100644 index 00000000000..429a5e5dda8 --- /dev/null +++ b/drivers/123_open/util.go @@ -0,0 +1,20 @@ +package _123Open + +import ( + "fmt" + "net/http" +) + +func (d *Open123) getFiles(parentFileId int64, limit int, lastFileId int64) (*FileListResp, error) { + var result FileListResp + url := fmt.Sprintf("%s?parentFileId=%d&limit=%d&lastFileId=%d", ApiFileList, parentFileId, limit, lastFileId) + + _, err := d.Request(url, http.MethodGet, nil, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("list error: %s", result.Message) + } + return &result, nil +} diff --git a/drivers/139/driver.go b/drivers/139/driver.go index a57609bc550..10d9d3e9e03 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -47,28 +47,12 @@ func (d *Yun139) Init(ctx context.Context) error { if err != nil { return err } - - // Query Route Policy - var resp QueryRoutePolicyResp - _, err = d.requestRoute(base.Json{ - "userInfo": base.Json{ - "userType": 1, - "accountType": 1, - "accountName": d.Account}, - "modAddrType": 1, - }, &resp) - if err != nil { - return err - } - for _, policyItem := range resp.Data.RoutePolicyList { - if policyItem.ModName == "personal" { - d.PersonalCloudHost = policyItem.HttpsUrl - break + if d.Addition.Type == MetaPersonalNew { + err = d.ensurePersonalCloudHost() + if err != nil { + return err } } - if len(d.PersonalCloudHost) == 0 { - return fmt.Errorf("PersonalCloudHost is empty") - } d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { @@ -92,6 +76,10 @@ func (d *Yun139) Init(ctx context.Context) error { d.RootFolderID = d.CloudID } case MetaFamily: + case "share": + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = "root" + } default: return errs.NotImplement } @@ -116,6 +104,7 @@ func (d *Yun139) Drop(ctx context.Context) error { } func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + log.Infof("[139Share-Debug] List called! Type: %s, DirID: %s", d.Addition.Type, dir.GetID()) switch d.Addition.Type { case MetaPersonalNew: return d.personalGetFiles(dir.GetID()) @@ -125,6 +114,8 @@ func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( return d.familyGetFiles(dir.GetID()) case MetaGroup: return d.groupGetFiles(dir.GetID()) + case "share": + return d.shareGetFiles(dir.GetID()) default: return nil, errs.NotImplement } @@ -142,6 +133,8 @@ func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) url, err = d.familyGetLink(file.GetID(), file.GetPath()) case MetaGroup: url, err = d.groupGetLink(file.GetID(), file.GetPath()) + case "share": + return d.shareGetLink(file.GetID()) default: return nil, errs.NotImplement } diff --git a/drivers/139/meta.go b/drivers/139/meta.go index c02b1347587..ea90df86cf9 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -6,14 +6,14 @@ import ( ) type Addition struct { - //Account string `json:"account" required:"true"` Authorization string `json:"authorization" type:"text" required:"true"` driver.RootID - Type string `json:"type" type:"select" options:"personal_new,family,group,personal" default:"personal_new"` + Type string `json:"type" type:"select" options:"personal_new,family,group,personal,share" default:"personal_new"` CloudID string `json:"cloud_id"` - CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` - ReportRealSize bool `json:"report_real_size" type:"bool" default:"true" help:"Enable to report the real file size during upload"` - UseLargeThumbnail bool `json:"use_large_thumbnail" type:"bool" default:"false" help:"Enable to use large thumbnail for images"` + LinkID string `json:"link_id"` + CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0"` + ReportRealSize bool `json:"report_real_size" type:"bool" default:"true"` + UseLargeThumbnail bool `json:"use_large_thumbnail" type:"bool" default:"false"` } var config = driver.Config{ diff --git a/drivers/139/types.go b/drivers/139/types.go index d5f025a1672..03f88aa7aba 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -312,3 +312,41 @@ type RefreshTokenResp struct { AccessToken string `xml:"accessToken"` Desc string `xml:"desc"` } + +type ShareCatalog struct { + CaID string `json:"caID"` + CaName string `json:"caName"` + UdTime string `json:"udTime"` +} + +type ShareContent struct { + CoID string `json:"coID"` + CoName string `json:"coName"` + CoSize int64 `json:"coSize"` + CoType int `json:"coType"` + UdTime string `json:"udTime"` + CoSuffix string `json:"coSuffix"` +} + +type ShareListResp struct { + Data struct { + CaLst []ShareCatalog `json:"caLst"` + CoLst []ShareContent `json:"coLst"` + } `json:"data"` +} + +type ShareLinkResp struct { + DownloadURL string `json:"downloadURL"` +} +type ShareContentInfo struct { + ContentName string `json:"contentName"` + ContentSize int64 `json:"contentSize"` + PresentURL string `json:"presentURL"` // HLS 播放地址 + DownloadURL string `json:"cdnDownLoadUrl"` // 真正的下载直链 +} + +type ShareContentInfoResp struct { + Data struct { + ContentInfo ShareContentInfo `json:"contentInfo"` + } `json:"data"` +} \ No newline at end of file diff --git a/drivers/139/util.go b/drivers/139/util.go index 5adc39b4116..3823569ad70 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -1,9 +1,16 @@ package _139 import ( + "bytes" + "compress/gzip" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" "encoding/base64" "errors" "fmt" + "io" "net/http" "net/url" "path" @@ -12,11 +19,14 @@ import ( "strings" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/gin-gonic/gin" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" @@ -215,6 +225,46 @@ func (d *Yun139) requestRoute(data interface{}, resp interface{}) ([]byte, error return res.Body(), nil } +func (d *Yun139) ensurePersonalCloudHost() error { + if d.ref != nil { + return d.ref.ensurePersonalCloudHost() + } + if d.PersonalCloudHost != "" { + return nil + } + if len(d.Authorization) == 0 { + return fmt.Errorf("authorization is empty") + } + if d.Account == "" { + if err := d.refreshToken(); err != nil { + return err + } + } + + var resp QueryRoutePolicyResp + _, err := d.requestRoute(base.Json{ + "userInfo": base.Json{ + "userType": 1, + "accountType": 1, + "accountName": d.Account, + }, + "modAddrType": 1, + }, &resp) + if err != nil { + return err + } + for _, policyItem := range resp.Data.RoutePolicyList { + if policyItem.ModName == "personal" && policyItem.HttpsUrl != "" { + d.PersonalCloudHost = strings.TrimRight(policyItem.HttpsUrl, "/") + break + } + } + if d.PersonalCloudHost == "" { + return fmt.Errorf("personal cloud host is empty") + } + return nil +} + func (d *Yun139) post(pathname string, data interface{}, resp interface{}) ([]byte, error) { return d.request(pathname, http.MethodPost, func(req *resty.Request) { req.SetBody(data) @@ -449,6 +499,9 @@ func unicode(str string) string { } func (d *Yun139) personalRequest(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if err := d.ensurePersonalCloudHost(); err != nil { + return nil, err + } url := d.getPersonalCloudHost() + pathname req := base.RestyClient.R() randStr := random.String(16) @@ -623,3 +676,335 @@ func (d *Yun139) getPersonalCloudHost() string { } return d.PersonalCloudHost } + +func (d *Yun139) sharePost(pathname string, data interface{}, resp interface{}) ([]byte, error) { + crypto := NewYunCrypto() + encryptedBody, err := crypto.Encrypt(data) + if err != nil { + return nil, err + } + + url := "https://share-kd-njs.yun.139.com" + pathname + req := base.RestyClient.R() + + auth := d.getAuthorization() + if !strings.HasPrefix(auth, "Basic ") { + auth = "Basic " + auth + } + // randStr := random.String(16) + // ts := time.Now().Format("2006-01-02 15:04:05") + // body, err := utils.Json.Marshal(req.Body) + // if err != nil { + // return nil, err + // } + // sign := calSign(string(body), ts, randStr) + // svcType := "1" + // if d.isFamily() { + // svcType = "2" + // } + req.SetHeaders(map[string]string{ + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0", + "Accept": "application/json, text/plain, */*", + "Content-Type": "application/json;charset=UTF-8", + "Authorization": auth, + "X-Deviceinfo": "||9|12.27.0|firefox|140.0|12b780037221ab547c682223327dc9cd||linux unknow|1920X526|zh-CN|||", + "hcy-cool-flag": "1", + "CMS-DEVICE": "default", + "x-m4c-caller": "PC", + "X-Yun-Api-Version": "v1", + "Origin": "https://yun.139.com", + "Referer": "https://yun.139.com/", + }) + req.SetBody(encryptedBody) + + res, err := req.Post(url) + if err != nil { + return nil, err + } + + decryptedText, err := crypto.Decrypt(res.String()) + if err != nil { + log.Errorf("[139Share] Decryption failed, raw response: %s", res.String()) + return nil, fmt.Errorf("decryption failed: %v, raw: %s", err, res.String()) + } + + if resp != nil { + err = utils.Json.Unmarshal([]byte(decryptedText), resp) + if err != nil { + return nil, err + } + } + return []byte(decryptedText), nil +} + +func (d *Yun139) shareGetFiles(pCaID string) ([]model.Obj, error) { + if pCaID == "" { + pCaID = "root" + } + data := base.Json{ + "getOutLinkInfoReq": base.Json{ + "account": d.getAccount(), + "linkID": d.LinkID, + "pCaID": pCaID, + }, + } + var resp ShareListResp + _, err := d.sharePost("/yun-share/richlifeApp/devapp/IOutLink/getOutLinkInfoV6", data, &resp) + if err != nil { + return nil, err + } + files := make([]model.Obj, 0) + // 直接从 Data 中读取 CaLst + for _, catalog := range resp.Data.CaLst { + modTime, _ := time.ParseInLocation("20060102150405", catalog.UdTime, utils.CNLoc) + f := model.Object{ + ID: catalog.CaID, + Name: catalog.CaName, + Modified: modTime, + IsFolder: true, + } + files = append(files, &f) + } + for _, content := range resp.Data.CoLst { + name := content.CoName + size := content.CoSize + // Force .m3u8 suffix for videos and declare 1MB size for padding logic + if content.CoType == 3 || strings.HasSuffix(strings.ToLower(name), ".mp4") { + if !strings.HasSuffix(name, ".m3u8") { + name += ".m3u8" + } + size = 1024 * 1024 // Key: declare 1MB to match RangeReadCloser padding + } + modTime, _ := time.ParseInLocation("20060102150405", content.UdTime, utils.CNLoc) + f := model.Object{ + ID: content.CoID, + Name: name, + Size: size, + Modified: modTime, + } + files = append(files, &f) + } + + return files, nil +} + + + +type YunCrypto struct { + Key []byte + BlockSize int +} + +func NewYunCrypto() *YunCrypto { + return &YunCrypto{ + Key: []byte("PVGDwmcvfs1uV3d1"), + BlockSize: aes.BlockSize, + } +} + +func (y *YunCrypto) PKCS7Padding(ciphertext []byte, blockSize int) []byte { + padding := blockSize - len(ciphertext)%blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + return append(ciphertext, padtext...) +} + +func (y *YunCrypto) PKCS7UnPadding(origData []byte) ([]byte, error) { + length := len(origData) + if length == 0 { + return nil, errors.New("data is empty") + } + unpadding := int(origData[length-1]) + if length < unpadding { + return nil, errors.New("unpadding error") + } + return origData[:(length - unpadding)], nil +} + +func (y *YunCrypto) Encrypt(data interface{}) (string, error) { + jsonData, err := utils.Json.Marshal(data) + if err != nil { + return "", err + } + iv := make([]byte, y.BlockSize) + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return "", err + } + block, err := aes.NewCipher(y.Key) + if err != nil { + return "", err + } + content := y.PKCS7Padding(jsonData, y.BlockSize) + ciphertext := make([]byte, len(content)) + mode := cipher.NewCBCEncrypter(block, iv) + mode.CryptBlocks(ciphertext, content) + result := append(iv, ciphertext...) + return base64.StdEncoding.EncodeToString(result), nil +} + +func (y *YunCrypto) Decrypt(b64Data string) (string, error) { + b64Data = strings.Join(strings.Fields(b64Data), "") + raw, err := base64.StdEncoding.DecodeString(b64Data) + if err != nil { + return "", err + } + if len(raw) < y.BlockSize { + return "", errors.New("data too short") + } + iv := raw[:y.BlockSize] + ciphertext := raw[y.BlockSize:] + block, err := aes.NewCipher(y.Key) + if err != nil { + return "", err + } + decrypted := make([]byte, len(ciphertext)) + mode := cipher.NewCBCDecrypter(block, iv) + mode.CryptBlocks(decrypted, ciphertext) + if len(decrypted) > 2 && decrypted[0] == 0x1f && decrypted[1] == 0x8b { + reader, err := gzip.NewReader(bytes.NewReader(decrypted)) + if err == nil { + defer reader.Close() + unzipped, err := io.ReadAll(reader) + if err == nil { + return string(unzipped), nil + } + } + } + unpadded, err := y.PKCS7UnPadding(decrypted) + if err != nil { + return strings.TrimSpace(string(decrypted)), nil + } + return string(unpadded), nil +} + +func (d *Yun139) rewriteM3U8(masterURL string) (string, error) { + client := resty.New().SetTimeout(10 * time.Second) + headers := map[string]string{ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Referer": "https://yun.139.com/", + } + + // 1. Get Master M3U8 + resp, err := client.R().SetHeaders(headers).Get(masterURL) + if err != nil { + return "", err + } + masterContent := resp.String() + + // 2. Find sub-playlist path + var subRelPath string + lines := strings.Split(masterContent, "\n") + for i, line := range lines { + if strings.Contains(line, "RESOLUTION=") { + if i+1 < len(lines) { + subRelPath = strings.TrimSpace(lines[i+1]) + if strings.Contains(line, "1920x1080") { + break + } + } + } + } + if subRelPath == "" { + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if line != "" && !strings.HasPrefix(line, "#") { + subRelPath = line + break + } + } + } + if subRelPath == "" { + return "", fmt.Errorf("sub playlist not found in master m3u8") + } + + // 3. Get sub-playlist content + base, _ := url.Parse(masterURL) + ref, _ := url.Parse(subRelPath) + subURL := base.ResolveReference(ref).String() + + resp, err = client.R().SetHeaders(headers).Get(subURL) + if err != nil { + return "", err + } + subContent := resp.String() + + // 4. Resolve relative TS paths to absolute URLs + subBase, _ := url.Parse(subURL) + subLines := strings.Split(subContent, "\n") + var finalLines []string + for _, line := range subLines { + cleanLine := strings.TrimSpace(line) + if cleanLine != "" && !strings.HasPrefix(cleanLine, "#") { + if !strings.HasPrefix(cleanLine, "http") { + tsRef, _ := url.Parse(cleanLine) + finalLines = append(finalLines, subBase.ResolveReference(tsRef).String()) + } else { + finalLines = append(finalLines, cleanLine) + } + } else { + finalLines = append(finalLines, line) + } + } + + finalM3U8 := strings.Join(finalLines, "\n") + + return finalM3U8, nil +} + +func (d *Yun139) Proxy(c *gin.Context, obj model.Obj) error { + return nil +} + +func (d *Yun139) shareGetLink(coID string) (*model.Link, error) { + data := base.Json{ + "getContentInfoFromOutLinkReq": base.Json{ + "contentId": coID, + "linkID": d.LinkID, + "account": d.getAccount(), + }, + } + var resp ShareContentInfoResp + _, err := d.sharePost("/yun-share/richlifeApp/devapp/IOutLink/getContentInfoFromOutLink", data, &resp) + if err != nil { + return nil, err + } + + res := resp.Data.ContentInfo + if res.PresentURL != "" { + m3u8Content, err := d.rewriteM3U8(res.PresentURL) + if err != nil { + return nil, err + } + + // Core logic: pad to 1MB to ensure compatibility with AList's size validation + targetSize := int64(1024 * 1024) + contentBytes := []byte(m3u8Content) + if int64(len(contentBytes)) < targetSize { + padding := bytes.Repeat([]byte(" "), int(targetSize-int64(len(contentBytes)))) + contentBytes = append(contentBytes, padding...) + } else { + // Truncate if M3U8 exceeds 1MB (extremely rare) + contentBytes = contentBytes[:targetSize] + } + + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{ + RangeReader: func(ctx context.Context, range_ http_range.Range) (io.ReadCloser, error) { + reader := bytes.NewReader(contentBytes) + // Handle AList Range requests + _, _ = reader.Seek(range_.Start, io.SeekStart) + // Wrap as ReadCloser + return io.NopCloser(reader), nil + }, + }, + Header: http.Header{ + "Content-Type": []string{"application/vnd.apple.mpegurl"}, + }, + }, nil + } + + if res.DownloadURL != "" { + return &model.Link{URL: res.DownloadURL}, nil + } + + return nil, fmt.Errorf("failed to get link") +} diff --git a/drivers/189/driver.go b/drivers/189/driver.go index 6fc4932640c..0c8db1134ee 100644 --- a/drivers/189/driver.go +++ b/drivers/189/driver.go @@ -80,9 +80,10 @@ func (d *Cloud189) Link(ctx context.Context, file model.Obj, args model.LinkArgs } func (d *Cloud189) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + safeName := d.sanitizeName(dirName) form := map[string]string{ "parentFolderId": parentDir.GetID(), - "folderName": dirName, + "folderName": safeName, } _, err := d.request("https://cloud.189.cn/api/open/file/createFolder.action", http.MethodPost, func(req *resty.Request) { req.SetFormData(form) @@ -126,9 +127,10 @@ func (d *Cloud189) Rename(ctx context.Context, srcObj model.Obj, newName string) idKey = "folderId" nameKey = "destFolderName" } + safeName := d.sanitizeName(newName) form := map[string]string{ idKey: srcObj.GetID(), - nameKey: newName, + nameKey: safeName, } _, err := d.request(url, http.MethodPost, func(req *resty.Request) { req.SetFormData(form) diff --git a/drivers/189/meta.go b/drivers/189/meta.go index ad621fb440d..da81406e683 100644 --- a/drivers/189/meta.go +++ b/drivers/189/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - Cookie string `json:"cookie" help:"Fill in the cookie if need captcha"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + Cookie string `json:"cookie" help:"Fill in the cookie if need captcha"` + StripEmoji bool `json:"strip_emoji" help:"Remove four-byte characters (e.g., emoji) before upload"` driver.RootID } diff --git a/drivers/189/util.go b/drivers/189/util.go index 16a5aa3996e..ee2b5061dd8 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -11,9 +11,11 @@ import ( "io" "math" "net/http" + "path" "strconv" "strings" "time" + "unicode/utf8" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -222,13 +224,37 @@ func (d *Cloud189) getFiles(fileId string) ([]model.Obj, error) { return res, nil } +func (d *Cloud189) sanitizeName(name string) string { + if !d.StripEmoji { + return name + } + b := strings.Builder{} + for _, r := range name { + if utf8.RuneLen(r) == 4 { + continue + } + b.WriteRune(r) + } + sanitized := b.String() + if sanitized == "" { + ext := path.Ext(name) + if ext != "" { + sanitized = "file" + ext + } else { + sanitized = "file" + } + } + return sanitized +} + func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error { + safeName := d.sanitizeName(file.GetName()) res, err := d.client.R().SetMultipartFormData(map[string]string{ "parentId": dstDir.GetID(), "sessionKey": "??", "opertype": "1", - "fname": file.GetName(), - }).SetMultipartField("Filedata", file.GetName(), file.GetMimetype(), file).Post("https://hb02.upload.cloud.189.cn/v1/DCIWebUploadAction") + "fname": safeName, + }).SetMultipartField("Filedata", safeName, file.GetMimetype(), file).Post("https://hb02.upload.cloud.189.cn/v1/DCIWebUploadAction") if err != nil { return err } @@ -313,9 +339,10 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F const DEFAULT int64 = 10485760 var count = int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT))) + safeName := d.sanitizeName(file.GetName()) res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ "parentFolderId": dstDir.GetID(), - "fileName": encode(file.GetName()), + "fileName": encode(safeName), "fileSize": strconv.FormatInt(file.GetSize(), 10), "sliceSize": strconv.FormatInt(DEFAULT, 10), "lazyCheck": "1", diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index c91caf2fb4f..9462cef6662 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -205,10 +205,11 @@ func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName s fullUrl += "/createFolder.action" var newFolder Cloud189Folder + safeName := y.sanitizeName(dirName) _, err := y.post(fullUrl, func(req *resty.Request) { req.SetContext(ctx) req.SetQueryParams(map[string]string{ - "folderName": dirName, + "folderName": safeName, "relativePath": "", }) if isFamily { @@ -225,6 +226,7 @@ func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName s if err != nil { return nil, err } + newFolder.Name = safeName return &newFolder, nil } @@ -258,21 +260,29 @@ func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName strin } var newObj model.Obj + safeName := y.sanitizeName(newName) switch f := srcObj.(type) { case *Cloud189File: fullUrl += "/renameFile.action" queryParam["fileId"] = srcObj.GetID() - queryParam["destFileName"] = newName + queryParam["destFileName"] = safeName newObj = &Cloud189File{Icon: f.Icon} // 复用预览 case *Cloud189Folder: fullUrl += "/renameFolder.action" queryParam["folderId"] = srcObj.GetID() - queryParam["destFolderName"] = newName + queryParam["destFolderName"] = safeName newObj = &Cloud189Folder{} default: return nil, errs.NotSupport } + switch obj := newObj.(type) { + case *Cloud189File: + obj.Name = safeName + case *Cloud189Folder: + obj.Name = safeName + } + _, err := y.request(fullUrl, method, func(req *resty.Request) { req.SetContext(ctx).SetQueryParams(queryParam) }, nil, newObj, isFamily) diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 1891c5c0ccd..d6edc063593 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - VCode string `json:"validate_code"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + VCode string `json:"validate_code"` + StripEmoji bool `json:"strip_emoji" help:"Remove four-byte characters (e.g., emoji) before upload"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index c391f7e676f..ca89251e278 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -12,11 +12,13 @@ import ( "net/http/cookiejar" "net/url" "os" + "path" "regexp" "sort" "strconv" "strings" "time" + "unicode/utf8" "golang.org/x/sync/semaphore" @@ -57,6 +59,29 @@ const ( CHANNEL_ID = "web_cloud.189.cn" ) +func (y *Cloud189PC) sanitizeName(name string) string { + if !y.StripEmoji { + return name + } + b := strings.Builder{} + for _, r := range name { + if utf8.RuneLen(r) == 4 { + continue + } + b.WriteRune(r) + } + sanitized := b.String() + if sanitized == "" { + ext := path.Ext(name) + if ext != "" { + sanitized = "file" + ext + } else { + sanitized = "file" + } + } + return sanitized +} + func (y *Cloud189PC) SignatureHeader(url, method, params string, isFamily bool) map[string]string { dateOfGmt := getHttpDateStr() sessionKey := y.getTokenInfo().SessionKey @@ -324,7 +349,7 @@ func (y *Cloud189PC) login() (err error) { _, err = y.client.R(). SetResult(&tokenInfo).SetError(&erron). SetQueryParams(clientSuffix()). - SetQueryParam("redirectURL", url.QueryEscape(loginresp.ToUrl)). + SetQueryParam("redirectURL", loginresp.ToUrl). Post(API_URL + "/getSessionForPC.action") if err != nil { return @@ -475,10 +500,11 @@ func (y *Cloud189PC) refreshSession() (err error) { func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { size := file.GetSize() sliceSize := partSize(size) + safeName := y.sanitizeName(file.GetName()) params := Params{ "parentFolderId": dstDir.GetID(), - "fileName": url.QueryEscape(file.GetName()), + "fileName": url.QueryEscape(safeName), "fileSize": fmt.Sprint(file.GetSize()), "sliceSize": fmt.Sprint(sliceSize), "lazyCheck": "1", @@ -596,7 +622,8 @@ func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream m return nil, errors.New("invalid hash") } - uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, stream.GetName(), fmt.Sprint(stream.GetSize()), isFamily) + safeName := y.sanitizeName(stream.GetName()) + uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, safeName, fmt.Sprint(stream.GetSize()), isFamily) if err != nil { return nil, err } @@ -615,6 +642,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode tmpF *os.File err error ) + safeName := y.sanitizeName(file.GetName()) size := file.GetSize() if _, ok := cache.(io.ReaderAt); !ok && size > 0 { tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") @@ -697,7 +725,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode //step.2 预上传 params := Params{ "parentFolderId": dstDir.GetID(), - "fileName": url.QueryEscape(file.GetName()), + "fileName": url.QueryEscape(safeName), "fileSize": fmt.Sprint(file.GetSize()), "fileMd5": fileMd5Hex, "sliceSize": fmt.Sprint(sliceSize), @@ -833,9 +861,10 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model return nil, err } rateLimited := driver.NewLimitedUploadStream(ctx, io.NopCloser(tempFile)) + safeName := y.sanitizeName(file.GetName()) // 创建上传会话 - uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, file.GetName(), fmt.Sprint(file.GetSize()), isFamily) + uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, safeName, fmt.Sprint(file.GetSize()), isFamily) if err != nil { return nil, err } diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index ac7e16a1d16..56f9c01e124 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -56,7 +56,7 @@ func (d *AListV3) Init(ctx context.Context) error { if err != nil { return err } - if resp.Data.Role == model.GUEST { + if utils.SliceContains(resp.Data.Role, model.GUEST) { u := d.Address + "/api/public/settings" res, err := base.RestyClient.R().Get(u) if err != nil { diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go index 1ae7926e078..3e8e2f71eac 100644 --- a/drivers/alist_v3/types.go +++ b/drivers/alist_v3/types.go @@ -1,6 +1,7 @@ package alist_v3 import ( + "encoding/json" "time" "github.com/alist-org/alist/v3/internal/model" @@ -72,15 +73,15 @@ type LoginResp struct { } type MeResp struct { - Id int `json:"id"` - Username string `json:"username"` - Password string `json:"password"` - BasePath string `json:"base_path"` - Role int `json:"role"` - Disabled bool `json:"disabled"` - Permission int `json:"permission"` - SsoId string `json:"sso_id"` - Otp bool `json:"otp"` + Id int `json:"id"` + Username string `json:"username"` + Password string `json:"password"` + BasePath string `json:"base_path"` + Role IntSlice `json:"role"` + Disabled bool `json:"disabled"` + Permission int `json:"permission"` + SsoId string `json:"sso_id"` + Otp bool `json:"otp"` } type ArchiveMetaReq struct { @@ -168,3 +169,17 @@ type DecompressReq struct { PutIntoNewDir bool `json:"put_into_new_dir"` SrcDir string `json:"src_dir"` } + +type IntSlice []int + +func (s *IntSlice) UnmarshalJSON(data []byte) error { + if len(data) > 0 && data[0] == '[' { + return json.Unmarshal(data, (*[]int)(s)) + } + var single int + if err := json.Unmarshal(data, &single); err != nil { + return err + } + *s = []int{single} + return nil +} diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index 105e28b2e98..606ff385e81 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -55,7 +55,7 @@ func (d *AliDrive) Init(ctx context.Context) error { if err != nil { return err } - d.DriveId = utils.Json.Get(res, "default_drive_id").ToString() + d.DriveId = d.Addition.DeviceID d.UserID = utils.Json.Get(res, "user_id").ToString() d.cron = cron.NewCron(time.Hour * 2) d.cron.Do(func() { diff --git a/drivers/aliyundrive/meta.go b/drivers/aliyundrive/meta.go index 9aee856908d..a0ae8a5917d 100644 --- a/drivers/aliyundrive/meta.go +++ b/drivers/aliyundrive/meta.go @@ -7,8 +7,8 @@ import ( type Addition struct { driver.RootID - RefreshToken string `json:"refresh_token" required:"true"` - //DeviceID string `json:"device_id" required:"true"` + RefreshToken string `json:"refresh_token" required:"true"` + DeviceID string `json:"device_id" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` RapidUpload bool `json:"rapid_upload"` diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index 394eadb1b8c..5ef814184e3 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -3,12 +3,10 @@ package aliyundrive_open import ( "context" "errors" - "fmt" "net/http" "path/filepath" "time" - "github.com/Xhofe/rateg" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -24,9 +22,8 @@ type AliyundriveOpen struct { DriveId string - limitList func(ctx context.Context, data base.Json) (*Files, error) - limitLink func(ctx context.Context, file model.Obj) (*model.Link, error) - ref *AliyundriveOpen + limiter *limiter + ref *AliyundriveOpen } func (d *AliyundriveOpen) Config() driver.Config { @@ -38,25 +35,23 @@ func (d *AliyundriveOpen) GetAddition() driver.Additional { } func (d *AliyundriveOpen) Init(ctx context.Context) error { + d.limiter = getLimiterForUser(globalLimiterUserID) if d.LIVPDownloadFormat == "" { d.LIVPDownloadFormat = "jpeg" } if d.DriveType == "" { d.DriveType = "default" } - res, err := d.request("/adrive/v1.0/user/getDriveInfo", http.MethodPost, nil) + res, err := d.request(ctx, limiterOther, "/adrive/v1.0/user/getDriveInfo", http.MethodPost, nil) if err != nil { + d.limiter.free() + d.limiter = nil return err } d.DriveId = utils.Json.Get(res, d.DriveType+"_drive_id").ToString() - d.limitList = rateg.LimitFnCtx(d.list, rateg.LimitFnOption{ - Limit: 4, - Bucket: 1, - }) - d.limitLink = rateg.LimitFnCtx(d.link, rateg.LimitFnOption{ - Limit: 1, - Bucket: 1, - }) + userID := utils.Json.Get(res, "user_id").ToString() + d.limiter.free() + d.limiter = getLimiterForUser(userID) return nil } @@ -70,6 +65,8 @@ func (d *AliyundriveOpen) InitReference(storage driver.Driver) error { } func (d *AliyundriveOpen) Drop(ctx context.Context) error { + d.limiter.free() + d.limiter = nil d.ref = nil return nil } @@ -87,9 +84,6 @@ func (d *AliyundriveOpen) GetRoot(ctx context.Context) (model.Obj, error) { } func (d *AliyundriveOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - if d.limitList == nil { - return nil, fmt.Errorf("driver not init") - } files, err := d.getFiles(ctx, dir.GetID()) if err != nil { return nil, err @@ -108,7 +102,7 @@ func (d *AliyundriveOpen) List(ctx context.Context, dir model.Obj, args model.Li } func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link, error) { - res, err := d.request("/adrive/v1.0/openFile/getDownloadUrl", http.MethodPost, func(req *resty.Request) { + res, err := d.request(ctx, limiterLink, "/adrive/v1.0/openFile/getDownloadUrl", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": file.GetID(), @@ -133,16 +127,13 @@ func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link } func (d *AliyundriveOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - if d.limitLink == nil { - return nil, fmt.Errorf("driver not init") - } - return d.limitLink(ctx, file) + return d.link(ctx, file) } func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { nowTime, _ := getNowTime() newDir := File{CreatedAt: nowTime, UpdatedAt: nowTime} - _, err := d.request("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "parent_file_id": parentDir.GetID(), @@ -168,7 +159,7 @@ func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirN func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { var resp MoveOrCopyResp - _, err := d.request("/adrive/v1.0/openFile/move", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/move", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), @@ -198,7 +189,7 @@ func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (m func (d *AliyundriveOpen) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { var newFile File - _, err := d.request("/adrive/v1.0/openFile/update", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/update", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), @@ -230,7 +221,7 @@ func (d *AliyundriveOpen) Rename(ctx context.Context, srcObj model.Obj, newName func (d *AliyundriveOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { var resp MoveOrCopyResp - _, err := d.request("/adrive/v1.0/openFile/copy", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/copy", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), @@ -256,7 +247,7 @@ func (d *AliyundriveOpen) Remove(ctx context.Context, obj model.Obj) error { if d.RemoveWay == "delete" { uri = "/adrive/v1.0/openFile/delete" } - _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, uri, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": obj.GetID(), @@ -295,7 +286,7 @@ func (d *AliyundriveOpen) Other(ctx context.Context, args model.OtherArgs) (inte default: return nil, errs.NotSupport } - _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, uri, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { diff --git a/drivers/aliyundrive_open/limiter.go b/drivers/aliyundrive_open/limiter.go new file mode 100644 index 00000000000..5138fbe2e50 --- /dev/null +++ b/drivers/aliyundrive_open/limiter.go @@ -0,0 +1,97 @@ +package aliyundrive_open + +import ( + "context" + "fmt" + "sync" + + "golang.org/x/time/rate" +) + +// Aliyun Open API rate limits are per user per app, so requests for the same +// user should share one limiter across all storage instances. +type limiterType int + +const ( + limiterList limiterType = iota + limiterLink + limiterOther +) + +const ( + listRateLimit = 3.9 + linkRateLimit = 0.9 + otherRateLimit = 14.9 + globalLimiterUserID = "" +) + +type limiter struct { + usedBy int + list *rate.Limiter + link *rate.Limiter + other *rate.Limiter +} + +var ( + limiters = make(map[string]*limiter) + limitersLock sync.Mutex +) + +func getLimiterForUser(userID string) *limiter { + limitersLock.Lock() + defer limitersLock.Unlock() + defer func() { + for id, lim := range limiters { + if lim.usedBy <= 0 && id != globalLimiterUserID { + delete(limiters, id) + } + } + }() + if lim, ok := limiters[userID]; ok { + lim.usedBy++ + return lim + } + lim := &limiter{ + usedBy: 1, + list: rate.NewLimiter(rate.Limit(listRateLimit), 1), + link: rate.NewLimiter(rate.Limit(linkRateLimit), 1), + other: rate.NewLimiter(rate.Limit(otherRateLimit), 1), + } + limiters[userID] = lim + return lim +} + +func (l *limiter) wait(ctx context.Context, typ limiterType) error { + if l == nil { + return fmt.Errorf("driver not init") + } + switch typ { + case limiterList: + return l.list.Wait(ctx) + case limiterLink: + return l.link.Wait(ctx) + case limiterOther: + return l.other.Wait(ctx) + default: + return fmt.Errorf("unknown limiter type") + } +} + +func (l *limiter) free() { + if l == nil { + return + } + limitersLock.Lock() + defer limitersLock.Unlock() + l.usedBy-- +} + +func (d *AliyundriveOpen) wait(ctx context.Context, typ limiterType) error { + if d == nil { + return fmt.Errorf("driver not init") + } + if d.ref != nil { + return d.ref.wait(ctx, typ) + } + return d.limiter.wait(ctx, typ) +} diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index 03f97f8b795..bb4354ddc11 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -11,7 +11,7 @@ type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` - OauthTokenURL string `json:"oauth_token_url" default:"https://api.nn.ci/alist/ali_open/token"` + OauthTokenURL string `json:"oauth_token_url" default:"https://api.alistgo.com/alist/ali_open/token"` ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index 4114c195182..19a3c3d0f05 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -50,10 +50,10 @@ func calPartSize(fileSize int64) int64 { return partSize } -func (d *AliyundriveOpen) getUploadUrl(count int, fileId, uploadId string) ([]PartInfo, error) { +func (d *AliyundriveOpen) getUploadUrl(ctx context.Context, count int, fileId, uploadId string) ([]PartInfo, error) { partInfoList := makePartInfos(count) var resp CreateResp - _, err := d.request("/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": fileId, @@ -84,10 +84,10 @@ func (d *AliyundriveOpen) uploadPart(ctx context.Context, r io.Reader, partInfo return nil } -func (d *AliyundriveOpen) completeUpload(fileId, uploadId string) (model.Obj, error) { +func (d *AliyundriveOpen) completeUpload(ctx context.Context, fileId, uploadId string) (model.Obj, error) { // 3. complete var newFile File - _, err := d.request("/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": fileId, @@ -183,7 +183,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m createData["pre_hash"] = hash } var createResp CreateResp - _, err, e := d.requestReturnErrResp("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err, e := d.requestReturnErrResp(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) if err != nil { @@ -208,7 +208,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m if err != nil { return nil, fmt.Errorf("cal proof code error: %s", err.Error()) } - _, err = d.request("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err = d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) if err != nil { @@ -229,7 +229,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m } // refresh upload url if 50 minutes passed if time.Since(preTime) > 50*time.Minute { - createResp.PartInfoList, err = d.getUploadUrl(count, createResp.FileId, createResp.UploadId) + createResp.PartInfoList, err = d.getUploadUrl(ctx, count, createResp.FileId, createResp.UploadId) if err != nil { return nil, err } @@ -266,5 +266,5 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m log.Debugf("[aliyundrive_open] create file success, resp: %+v", createResp) // 3. complete - return d.completeUpload(createResp.FileId, createResp.UploadId) + return d.completeUpload(ctx, createResp.FileId, createResp.UploadId) } diff --git a/drivers/aliyundrive_open/util.go b/drivers/aliyundrive_open/util.go index c3cda10aa88..544c8bdd10b 100644 --- a/drivers/aliyundrive_open/util.go +++ b/drivers/aliyundrive_open/util.go @@ -19,13 +19,94 @@ import ( // do others that not defined in Driver interface -func (d *AliyundriveOpen) _refreshToken() (string, string, error) { - url := API_URL + "/oauth/access_token" +const legacyOauthTokenURL = "https://api.alistgo.com/alist/ali_open/token" + +type refreshRateLimitError struct { + message string + retryAfter time.Duration +} + +func (e *refreshRateLimitError) Error() string { + if e.retryAfter > 0 { + return fmt.Sprintf("%s, retry after %s", e.message, e.retryAfter.Round(time.Second)) + } + return e.message +} + +func (d *AliyundriveOpen) _refreshToken(ctx context.Context) (string, string, error) { if d.OauthTokenURL != "" && d.ClientID == "" { - url = d.OauthTokenURL + return d.refreshTokenWithOnlineAPI(ctx) + } + + if d.ClientID == "" || d.ClientSecret == "" { + return "", "", fmt.Errorf("empty ClientID or ClientSecret") + } + return d.refreshTokenWithPost(ctx, API_URL+"/oauth/access_token") +} + +func (d *AliyundriveOpen) refreshTokenWithOnlineAPI(ctx context.Context) (string, string, error) { + // New hosted renew endpoint uses a GET API and returns {"refresh_token","access_token","text"}. + // Older hosted endpoints still expect the legacy POST payload, so we fall back when we detect that shape. + var resp struct { + RefreshToken string `json:"refresh_token"` + AccessToken string `json:"access_token"` + ErrorMessage string `json:"text"` + } + var e ErrResp + if err := d.wait(ctx, limiterOther); err != nil { + return "", "", err + } + res, err := base.RestyClient.R(). + SetResult(&resp). + SetError(&e). + SetQueryParams(map[string]string{ + "refresh_ui": d.RefreshToken, + "server_use": "true", + "driver_txt": "alicloud_qr", + }). + Get(d.OauthTokenURL) + if err != nil { + return "", "", err + } + if resp.RefreshToken != "" && resp.AccessToken != "" { + return resp.RefreshToken, resp.AccessToken, nil + } + if resp.ErrorMessage != "" { + if res != nil && res.StatusCode() == http.StatusTooManyRequests { + return d.refreshTokenWithLegacyFallback(ctx, resp.ErrorMessage, retryAfterFromResponse(res)) + } + return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) + } + if res != nil && res.StatusCode() == http.StatusTooManyRequests { + return d.refreshTokenWithLegacyFallback(ctx, http.StatusText(http.StatusTooManyRequests), retryAfterFromResponse(res)) } + if e.Code != "" || e.Message != "" { + return d.refreshTokenWithPost(ctx, d.OauthTokenURL) + } + return "", "", fmt.Errorf("empty token returned from online API") +} + +func (d *AliyundriveOpen) refreshTokenWithLegacyFallback(ctx context.Context, message string, retryAfter time.Duration) (string, string, error) { + if d.OauthTokenURL != legacyOauthTokenURL { + log.Warnf("[ali_open] online refresh API is rate-limited, trying legacy fallback: %s", legacyOauthTokenURL) + if refresh, access, err := d.refreshTokenWithPost(ctx, legacyOauthTokenURL); err == nil { + return refresh, access, nil + } else if _, ok := err.(*refreshRateLimitError); !ok { + return "", "", err + } + } + return "", "", &refreshRateLimitError{ + message: fmt.Sprintf("failed to refresh token: %s", message), + retryAfter: retryAfter, + } +} + +func (d *AliyundriveOpen) refreshTokenWithPost(ctx context.Context, url string) (string, string, error) { //var resp base.TokenResp var e ErrResp + if err := d.wait(ctx, limiterOther); err != nil { + return "", "", err + } res, err := base.RestyClient.R(). //ForceContentType("application/json"). SetBody(base.Json{ @@ -41,6 +122,16 @@ func (d *AliyundriveOpen) _refreshToken() (string, string, error) { return "", "", err } log.Debugf("[ali_open] refresh token response: %s", res.String()) + if res.StatusCode() == http.StatusTooManyRequests { + message := e.Message + if message == "" { + message = http.StatusText(http.StatusTooManyRequests) + } + return "", "", &refreshRateLimitError{ + message: fmt.Sprintf("failed to refresh token: %s", message), + retryAfter: retryAfterFromResponse(res), + } + } if e.Code != "" { return "", "", fmt.Errorf("failed to refresh token: %s", e.Message) } @@ -74,18 +165,29 @@ func getSub(token string) (string, error) { return utils.Json.Get(bs, "sub").ToString(), nil } -func (d *AliyundriveOpen) refreshToken() error { +func (d *AliyundriveOpen) refreshToken(ctx context.Context) error { if d.ref != nil { - return d.ref.refreshToken() + return d.ref.refreshToken(ctx) } - refresh, access, err := d._refreshToken() + refresh, access, err := d._refreshToken(ctx) for i := 0; i < 3; i++ { if err == nil { break + } + if rateLimitErr, ok := err.(*refreshRateLimitError); ok { + wait := rateLimitErr.retryAfter + if wait <= 0 { + wait = time.Duration(i+1) * 2 * time.Second + } + if wait > 15*time.Second { + wait = 15 * time.Second + } + log.Warnf("[ali_open] %s", rateLimitErr.Error()) + time.Sleep(wait) } else { log.Errorf("[ali_open] failed to refresh token: %s", err) } - refresh, access, err = d._refreshToken() + refresh, access, err = d._refreshToken(ctx) } if err != nil { return err @@ -96,12 +198,29 @@ func (d *AliyundriveOpen) refreshToken() error { return nil } -func (d *AliyundriveOpen) request(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { - b, err, _ := d.requestReturnErrResp(uri, method, callback, retry...) +func retryAfterFromResponse(res *resty.Response) time.Duration { + if res == nil { + return 0 + } + retryAfter := strings.TrimSpace(res.Header().Get("Retry-After")) + if retryAfter == "" { + return 0 + } + if seconds, err := time.ParseDuration(retryAfter + "s"); err == nil { + return seconds + } + if t, err := http.ParseTime(retryAfter); err == nil { + return time.Until(t) + } + return 0 +} + +func (d *AliyundriveOpen) request(ctx context.Context, limitTy limiterType, uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { + b, err, _ := d.requestReturnErrResp(ctx, limitTy, uri, method, callback, retry...) return b, err } -func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error, *ErrResp) { +func (d *AliyundriveOpen) requestReturnErrResp(ctx context.Context, limitTy limiterType, uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error, *ErrResp) { req := base.RestyClient.R() // TODO check whether access_token is expired req.SetHeader("Authorization", "Bearer "+d.getAccessToken()) @@ -113,6 +232,9 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base } var e ErrResp req.SetError(&e) + if err := d.wait(ctx, limitTy); err != nil { + return nil, err, nil + } res, err := req.Execute(method, API_URL+uri) if err != nil { if res != nil { @@ -123,11 +245,11 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base isRetry := len(retry) > 0 && retry[0] if e.Code != "" { if !isRetry && (utils.SliceContains([]string{"AccessTokenInvalid", "AccessTokenExpired", "I400JD"}, e.Code) || d.getAccessToken() == "") { - err = d.refreshToken() + err = d.refreshToken(ctx) if err != nil { return nil, err, nil } - return d.requestReturnErrResp(uri, method, callback, true) + return d.requestReturnErrResp(ctx, limitTy, uri, method, callback, true) } return nil, fmt.Errorf("%s:%s", e.Code, e.Message), &e } @@ -136,7 +258,7 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base func (d *AliyundriveOpen) list(ctx context.Context, data base.Json) (*Files, error) { var resp Files - _, err := d.request("/adrive/v1.0/openFile/list", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterList, "/adrive/v1.0/openFile/list", http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { @@ -165,7 +287,7 @@ func (d *AliyundriveOpen) getFiles(ctx context.Context, fileId string) ([]File, //"video_thumbnail_width": 480, //"image_thumbnail_width": 480, } - resp, err := d.limitList(ctx, data) + resp, err := d.list(ctx, data) if err != nil { return nil, err } diff --git a/drivers/all.go b/drivers/all.go index 0b8ce3aa61d..3dc90424cbe 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -6,6 +6,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/115_share" _ "github.com/alist-org/alist/v3/drivers/123" _ "github.com/alist-org/alist/v3/drivers/123_link" + _ "github.com/alist-org/alist/v3/drivers/123_open" _ "github.com/alist-org/alist/v3/drivers/123_share" _ "github.com/alist-org/alist/v3/drivers/139" _ "github.com/alist-org/alist/v3/drivers/189" @@ -20,18 +21,28 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_netdisk" _ "github.com/alist-org/alist/v3/drivers/baidu_photo" _ "github.com/alist-org/alist/v3/drivers/baidu_share" + _ "github.com/alist-org/alist/v3/drivers/baidu_youth" + _ "github.com/alist-org/alist/v3/drivers/bitqiu" _ "github.com/alist-org/alist/v3/drivers/chaoxing" + _ "github.com/alist-org/alist/v3/drivers/chunker" _ "github.com/alist-org/alist/v3/drivers/cloudreve" + _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" _ "github.com/alist-org/alist/v3/drivers/crypt" + _ "github.com/alist-org/alist/v3/drivers/darkibox" _ "github.com/alist-org/alist/v3/drivers/doubao" + _ "github.com/alist-org/alist/v3/drivers/doubao_new" _ "github.com/alist-org/alist/v3/drivers/doubao_share" _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" + _ "github.com/alist-org/alist/v3/drivers/ftps" + _ "github.com/alist-org/alist/v3/drivers/gitee" _ "github.com/alist-org/alist/v3/drivers/github" _ "github.com/alist-org/alist/v3/drivers/github_releases" + _ "github.com/alist-org/alist/v3/drivers/gofile" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" + _ "github.com/alist-org/alist/v3/drivers/guangyapan" _ "github.com/alist-org/alist/v3/drivers/halalcloud" _ "github.com/alist-org/alist/v3/drivers/ilanzou" _ "github.com/alist-org/alist/v3/drivers/ipfs_api" @@ -39,6 +50,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/lanzou" _ "github.com/alist-org/alist/v3/drivers/lenovonas_share" _ "github.com/alist-org/alist/v3/drivers/local" + _ "github.com/alist-org/alist/v3/drivers/mediafire" _ "github.com/alist-org/alist/v3/drivers/mediatrack" _ "github.com/alist-org/alist/v3/drivers/mega" _ "github.com/alist-org/alist/v3/drivers/misskey" @@ -47,8 +59,10 @@ import ( _ "github.com/alist-org/alist/v3/drivers/onedrive" _ "github.com/alist-org/alist/v3/drivers/onedrive_app" _ "github.com/alist-org/alist/v3/drivers/onedrive_sharelink" + _ "github.com/alist-org/alist/v3/drivers/pcloud" _ "github.com/alist-org/alist/v3/drivers/pikpak" _ "github.com/alist-org/alist/v3/drivers/pikpak_share" + _ "github.com/alist-org/alist/v3/drivers/proton_drive" _ "github.com/alist-org/alist/v3/drivers/quark_uc" _ "github.com/alist-org/alist/v3/drivers/quark_uc_tv" _ "github.com/alist-org/alist/v3/drivers/quqi" @@ -56,6 +70,8 @@ import ( _ "github.com/alist-org/alist/v3/drivers/seafile" _ "github.com/alist-org/alist/v3/drivers/sftp" _ "github.com/alist-org/alist/v3/drivers/smb" + _ "github.com/alist-org/alist/v3/drivers/streamtape" + _ "github.com/alist-org/alist/v3/drivers/strm" _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" @@ -69,7 +85,9 @@ import ( _ "github.com/alist-org/alist/v3/drivers/webdav" _ "github.com/alist-org/alist/v3/drivers/weiyun" _ "github.com/alist-org/alist/v3/drivers/wopan" + _ "github.com/alist-org/alist/v3/drivers/wukong" _ "github.com/alist-org/alist/v3/drivers/yandex_disk" + _ "github.com/alist-org/alist/v3/drivers/yunpan360" ) // All do nothing,just for import diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index c33e0b32b05..64539dc7d89 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -5,23 +5,26 @@ import ( "crypto/md5" "encoding/hex" "errors" + "fmt" "io" "net/url" "os" stdpath "path" "strconv" + "strings" + "sync" "time" - "golang.org/x/sync/semaphore" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/errgroup" + "github.com/alist-org/alist/v3/pkg/singleflight" "github.com/alist-org/alist/v3/pkg/utils" "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -31,8 +34,16 @@ type BaiduNetdisk struct { uploadThread int vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) + + upClient *resty.Client // 上传文件使用的http客户端 + uploadUrlG singleflight.Group[string] + uploadUrlMu sync.RWMutex + uploadUrl string // 上传域名 + uploadUrlUpdateTime time.Time // 上传域名上次更新时间 } +var ErrUploadIDExpired = errors.New("uploadid expired") + func (d *BaiduNetdisk) Config() driver.Config { return config } @@ -42,19 +53,26 @@ func (d *BaiduNetdisk) GetAddition() driver.Additional { } func (d *BaiduNetdisk) Init(ctx context.Context) error { + d.upClient = base.NewRestyClient(). + SetTimeout(UPLOAD_TIMEOUT). + SetRetryCount(UPLOAD_RETRY_COUNT). + SetRetryWaitTime(UPLOAD_RETRY_WAIT_TIME). + SetRetryMaxWaitTime(UPLOAD_RETRY_MAX_WAIT_TIME) d.uploadThread, _ = strconv.Atoi(d.UploadThread) - if d.uploadThread < 1 || d.uploadThread > 32 { - d.uploadThread, d.UploadThread = 3, "3" + if d.uploadThread < 1 { + d.uploadThread, d.UploadThread = 1, "1" + } else if d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 32, "32" } if _, err := url.Parse(d.UploadAPI); d.UploadAPI == "" || err != nil { - d.UploadAPI = "https://d.pcs.baidu.com" + d.UploadAPI = UPLOAD_FALLBACK_API } res, err := d.get("/xpan/nas", map[string]string{ "method": "uinfo", }, nil) - log.Debugf("[baidu] get uinfo: %s", string(res)) + log.Debugf("[baidu_netdisk] get uinfo: %s", string(res)) if err != nil { return err } @@ -181,6 +199,11 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo // **注意**: 截至 2024/04/20 百度云盘 api 接口返回的时间永远是当前时间，而不是文件时间。 // 而实际上云盘存储的时间是文件时间，所以此处需要覆盖时间，保证缓存与云盘的数据一致 func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // 百度网盘不允许上传空文件 + if stream.GetSize() < 1 { + return nil, ErrBaiduEmptyFilesNotAllowed + } + // rapid upload if newObj, err := d.PutRapid(ctx, dstDir, stream); err == nil { return newObj, nil @@ -245,7 +268,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } if tmpF != nil { if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize) + return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) } _, err = tmpF.Seek(0, io.SeekStart) if err != nil { @@ -259,82 +282,97 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 预上传 - // 尝试获取之前的进度 + // step.1 尝试读取已保存进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { - params := map[string]string{ - "method": "precreate", - } - form := map[string]string{ - "path": path, - "size": strconv.FormatInt(streamSize, 10), - "isdir": "0", - "autoinit": "1", - "rtype": "3", - "block_list": blockListStr, - "content-md5": contentMd5, - "slice-md5": sliceMd5, - } - joinTime(form, ctime, mtime) - - log.Debugf("[baidu_netdisk] precreate data: %s", form) - _, err = d.postForm("/xpan/file", params, form, &precreateResp) + // 没有进度，走预上传 + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) if err != nil { return nil, err } - log.Debugf("%+v", precreateResp) if precreateResp.ReturnType == 2 { //rapid upload, since got md5 match from baidu server // 修复时间，具体原因见 Put 方法注释的 **注意** - precreateResp.File.Ctime = ctime - precreateResp.File.Mtime = mtime return fileToObj(precreateResp.File), nil } } + // step.2 上传分片 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(1), - retry.Delay(time.Second), - retry.DelayType(retry.BackOffDelay)) - sem := semaphore.NewWeighted(3) - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break +uploadLoop: + for attempt := 0; attempt < 2; attempt++ { + // 获取上传域名 + uploadUrl := d.getUploadUrl(path, precreateResp.Uploadid) + // 并发上传 + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + cacheReaderAt, okReaderAt := cache.(io.ReaderAt) + if !okReaderAt { + return nil, fmt.Errorf("cache object must implement io.ReaderAt interface for upload operations") } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - byteSize = lastBlockSize - } - threadG.Go(func(ctx context.Context) error { - if err = sem.Acquire(ctx, 1); err != nil { - return err - } - defer sem.Release(1) - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), + totalParts := len(precreateResp.BlockList) + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) || partseq < 0 { + continue } - err := d.uploadSlice(ctx, params, stream.GetName(), - driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize))) - if err != nil { - return err + + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize } - up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) - precreateResp.BlockList[i] = -1 - return nil - }) - } - if err = threadG.Wait(); err != nil { - // 如果属于用户主动取消，则保存上传进度 + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": precreateResp.Uploadid, + "partseq": strconv.Itoa(partseq), + } + section := io.NewSectionReader(cacheReaderAt, offset, size) + err := d.uploadSlice(ctx, uploadUrl, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section)) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + // 当前goroutine还没退出，+1才是真正成功的数量 + success := threadG.Success() + 1 + progress := float64(success) * 100 / float64(totalParts) + up(progress) + return nil + }) + } + + err = threadG.Wait() + if err == nil { + break uploadLoop + } + + // 保存进度（所有错误都会保存） + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + if errors.Is(err, context.Canceled) { - precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch") + // 重新 precreate（所有分片都要重传） + newPre, err2 := d.precreate(ctx, path, streamSize, blockListStr, "", "", ctime, mtime) + if err2 != nil { + return nil, err2 + } + if newPre.ReturnType == 2 { + return fileToObj(newPre.File), nil + } + precreateResp = newPre + // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + continue uploadLoop } return nil, err } @@ -348,23 +386,67 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // 修复时间，具体原因见 Put 方法注释的 **注意** newFile.Ctime = ctime newFile.Mtime = mtime + // 上传成功清理进度 + base.SaveUploadProgress(d, nil, d.AccessToken, contentMd5) return fileToObj(newFile), nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { - res, err := base.RestyClient.R(). +// precreate 执行预上传操作，支持首次上传和 uploadid 过期重试 +func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + params := map[string]string{"method": "precreate"} + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + } + + // 只有在首次上传时才包含 content-md5 和 slice-md5 + if contentMd5 != "" && sliceMd5 != "" { + form["content-md5"] = contentMd5 + form["slice-md5"] = sliceMd5 + } + + joinTime(form, ctime, mtime) + + var precreateResp PrecreateResp + _, err := d.postForm("/xpan/file", params, form, &precreateResp) + if err != nil { + return nil, err + } + + // 修复时间，具体原因见 Put 方法注释的 **注意** + if precreateResp.ReturnType == 2 { + precreateResp.File.Ctime = ctime + precreateResp.File.Mtime = mtime + } + + return &precreateResp, nil +} + +func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file io.Reader) error { + res, err := d.upClient.R(). SetContext(ctx). SetQueryParams(params). SetFileReader("file", fileName, file). - Post(d.UploadAPI + "/rest/2.0/pcs/superfile2") + Post(uploadUrl + "/rest/2.0/pcs/superfile2") if err != nil { return err } log.Debugln(res.RawResponse.Status + res.String()) errCode := utils.Json.Get(res.Body(), "error_code").ToInt() errNo := utils.Json.Get(res.Body(), "errno").ToInt() + respStr := res.String() + lower := strings.ToLower(respStr) + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error in uploading to baidu, will retry. response=%s", res.String()) + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", res.String()) } return nil } diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 27571056e11..b75650ef063 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -1,6 +1,8 @@ package baidu_netdisk import ( + "time" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/op" ) @@ -11,17 +13,27 @@ type Addition struct { OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` DownloadAPI string `json:"download_api" type:"select" options:"official,crack,crack_video" default:"official"` - ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` - ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` + ClientID string `json:"client_id" required:"true" default:"hq9yQ9w9kR4YHj1kyYafLygVocobh7Sf"` + ClientSecret string `json:"client_secret" required:"true" default:"YH2VpZcFJHYNnV6vLfHQXDBhcE7ZChyE"` CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` AccessToken string UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` + UseDynamicUploadAPI bool `json:"use_dynamic_upload_api" default:"true" help:"dynamically get upload api domain, when enabled, the 'Upload API' setting will be used as a fallback if failed to get"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` LowBandwithUploadMode bool `json:"low_bandwith_upload_mode" default:"false"` OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` } +const ( + UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 + UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) + UPLOAD_TIMEOUT = time.Minute * 30 // 上传请求超时时间 + UPLOAD_RETRY_COUNT = 3 + UPLOAD_RETRY_WAIT_TIME = time.Second * 1 + UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 +) + var config = driver.Config{ Name: "BaiduNetdisk", DefaultRoot: "/", diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index ed9b09df8ee..a158956d09b 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -1,6 +1,7 @@ package baidu_netdisk import ( + "errors" "path" "strconv" "time" @@ -9,6 +10,10 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" ) +var ( + ErrBaiduEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu netdisk") +) + type TokenErrResp struct { ErrorDescription string `json:"error_description"` Error string `json:"error"` @@ -189,3 +194,27 @@ type PrecreateResp struct { // return_type=2 File File `json:"info"` } + +type UploadServerResp struct { + BakServer []any `json:"bak_server"` + BakServers []struct { + Server string `json:"server"` + } `json:"bak_servers"` + ClientIP string `json:"client_ip"` + ErrorCode int `json:"error_code"` + ErrorMsg string `json:"error_msg"` + Expire int `json:"expire"` + Host string `json:"host"` + Newno string `json:"newno"` + QuicServer []any `json:"quic_server"` + QuicServers []struct { + Server string `json:"server"` + } `json:"quic_servers"` + RequestID int64 `json:"request_id"` + Server []any `json:"server"` + ServerTime int `json:"server_time"` + Servers []struct { + Server string `json:"server"` + } `json:"servers"` + Sl int `json:"sl"` +} diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 1249b3f470f..c5a7334315d 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -73,7 +73,7 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall errno := utils.Json.Get(res.Body(), "errno").ToInt() if errno != 0 { if utils.SliceContains([]int{111, -6}, errno) { - log.Info("refreshing baidu_netdisk token.") + log.Info("[baidu_netdisk] refreshing baidu_netdisk token.") err2 := d.refreshToken() if err2 != nil { return retry.Unrecoverable(err2) @@ -284,10 +284,10 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { // 非会员固定为 4MB if d.vipType == 0 { if d.CustomUploadPartSize != 0 { - log.Warnf("CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") + log.Warnf("[baidu_netdisk] CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") } if filesize > MaxSliceNum*DefaultSliceSize { - log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return DefaultSliceSize @@ -295,17 +295,17 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { if d.CustomUploadPartSize != 0 { if d.CustomUploadPartSize < DefaultSliceSize { - log.Warnf("CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) return DefaultSliceSize } if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize { - log.Warnf("CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) return VipSliceSize } if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize { - log.Warnf("CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) return SVipSliceSize } @@ -335,12 +335,89 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { } if filesize > MaxSliceNum*maxSliceSize { - log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return maxSliceSize } +// getUploadUrl 从开放平台获取上传域名/地址，并发请求会被合并，结果会被缓存1h。 +// 如果获取失败，则返回 Upload API设置项。 +func (d *BaiduNetdisk) getUploadUrl(path, uploadId string) string { + if !d.UseDynamicUploadAPI { + return d.UploadAPI + } + getCachedUrlFunc := func() string { + d.uploadUrlMu.RLock() + defer d.uploadUrlMu.RUnlock() + if d.uploadUrl != "" && time.Since(d.uploadUrlUpdateTime) < UPLOAD_URL_EXPIRE_TIME { + return d.uploadUrl + } + return "" + } + // 检查地址缓存 + if uploadUrl := getCachedUrlFunc(); uploadUrl != "" { + return uploadUrl + } + + uploadUrlGetFunc := func() (string, error) { + // 双重检查缓存 + if uploadUrl := getCachedUrlFunc(); uploadUrl != "" { + return uploadUrl, nil + } + + uploadUrl, err := d.requestForUploadUrl(path, uploadId) + if err != nil { + return "", err + } + + d.uploadUrlMu.Lock() + defer d.uploadUrlMu.Unlock() + d.uploadUrl = uploadUrl + d.uploadUrlUpdateTime = time.Now() + return uploadUrl, nil + } + + uploadUrl, err, _ := d.uploadUrlG.Do("", uploadUrlGetFunc) + if err != nil { + fallback := d.UploadAPI + log.Warnf("[baidu_netdisk] get upload URL failed (%v), will use fallback URL: %s", err, fallback) + return fallback + } + return uploadUrl +} + +// requestForUploadUrl 请求获取上传地址。 +// 实测此接口不需要认证，传method和upload_version就行，不过还是按文档规范调用。 +// https://pan.baidu.com/union/doc/Mlvw5hfnr +func (d *BaiduNetdisk) requestForUploadUrl(path, uploadId string) (string, error) { + params := map[string]string{ + "method": "locateupload", + "appid": "250528", + "path": path, + "uploadid": uploadId, + "upload_version": "2.0", + } + apiUrl := "https://d.pcs.baidu.com/rest/2.0/pcs/file" + var resp UploadServerResp + _, err := d.request(apiUrl, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(params) + }, &resp) + if err != nil { + return "", err + } + var uploadUrl string + if len(resp.Servers) > 0 { + uploadUrl = resp.Servers[0].Server + } else if len(resp.BakServers) > 0 { + uploadUrl = resp.BakServers[0].Server + } + if uploadUrl == "" { + return "", errors.New("upload URL is empty") + } + return uploadUrl, nil +} + // func encodeURIComponent(str string) string { // r := url.QueryEscape(str) // r = strings.ReplaceAll(r, "+", "%20") diff --git a/drivers/baidu_youth/driver.go b/drivers/baidu_youth/driver.go new file mode 100644 index 00000000000..a0b6a4853ec --- /dev/null +++ b/drivers/baidu_youth/driver.go @@ -0,0 +1,425 @@ +package baidu_youth + +import ( + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "os" + stdpath "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/errgroup" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" +) + +type BaiduYouth struct { + model.Storage + Addition + + uk int64 + bdstoken string + sk string + uploadThread int + upClient *resty.Client +} + +var ErrUploadIDExpired = errors.New("uploadid expired") + +func (d *BaiduYouth) Config() driver.Config { + return config +} + +func (d *BaiduYouth) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *BaiduYouth) Init(ctx context.Context) error { + d.Cookie = strings.TrimSpace(d.Cookie) + if d.Storage.Addition != "" { + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(d.Storage.Addition), &raw); err == nil { + if _, ok := raw["force_proxy"]; !ok { + d.ForceProxy = true + } + } + } + d.upClient = base.NewRestyClient(). + SetTimeout(UPLOAD_TIMEOUT). + SetRetryCount(UPLOAD_RETRY_COUNT). + SetRetryWaitTime(UPLOAD_RETRY_WAIT_TIME). + SetRetryMaxWaitTime(UPLOAD_RETRY_MAX_WAIT_TIME) + + d.uploadThread, _ = strconv.Atoi(d.UploadThread) + if d.uploadThread < 1 { + d.uploadThread, d.UploadThread = 1, "1" + } else if d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 32, "32" + } + + u, err := url.Parse(d.UploadAPI) + if d.UploadAPI == "" || err != nil || u.Scheme == "" || u.Host == "" { + d.UploadAPI = UPLOAD_FALLBACK_API + } else { + d.UploadAPI = strings.TrimRight(d.UploadAPI, "/") + } + + uk, bdstoken, sk, err := d.getUserSession(ctx) + if err != nil { + return err + } + d.uk = uk + d.bdstoken = bdstoken + d.sk = sk + return nil +} + +func (d *BaiduYouth) ShouldProxyDownloads() bool { + return d.ForceProxy +} + +func (d *BaiduYouth) Drop(ctx context.Context) error { + d.uk = 0 + d.bdstoken = "" + d.sk = "" + return nil +} + +func (d *BaiduYouth) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(ctx, dir.GetPath()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *BaiduYouth) Get(ctx context.Context, path string) (model.Obj, error) { + return d.getByPath(ctx, path) +} + +func (d *BaiduYouth) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if d.DownloadAPI == "crack" { + return d.linkCrack(ctx, file) + } + return d.linkOfficial(ctx, file) +} + +func (d *BaiduYouth) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + path := stdpath.Join(parentDir.GetPath(), dirName) + var resp CreateResp + _, err := d.postForm(ctx, "/youth/api/create", map[string]string{ + "a": "commit", + "bdstoken": d.bdstoken, + }, map[string]string{ + "block_list": "[]", + "isdir": "1", + "path": path, + }, &resp) + if err != nil { + return nil, err + } + if result := resp.ResultFile(); result.Path != "" || result.FsId != 0 { + return fileToObj(result), nil + } + return d.getByPath(ctx, path) +} + +func (d *BaiduYouth) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + _, err := d.manage(ctx, "move", []base.Json{ + { + "dest": dstDir.GetPath(), + "newname": srcObj.GetName(), + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + newPath := stdpath.Join(dstDir.GetPath(), srcObj.GetName()) + if obj, ok := srcObj.(*model.ObjThumb); ok { + obj.SetPath(newPath) + obj.Modified = time.Now() + return obj, nil + } + return d.getByPath(ctx, newPath) +} + +func (d *BaiduYouth) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + _, err := d.manage(ctx, "rename", []base.Json{ + { + "id": srcObj.GetID(), + "newname": newName, + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + newPath := stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName) + if obj, ok := srcObj.(*model.ObjThumb); ok { + obj.SetPath(newPath) + obj.Name = newName + obj.Modified = time.Now() + return obj, nil + } + return d.getByPath(ctx, newPath) +} + +func (d *BaiduYouth) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + newPath := stdpath.Join(dstDir.GetPath(), srcObj.GetName()) + _, err := d.manage(ctx, "copy", []base.Json{ + { + "dest": dstDir.GetPath(), + "newname": srcObj.GetName(), + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + if obj, ok := srcObj.(*model.ObjThumb); ok { + copied := *obj + copied.SetPath(newPath) + copied.Modified = time.Now() + return &copied, nil + } + // Youth copy returns success before /api/filemetas can resolve the new path. + // Avoid turning a successful copy into a false failure because the immediate + // post-copy lookup is temporarily unavailable. + return &model.Object{ + ID: newPath, + Path: newPath, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + HashInfo: srcObj.GetHash(), + }, nil +} + +func (d *BaiduYouth) Remove(ctx context.Context, obj model.Obj) error { + _, err := d.manage(ctx, "delete", []string{obj.GetPath()}) + return err +} + +func (d *BaiduYouth) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if stream.GetSize() < 1 { + return nil, ErrBaiduYouthEmptyFilesNotAllowed + } + + var ( + cache = stream.GetFile() + tmpF *os.File + err error + ) + if _, ok := cache.(io.ReaderAt); !ok { + tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + defer func() { + _ = tmpF.Close() + _ = os.Remove(tmpF.Name()) + }() + cache = tmpF + } + + streamSize := stream.GetSize() + sliceSize := DefaultSliceSize + count := int(streamSize / sliceSize) + lastBlockSize := streamSize % sliceSize + if lastBlockSize > 0 { + count++ + } else { + lastBlockSize = sliceSize + } + + const sliceMD5Size int64 = 256 * utils.KB + blockList := make([]string, 0, count) + byteSize := sliceSize + fileMd5H := md5.New() + sliceMd5H := md5.New() + sliceMd5H2 := md5.New() + sliceMd5H2Writer := utils.LimitWriter(sliceMd5H2, sliceMD5Size) + writers := []io.Writer{fileMd5H, sliceMd5H, sliceMd5H2Writer} + if tmpF != nil { + writers = append(writers, tmpF) + } + + written := int64(0) + for i := 1; i <= count; i++ { + if utils.IsCanceled(ctx) { + return nil, ctx.Err() + } + if i == count { + byteSize = lastBlockSize + } + n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) + written += n + if err != nil && err != io.EOF { + return nil, err + } + blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) + sliceMd5H.Reset() + } + + if tmpF != nil { + if written != streamSize { + return nil, errs.NewErr(errs.StreamIncomplete, "temp file size mismatch: %d != %d", written, streamSize) + } + if _, err = tmpF.Seek(0, io.SeekStart); err != nil { + return nil, err + } + } + + contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) + sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) + blockListStr, err := utils.Json.MarshalToString(blockList) + if err != nil { + return nil, err + } + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) + mtime := stream.ModTime().Unix() + ctime := stream.CreateTime().Unix() + + progressKey := d.uploadProgressKey() + precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, progressKey, contentMd5) + if !ok { + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) + if err != nil { + return nil, err + } + } + + if precreateResp.ReturnType >= 2 { + result := precreateResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil + } + + cacheReaderAt, ok := cache.(io.ReaderAt) + if !ok { + return nil, fmt.Errorf("cache object must implement io.ReaderAt") + } + +uploadLoop: + for attempt := 0; attempt < 2; attempt++ { + completed := count - len(precreateResp.BlockList) + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) || partseq < 0 { + continue + } + + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize + } + + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "partseq": strconv.Itoa(partseq), + "path": path, + "type": "tmpfile", + "uploadid": precreateResp.Uploadid, + } + if precreateResp.Uploadsign != "" { + params["uploadsign"] = precreateResp.Uploadsign + } + section := io.NewSectionReader(cacheReaderAt, offset, size) + if err := d.uploadSlice(ctx, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section)); err != nil { + return err + } + precreateResp.BlockList[i] = -1 + success := completed + int(threadG.Success()) + 1 + up(float64(success) * 100 / float64(count)) + return nil + }) + } + + err = threadG.Wait() + if err == nil { + break uploadLoop + } + + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(part int) bool { + return part >= 0 + }) + base.SaveUploadProgress(d, precreateResp, progressKey, contentMd5) + + if errors.Is(err, context.Canceled) { + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) + if err != nil { + return nil, err + } + if precreateResp.ReturnType >= 2 { + result := precreateResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil + } + base.SaveUploadProgress(d, precreateResp, progressKey, contentMd5) + continue uploadLoop + } + return nil, err + } + + var createResp CreateResp + _, err = d.createFile(ctx, path, stdpath.Dir(path), streamSize, precreateResp.Uploadid, precreateResp.Uploadsign, blockListStr, &createResp, mtime, ctime) + if err != nil { + return nil, err + } + + base.SaveUploadProgress(d, nil, progressKey, contentMd5) + result := createResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil +} + +var _ driver.Driver = (*BaiduYouth)(nil) +var _ driver.Getter = (*BaiduYouth)(nil) +var _ driver.MkdirResult = (*BaiduYouth)(nil) +var _ driver.MoveResult = (*BaiduYouth)(nil) +var _ driver.RenameResult = (*BaiduYouth)(nil) +var _ driver.CopyResult = (*BaiduYouth)(nil) +var _ driver.Remove = (*BaiduYouth)(nil) +var _ driver.PutResult = (*BaiduYouth)(nil) diff --git a/drivers/baidu_youth/meta.go b/drivers/baidu_youth/meta.go new file mode 100644 index 00000000000..7d5cb6a8698 --- /dev/null +++ b/drivers/baidu_youth/meta.go @@ -0,0 +1,40 @@ +package baidu_youth + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Cookie string `json:"cookie" required:"true"` + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + ForceProxy bool `json:"force_proxy" type:"bool" default:"true" help:"Proxy downloads through AList. Disable to redirect the browser to a fresh Baidu direct link."` + DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` + UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` +} + +const ( + UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" + UPLOAD_TIMEOUT = time.Minute * 30 + UPLOAD_RETRY_COUNT = 3 + UPLOAD_RETRY_WAIT_TIME = time.Second + UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 + DefaultSliceSize int64 = 4 * 1024 * 1024 +) + +var config = driver.Config{ + Name: "BaiduYouth", + DefaultRoot: "/", + OnlyProxy: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &BaiduYouth{} + }) +} diff --git a/drivers/baidu_youth/types.go b/drivers/baidu_youth/types.go new file mode 100644 index 00000000000..82024009403 --- /dev/null +++ b/drivers/baidu_youth/types.go @@ -0,0 +1,130 @@ +package baidu_youth + +import ( + "errors" + "path" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +var ( + ErrBaiduYouthEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu youth") +) + +type File struct { + Category int `json:"category"` + FsId int64 `json:"fs_id"` + Thumbs struct { + Url3 string `json:"url3"` + } `json:"thumbs"` + Size int64 `json:"size"` + Path string `json:"path"` + ServerFilename string `json:"server_filename"` + Md5 string `json:"md5"` + Isdir int `json:"isdir"` + ServerCtime int64 `json:"server_ctime"` + ServerMtime int64 `json:"server_mtime"` + LocalMtime int64 `json:"local_mtime"` + LocalCtime int64 `json:"local_ctime"` + Ctime int64 `json:"ctime"` + Mtime int64 `json:"mtime"` + Dlink string `json:"dlink"` +} + +func fileToObj(f File) *model.ObjThumb { + if f.ServerFilename == "" { + f.ServerFilename = path.Base(f.Path) + } + if f.ServerCtime == 0 { + if f.LocalCtime != 0 { + f.ServerCtime = f.LocalCtime + } else { + f.ServerCtime = f.Ctime + } + } + if f.ServerMtime == 0 { + if f.LocalMtime != 0 { + f.ServerMtime = f.LocalMtime + } else { + f.ServerMtime = f.Mtime + } + } + return &model.ObjThumb{ + Object: model.Object{ + ID: strconv.FormatInt(f.FsId, 10), + Path: f.Path, + Name: f.ServerFilename, + Size: f.Size, + Modified: time.Unix(f.ServerMtime, 0), + Ctime: time.Unix(f.ServerCtime, 0), + IsFolder: f.Isdir == 1, + HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, + } +} + +type ListResp struct { + Errno int `json:"errno"` + List []File `json:"list"` +} + +type FileMetaResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Info []File `json:"info"` + List []File `json:"list"` +} + +type LocateDownloadResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Path string `json:"path"` + URL string `json:"url"` +} + +type MediaInfoResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Info File `json:"info"` +} + +type CreateResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Info File `json:"info"` + File +} + +func (r CreateResp) ResultFile() File { + if r.Info.Path != "" || r.Info.FsId != 0 { + return r.Info + } + return r.File +} + +type PrecreateResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + ReturnType int `json:"return_type"` + Path string `json:"path"` + Uploadid string `json:"uploadid"` + Uploadsign string `json:"uploadsign"` + BlockList []int `json:"block_list"` + Info File `json:"info"` + File +} + +func (r PrecreateResp) ResultFile() File { + if r.Info.Path != "" || r.Info.FsId != 0 { + return r.Info + } + return r.File +} diff --git a/drivers/baidu_youth/util.go b/drivers/baidu_youth/util.go new file mode 100644 index 00000000000..b83518fe350 --- /dev/null +++ b/drivers/baidu_youth/util.go @@ -0,0 +1,606 @@ +package baidu_youth + +import ( + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/http" + stdpath "path" + "strconv" + "strings" + "time" + "unicode" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + panBaseURL = "https://pan.baidu.com" + panReferer = panBaseURL + "/" + panUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36" + youthReferer = panBaseURL + "/youth/pan/main#/index?category=all" + youthAppID = "250528" + uploadAppID = "25179614" + videoAPIChannel = "android_15_25010PN30C_bd-netdisk_1523a" + videoAPIDevUID = "0%1" +) + +func (d *BaiduYouth) normalizeURL(furl string) string { + if strings.HasPrefix(furl, "http://") || strings.HasPrefix(furl, "https://") { + return furl + } + return panBaseURL + furl +} + +func (d *BaiduYouth) commonHeaders() map[string]string { + return map[string]string{ + "Accept": "application/json, text/plain, */*", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Cache-Control": "no-cache", + "Cookie": d.Cookie, + "Origin": panBaseURL, + "Pragma": "no-cache", + "Referer": youthReferer, + "User-Agent": panUserAgent, + "X-Requested-With": "XMLHttpRequest", + } +} + +func youthQueryParams() map[string]string { + return map[string]string{ + "app_id": youthAppID, + "clienttype": "0", + "web": "1", + } +} + +func youthUploadQueryParams() map[string]string { + return map[string]string{ + "app_id": uploadAppID, + "channel": "chunlei", + "clienttype": "0", + "web": "1", + } +} + +func extractBaiduMessage(body []byte) string { + for _, path := range [][]interface{}{ + {"show_msg"}, + {"errmsg"}, + {"error_msg"}, + {"error_description"}, + {"message"}, + } { + msg := utils.Json.Get(body, path...).ToString() + if msg != "" { + return msg + } + } + return "" +} + +func (d *BaiduYouth) doRequest(furl string, method string, defaultQuery map[string]string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R().SetHeaders(d.commonHeaders()) + if defaultQuery != nil { + req.SetQueryParams(defaultQuery) + } + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, d.normalizeURL(furl)) + if err != nil { + return nil, err + } + body := res.Body() + if errno := utils.Json.Get(body, "errno").ToInt(); errno != 0 { + msg := extractBaiduMessage(body) + if errno == -6 && msg == "" { + msg = "cookie expired or invalid" + } + if msg == "" { + msg = "request failed" + } + return nil, fmt.Errorf("[baidu_youth] %s (errno=%d)", msg, errno) + } + return body, nil +} + +func (d *BaiduYouth) get(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, youthQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) postForm(ctx context.Context, pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodPost, youthQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + req.SetFormData(form) + }, resp) +} + +func (d *BaiduYouth) getUpload(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, youthUploadQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) postUploadForm(ctx context.Context, pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodPost, youthUploadQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + req.SetFormData(form) + }, resp) +} + +func (d *BaiduYouth) getBare(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, nil, func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) getUserSK(ctx context.Context) (string, error) { + body, err := d.get(ctx, "/youth/api/report/user", map[string]string{ + "action": "sapi_auth", + "timestamp": strconv.FormatInt(time.Now().UnixMilli(), 10), + }, nil) + if err != nil { + return "", err + } + return utils.Json.Get(body, "uinfo").ToString(), nil +} + +func (d *BaiduYouth) getUserSession(ctx context.Context) (int64, string, string, error) { + body, err := d.get(ctx, "/youth/api/user/getinfo", map[string]string{ + "need_selfinfo": "1", + }, nil) + if err != nil { + return 0, "", "", err + } + uk := int64(utils.Json.Get(body, "records", 0, "uk").ToInt()) + bdstoken := utils.Json.Get(body, "records", 0, "bdstoken").ToString() + sk := utils.Json.Get(body, "records", 0, "sk").ToString() + if bdstoken == "" || uk == 0 || sk == "" { + body, err = d.getBare(ctx, "/api/gettemplatevariable", map[string]string{ + "fields": `["bdstoken","uk","sk"]`, + }, nil) + if err != nil { + return 0, "", "", err + } + if uk == 0 { + uk = int64(utils.Json.Get(body, "result", "uk").ToInt()) + } + if bdstoken == "" { + bdstoken = utils.Json.Get(body, "result", "bdstoken").ToString() + } + if sk == "" { + sk = utils.Json.Get(body, "result", "sk").ToString() + } + } + if sk == "" { + sk, _ = d.getUserSK(ctx) + } + if bdstoken == "" { + return 0, "", "", fmt.Errorf("failed to get bdstoken from baidu youth cookie") + } + if uk == 0 { + return 0, "", "", fmt.Errorf("failed to get uk from baidu youth cookie") + } + return uk, bdstoken, sk, nil +} + +func (d *BaiduYouth) getFiles(ctx context.Context, dir string) ([]File, error) { + page := 1 + num := 1000 + params := map[string]string{ + "dir": dir, + } + if d.OrderBy != "" { + params["order"] = d.OrderBy + if d.OrderDirection == "desc" { + params["desc"] = "1" + } else { + params["desc"] = "0" + } + } + files := make([]File, 0) + for { + params["page"] = strconv.Itoa(page) + params["num"] = strconv.Itoa(num) + var resp ListResp + _, err := d.get(ctx, "/youth/api/list", params, &resp) + if err != nil { + return nil, err + } + if len(resp.List) == 0 { + return files, nil + } + files = append(files, resp.List...) + if len(resp.List) < num { + return files, nil + } + page++ + } +} + +func (d *BaiduYouth) getFileByPath(ctx context.Context, path string) (File, error) { + if path == "/" { + return File{ + Path: "/", + ServerFilename: "/", + Isdir: 1, + }, nil + } + target, err := utils.Json.MarshalToString([]string{path}) + if err != nil { + return File{}, err + } + var resp FileMetaResp + _, err = d.get(ctx, "/api/filemetas", map[string]string{ + "target": target, + }, &resp) + if err != nil { + return File{}, err + } + if len(resp.Info) > 0 { + return resp.Info[0], nil + } + if len(resp.List) > 0 { + return resp.List[0], nil + } + return File{}, errs.NewErr(errs.ObjectNotFound, "baidu youth path not found: %s", path) +} + +func (d *BaiduYouth) getByPath(ctx context.Context, path string) (model.Obj, error) { + if path == "/" { + return &model.Object{ + Path: "/", + Name: "/", + IsFolder: true, + }, nil + } + file, err := d.getFileByPath(ctx, path) + if err != nil { + return nil, err + } + return fileToObj(file), nil +} + +func (d *BaiduYouth) linkOfficial(ctx context.Context, file model.Obj) (*model.Link, error) { + return d.buildDownloadLink(ctx, file) +} + +func (d *BaiduYouth) linkCrack(ctx context.Context, file model.Obj) (*model.Link, error) { + return d.buildDownloadLink(ctx, file) +} + +func (d *BaiduYouth) downloadHeaders() http.Header { + return http.Header{ + "Accept": []string{"*/*"}, + "Accept-Language": []string{"zh-CN,zh;q=0.9,en;q=0.8"}, + "Cache-Control": []string{"no-cache"}, + "Pragma": []string{"no-cache"}, + "Referer": []string{panReferer}, + "User-Agent": []string{panUserAgent}, + } +} + +func nextDPLogID() string { + return strconv.FormatInt(time.Now().UnixNano(), 10) +} + +func (d *BaiduYouth) getCurrentUserSK(ctx context.Context) (string, error) { + sk, err := d.getUserSK(ctx) + if err == nil && sk != "" { + return sk, nil + } + if d.sk != "" { + return d.sk, nil + } + if err != nil { + return "", err + } + return "", fmt.Errorf("baidu youth sk is empty") +} + +func (d *BaiduYouth) locatedownloadRand(sk string, nowMilli int64) string { + sum := sha1.Sum([]byte(strconv.FormatInt(d.uk, 10) + sk + strconv.FormatInt(nowMilli, 10) + "0")) + return hex.EncodeToString(sum[:]) +} + +func (d *BaiduYouth) locatedownloadSign(fileMD5 string, fileID string, nowMilli int64) string { + sum := md5.Sum([]byte(fileMD5 + "_" + strconv.FormatInt(d.uk, 10) + "_" + fileID + "_" + strconv.FormatInt(nowMilli, 10))) + return hex.EncodeToString(sum[:]) +} + +func (d *BaiduYouth) resolveDownloadMeta(ctx context.Context, file model.Obj) (string, string, string, error) { + parentDir := stdpath.Dir(file.GetPath()) + if parentDir == "." { + parentDir = "/" + } + + files, err := d.getFiles(ctx, parentDir) + if err != nil { + return "", "", "", err + } + for _, listedFile := range files { + if listedFile.Path != file.GetPath() { + continue + } + fileID := strconv.FormatInt(listedFile.FsId, 10) + if listedFile.Path != "" && fileID != "" && listedFile.Md5 != "" { + return listedFile.Path, fileID, listedFile.Md5, nil + } + return "", "", "", fmt.Errorf("baidu youth list metadata incomplete for %s", file.GetPath()) + } + return "", "", "", errs.NewErr(errs.ObjectNotFound, "baidu youth list metadata not found: %s", file.GetPath()) +} + +func normalizeLocatedownloadURL(rawURL string) (string, error) { + if rawURL == "" { + return "", fmt.Errorf("baidu youth locatedownload url is empty") + } + if !strings.Contains(rawURL, "response-cache-control=") { + sep := "&" + if !strings.Contains(rawURL, "?") { + sep = "?" + } + rawURL += sep + "response-cache-control=private" + } + return rawURL, nil +} + +func (d *BaiduYouth) getMediaInfoDLink(ctx context.Context, file model.Obj) (string, error) { + path, fileID, _, err := d.resolveDownloadMeta(ctx, file) + if err != nil { + return "", err + } + + var resp MediaInfoResp + _, err = d.doRequest("/youth/api/mediainfo", http.MethodGet, nil, func(req *resty.Request) { + req.SetContext(ctx) + req.SetQueryParams(map[string]string{ + "channel": videoAPIChannel, + "clienttype": "1", + "devuid": videoAPIDevUID, + "dlink": "1", + "fs_id": fileID, + "media": "1", + "nom3u8": "1", + "origin": "dlna", + "path": path, + "type": "VideoURL", + }) + }, &resp) + if err != nil { + return "", err + } + if resp.Info.Dlink == "" { + return "", fmt.Errorf("baidu youth video dlink not found for %s", path) + } + return resp.Info.Dlink, nil +} + +func (d *BaiduYouth) buildVideoLink(ctx context.Context, file model.Obj) (*model.Link, error) { + dlink, err := d.getMediaInfoDLink(ctx, file) + if err != nil { + return nil, err + } + return &model.Link{ + URL: dlink, + Header: http.Header{ + "Referer": []string{panReferer}, + }, + }, nil +} + +func (d *BaiduYouth) requestLocateDownloadURL(ctx context.Context, path string, fileID string, fileMD5 string, sk string) (string, error) { + nowMilli := time.Now().UnixMilli() + + var resp LocateDownloadResp + _, err := d.get(ctx, "/youth/api/locatedownload", map[string]string{ + "devuid": "0", + "dp-logid": nextDPLogID(), + "path": path, + "rand": d.locatedownloadRand(sk, nowMilli), + "sign": d.locatedownloadSign(fileMD5, fileID, nowMilli), + "time": strconv.FormatInt(nowMilli, 10), + }, &resp) + if err != nil { + return "", err + } + if resp.URL == "" { + return "", fmt.Errorf("baidu youth locatedownload url not found for %s", path) + } + return normalizeLocatedownloadURL(resp.URL) +} + +func (d *BaiduYouth) getLocateDownloadURL(ctx context.Context, file model.Obj) (string, error) { + path, fileID, fileMD5, err := d.resolveDownloadMeta(ctx, file) + if err != nil { + return "", err + } + + sk, err := d.getCurrentUserSK(ctx) + if err != nil { + return "", err + } + downloadURL, err := d.requestLocateDownloadURL(ctx, path, fileID, fileMD5, sk) + if err == nil { + return downloadURL, nil + } + + if !strings.Contains(err.Error(), "errno=-30006") { + return "", err + } + sk, refreshErr := d.getUserSK(ctx) + if refreshErr != nil { + return "", err + } + if sk == "" { + return "", err + } + return d.requestLocateDownloadURL(ctx, path, fileID, fileMD5, sk) +} + +func (d *BaiduYouth) buildDownloadLink(ctx context.Context, file model.Obj) (*model.Link, error) { + downloadURL, err := d.getLocateDownloadURL(ctx, file) + if err != nil { + return nil, err + } + return &model.Link{ + URL: downloadURL, + Header: d.downloadHeaders(), + }, nil +} + +func (d *BaiduYouth) manage(ctx context.Context, opera string, filelist any) ([]byte, error) { + marshal, err := utils.Json.MarshalToString(filelist) + if err != nil { + return nil, err + } + return d.postForm(ctx, "/youth/api/filemanager", map[string]string{ + "async": "0", + "bdstoken": d.bdstoken, + "onnest": "fail", + "opera": opera, + }, map[string]string{ + "filelist": marshal, + "ondup": "fail", + }, nil) +} + +func (d *BaiduYouth) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + form := map[string]string{ + "autoinit": "1", + "block_list": blockListStr, + "isdir": "0", + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "target_path": stdpath.Dir(path), + } + if contentMd5 != "" { + form["content-md5"] = contentMd5 + } + if sliceMd5 != "" { + form["slice-md5"] = sliceMd5 + } + joinTime(form, ctime, mtime) + var resp PrecreateResp + _, err := d.postUploadForm(ctx, "/youth/api/precreate", map[string]string{ + "bdstoken": d.bdstoken, + }, form, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *BaiduYouth) createFile(ctx context.Context, path, targetPath string, size int64, uploadID, uploadSign, blockList string, resp interface{}, mtime, ctime int64) ([]byte, error) { + form := map[string]string{ + "block_list": blockList, + "path": path, + "size": strconv.FormatInt(size, 10), + "target_path": targetPath, + "uploadid": uploadID, + } + if uploadSign != "" { + form["uploadsign"] = uploadSign + } + joinTime(form, ctime, mtime) + return d.postUploadForm(ctx, "/youth/api/create", map[string]string{ + "bdstoken": d.bdstoken, + "isdir": "0", + }, form, resp) +} + +func joinTime(form map[string]string, ctime, mtime int64) { + if ctime != 0 { + form["local_ctime"] = strconv.FormatInt(ctime, 10) + } + if mtime != 0 { + form["local_mtime"] = strconv.FormatInt(mtime, 10) + } +} + +func (d *BaiduYouth) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { + res, err := d.upClient.R(). + SetContext(ctx). + SetHeaders(d.commonHeaders()). + SetQueryParams(youthUploadQueryParams()). + SetQueryParams(params). + SetFileReader("file", fileName, file). + Post(d.UploadAPI + "/rest/2.0/pcs/superfile2") + if err != nil { + return err + } + body := res.Body() + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() + lower := strings.ToLower(string(body)) + if strings.Contains(lower, "uploadid") && (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + if errCode != 0 || errNo != 0 { + msg := extractBaiduMessage(body) + if msg == "" { + msg = "error uploading to baidu youth" + } + return errs.NewErr(errs.StreamIncomplete, "%s: %s", msg, string(body)) + } + return nil +} + +func (d *BaiduYouth) uploadProgressKey() string { + if d.uk != 0 { + return strconv.FormatInt(d.uk, 10) + } + sum := md5.Sum([]byte(d.Cookie)) + return hex.EncodeToString(sum[:]) +} + +func DecryptMd5(encryptMd5 string) string { + if encryptMd5 == "" { + return "" + } + if _, err := hex.DecodeString(encryptMd5); err == nil { + return encryptMd5 + } + + var out strings.Builder + out.Grow(len(encryptMd5)) + for i, n := 0, int64(0); i < len(encryptMd5); i++ { + if i == 9 { + n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') + } else { + n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) + } + out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) + } + + encryptMd5 = out.String() + return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] +} diff --git a/drivers/bitqiu/driver.go b/drivers/bitqiu/driver.go new file mode 100644 index 00000000000..048377fee93 --- /dev/null +++ b/drivers/bitqiu/driver.go @@ -0,0 +1,767 @@ +package bitqiu + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http/cookiejar" + "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + streamPkg "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" +) + +const ( + baseURL = "https://pan.bitqiu.com" + loginURL = baseURL + "/loginServer/login" + userInfoURL = baseURL + "/user/getInfo" + listURL = baseURL + "/apiToken/cfi/fs/resources/pages" + uploadInitializeURL = baseURL + "/apiToken/cfi/fs/upload/v2/initialize" + uploadCompleteURL = baseURL + "/apiToken/cfi/fs/upload/v2/complete" + downloadURL = baseURL + "/download/getUrl" + createDirURL = baseURL + "/resource/create" + moveResourceURL = baseURL + "/resource/remove" + renameResourceURL = baseURL + "/resource/rename" + copyResourceURL = baseURL + "/apiToken/cfi/fs/async/copy" + copyManagerURL = baseURL + "/apiToken/cfi/fs/async/manager" + deleteResourceURL = baseURL + "/resource/delete" + + successCode = "10200" + uploadSuccessCode = "30010" + copySubmittedCode = "10300" + orgChannel = "default|default|default" +) + +const ( + copyPollInterval = time.Second + copyPollMaxAttempts = 60 + chunkSize = int64(1 << 20) +) + +const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36" + +type BitQiu struct { + model.Storage + Addition + + client *resty.Client + userID string +} + +func (d *BitQiu) Config() driver.Config { + return config +} + +func (d *BitQiu) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *BitQiu) Init(ctx context.Context) error { + if d.Addition.UserPlatform == "" { + d.Addition.UserPlatform = uuid.NewString() + op.MustSaveDriverStorage(d) + } + + if d.client == nil { + jar, err := cookiejar.New(nil) + if err != nil { + return err + } + d.client = base.NewRestyClient() + d.client.SetBaseURL(baseURL) + d.client.SetCookieJar(jar) + } + d.client.SetHeader("user-agent", d.userAgent()) + + return d.login(ctx) +} + +func (d *BitQiu) Drop(ctx context.Context) error { + d.client = nil + d.userID = "" + return nil +} + +func (d *BitQiu) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + parentID := d.resolveParentID(dir) + dirPath := "" + if dir != nil { + dirPath = dir.GetPath() + } + pageSize := d.pageSize() + orderType := d.orderType() + desc := d.orderDesc() + + var results []model.Obj + page := 1 + for { + form := map[string]string{ + "parentId": parentID, + "limit": strconv.Itoa(pageSize), + "orderType": orderType, + "desc": desc, + "model": "1", + "userId": d.userID, + "currentPage": strconv.Itoa(page), + "page": strconv.Itoa(page), + "org_channel": orgChannel, + } + var resp Response[ResourcePage] + if err := d.postForm(ctx, listURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != successCode { + if resp.Code == "10401" || resp.Code == "10404" { + if err := d.login(ctx); err != nil { + return nil, err + } + continue + } + return nil, fmt.Errorf("list failed: %s", resp.Message) + } + + objs, err := utils.SliceConvert(resp.Data.Data, func(item Resource) (model.Obj, error) { + return item.toObject(parentID, dirPath) + }) + if err != nil { + return nil, err + } + results = append(results, objs...) + + if !resp.Data.HasNext || len(resp.Data.Data) == 0 { + break + } + page++ + } + + return results, nil +} + +func (d *BitQiu) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + form := map[string]string{ + "fileIds": file.GetID(), + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[DownloadData] + if err := d.postForm(ctx, downloadURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + if resp.Data.URL == "" { + return nil, fmt.Errorf("empty download url returned") + } + return &model.Link{URL: resp.Data.URL}, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("get link failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("get link failed: retry limit reached") +} + +func (d *BitQiu) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + parentID := d.resolveParentID(parentDir) + parentPath := "" + if parentDir != nil { + parentPath = parentDir.GetPath() + } + form := map[string]string{ + "parentId": parentID, + "name": dirName, + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[CreateDirData] + if err := d.postForm(ctx, createDirURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + newParentID := parentID + if resp.Data.ParentID != "" { + newParentID = resp.Data.ParentID + } + name := resp.Data.Name + if name == "" { + name = dirName + } + resource := Resource{ + ResourceID: resp.Data.DirID, + ResourceType: 1, + Name: name, + ParentID: newParentID, + } + obj, err := resource.toObject(newParentID, parentPath) + if err != nil { + return nil, err + } + if o, ok := obj.(*Object); ok { + o.ParentID = newParentID + } + return obj, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("create folder failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("create folder failed: retry limit reached") +} + +func (d *BitQiu) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + targetParentID := d.resolveParentID(dstDir) + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "parentId": targetParentID, + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["dirIds"] = srcObj.GetID() + } else { + form["fileIds"] = srcObj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, moveResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + dstPath := "" + if dstDir != nil { + dstPath = dstDir.GetPath() + } + if setter, ok := srcObj.(model.SetPath); ok { + setter.SetPath(path.Join(dstPath, srcObj.GetName())) + } + if o, ok := srcObj.(*Object); ok { + o.ParentID = targetParentID + } + return srcObj, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("move failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("move failed: retry limit reached") +} + +func (d *BitQiu) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + form := map[string]string{ + "resourceId": srcObj.GetID(), + "name": newName, + "type": "0", + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["type"] = "1" + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, renameResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + return updateObjectName(srcObj, newName), nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("rename failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("rename failed: retry limit reached") +} + +func (d *BitQiu) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + targetParentID := d.resolveParentID(dstDir) + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "parentId": targetParentID, + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["dirIds"] = srcObj.GetID() + } else { + form["fileIds"] = srcObj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, copyResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode, copySubmittedCode: + return d.waitForCopiedObject(ctx, srcObj, dstDir) + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("copy failed: %s", resp.Message) + } + } + + return nil, fmt.Errorf("copy failed: retry limit reached") +} + +func (d *BitQiu) Remove(ctx context.Context, obj model.Obj) error { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return err + } + } + + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "org_channel": orgChannel, + } + if obj.IsDir() { + form["dirIds"] = obj.GetID() + } else { + form["fileIds"] = obj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, deleteResourceURL, form, &resp); err != nil { + return err + } + switch resp.Code { + case successCode: + return nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return err + } + default: + return fmt.Errorf("remove failed: %s", resp.Message) + } + } + return fmt.Errorf("remove failed: retry limit reached") +} + +func (d *BitQiu) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + up(0) + tmpFile, md5sum, err := streamPkg.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return nil, err + } + defer tmpFile.Close() + + parentID := d.resolveParentID(dstDir) + parentPath := "" + if dstDir != nil { + parentPath = dstDir.GetPath() + } + form := map[string]string{ + "parentId": parentID, + "name": file.GetName(), + "size": strconv.FormatInt(file.GetSize(), 10), + "hash": md5sum, + "sampleMd5": md5sum, + "org_channel": orgChannel, + } + var resp Response[json.RawMessage] + if err = d.postForm(ctx, uploadInitializeURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != uploadSuccessCode { + switch resp.Code { + case successCode: + var initData UploadInitData + if err := json.Unmarshal(resp.Data, &initData); err != nil { + return nil, fmt.Errorf("parse upload init response failed: %w", err) + } + serverCode, err := d.uploadFileInChunks(ctx, tmpFile, file.GetSize(), md5sum, initData, up) + if err != nil { + return nil, err + } + obj, err := d.completeChunkUpload(ctx, initData, parentID, parentPath, file.GetName(), file.GetSize(), md5sum, serverCode) + if err != nil { + return nil, err + } + up(100) + return obj, nil + default: + return nil, fmt.Errorf("upload failed: %s", resp.Message) + } + } + + var resource Resource + if err := json.Unmarshal(resp.Data, &resource); err != nil { + return nil, fmt.Errorf("parse upload response failed: %w", err) + } + obj, err := resource.toObject(parentID, parentPath) + if err != nil { + return nil, err + } + up(100) + return obj, nil +} + +func (d *BitQiu) uploadFileInChunks(ctx context.Context, tmpFile model.File, size int64, md5sum string, initData UploadInitData, up driver.UpdateProgress) (string, error) { + if d.client == nil { + return "", fmt.Errorf("client not initialized") + } + if size <= 0 { + return "", fmt.Errorf("invalid file size") + } + buf := make([]byte, chunkSize) + offset := int64(0) + var finishedFlag string + + for offset < size { + chunkLen := chunkSize + remaining := size - offset + if remaining < chunkLen { + chunkLen = remaining + } + + reader := io.NewSectionReader(tmpFile, offset, chunkLen) + chunkBuf := buf[:chunkLen] + if _, err := io.ReadFull(reader, chunkBuf); err != nil { + return "", fmt.Errorf("read chunk failed: %w", err) + } + + headers := map[string]string{ + "accept": "*/*", + "content-type": "application/octet-stream", + "appid": initData.AppID, + "token": initData.Token, + "userid": strconv.FormatInt(initData.UserID, 10), + "serialnumber": initData.SerialNumber, + "hash": md5sum, + "len": strconv.FormatInt(chunkLen, 10), + "offset": strconv.FormatInt(offset, 10), + "user-agent": d.userAgent(), + } + + var chunkResp ChunkUploadResponse + req := d.client.R(). + SetContext(ctx). + SetHeaders(headers). + SetBody(chunkBuf). + SetResult(&chunkResp) + + if _, err := req.Post(initData.UploadURL); err != nil { + return "", err + } + if chunkResp.ErrCode != 0 { + return "", fmt.Errorf("chunk upload failed with code %d", chunkResp.ErrCode) + } + finishedFlag = chunkResp.FinishedFlag + offset += chunkLen + up(float64(offset) * 100 / float64(size)) + } + + if finishedFlag == "" { + return "", fmt.Errorf("upload finished without server code") + } + return finishedFlag, nil +} + +func (d *BitQiu) completeChunkUpload(ctx context.Context, initData UploadInitData, parentID, parentPath, name string, size int64, md5sum, serverCode string) (model.Obj, error) { + form := map[string]string{ + "currentPage": "1", + "limit": "1", + "userId": strconv.FormatInt(initData.UserID, 10), + "status": "0", + "parentId": parentID, + "name": name, + "fileUid": initData.FileUID, + "fileSid": initData.FileSID, + "size": strconv.FormatInt(size, 10), + "serverCode": serverCode, + "snapTime": "", + "hash": md5sum, + "sampleMd5": md5sum, + "org_channel": orgChannel, + } + + var resp Response[Resource] + if err := d.postForm(ctx, uploadCompleteURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != successCode { + return nil, fmt.Errorf("complete upload failed: %s", resp.Message) + } + + return resp.Data.toObject(parentID, parentPath) +} + +func (d *BitQiu) login(ctx context.Context) error { + if d.client == nil { + return fmt.Errorf("client not initialized") + } + + form := map[string]string{ + "passport": d.Username, + "password": utils.GetMD5EncodeStr(d.Password), + "remember": "0", + "captcha": "", + "org_channel": orgChannel, + } + var resp Response[LoginData] + if err := d.postForm(ctx, loginURL, form, &resp); err != nil { + return err + } + if resp.Code != successCode { + return fmt.Errorf("login failed: %s", resp.Message) + } + d.userID = strconv.FormatInt(resp.Data.UserID, 10) + return d.ensureRootFolderID(ctx) +} + +func (d *BitQiu) ensureRootFolderID(ctx context.Context) error { + rootID := d.Addition.GetRootId() + if rootID != "" && rootID != "0" { + return nil + } + + form := map[string]string{ + "org_channel": orgChannel, + } + var resp Response[UserInfoData] + if err := d.postForm(ctx, userInfoURL, form, &resp); err != nil { + return err + } + if resp.Code != successCode { + return fmt.Errorf("get user info failed: %s", resp.Message) + } + if resp.Data.RootDirID == "" { + return fmt.Errorf("get user info failed: empty root dir id") + } + if d.Addition.RootFolderID != resp.Data.RootDirID { + d.Addition.RootFolderID = resp.Data.RootDirID + op.MustSaveDriverStorage(d) + } + return nil +} + +func (d *BitQiu) postForm(ctx context.Context, url string, form map[string]string, result interface{}) error { + if d.client == nil { + return fmt.Errorf("client not initialized") + } + req := d.client.R(). + SetContext(ctx). + SetHeaders(d.commonHeaders()). + SetFormData(form) + if result != nil { + req = req.SetResult(result) + } + _, err := req.Post(url) + return err +} + +func (d *BitQiu) waitForCopiedObject(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + expectedName := srcObj.GetName() + expectedIsDir := srcObj.IsDir() + var lastListErr error + + for attempt := 0; attempt < copyPollMaxAttempts; attempt++ { + if attempt > 0 { + if err := waitWithContext(ctx, copyPollInterval); err != nil { + return nil, err + } + } + + if err := d.checkCopyFailure(ctx); err != nil { + return nil, err + } + + obj, err := d.findObjectInDir(ctx, dstDir, expectedName, expectedIsDir) + if err != nil { + lastListErr = err + continue + } + if obj != nil { + return obj, nil + } + } + if lastListErr != nil { + return nil, lastListErr + } + return nil, fmt.Errorf("copy task timed out waiting for completion") +} + +func (d *BitQiu) checkCopyFailure(ctx context.Context) error { + form := map[string]string{ + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[AsyncManagerData] + if err := d.postForm(ctx, copyManagerURL, form, &resp); err != nil { + return err + } + switch resp.Code { + case successCode: + if len(resp.Data.FailTasks) > 0 { + return fmt.Errorf("copy failed: %s", resp.Data.FailTasks[0].ErrorMessage()) + } + return nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return err + } + default: + return fmt.Errorf("query copy status failed: %s", resp.Message) + } + } + return fmt.Errorf("query copy status failed: retry limit reached") +} + +func (d *BitQiu) findObjectInDir(ctx context.Context, dir model.Obj, name string, isDir bool) (model.Obj, error) { + objs, err := d.List(ctx, dir, model.ListArgs{}) + if err != nil { + return nil, err + } + for _, obj := range objs { + if obj.GetName() == name && obj.IsDir() == isDir { + return obj, nil + } + } + return nil, nil +} + +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (d *BitQiu) commonHeaders() map[string]string { + headers := map[string]string{ + "accept": "application/json, text/plain, */*", + "accept-language": "en-US,en;q=0.9", + "cache-control": "no-cache", + "pragma": "no-cache", + "user-platform": d.Addition.UserPlatform, + "x-kl-saas-ajax-request": "Ajax_Request", + "x-requested-with": "XMLHttpRequest", + "referer": baseURL + "/", + "origin": baseURL, + "user-agent": d.userAgent(), + } + return headers +} + +func (d *BitQiu) userAgent() string { + if ua := strings.TrimSpace(d.Addition.UserAgent); ua != "" { + return ua + } + return defaultUserAgent +} + +func (d *BitQiu) resolveParentID(dir model.Obj) string { + if dir != nil && dir.GetID() != "" { + return dir.GetID() + } + if root := d.Addition.GetRootId(); root != "" { + return root + } + return config.DefaultRoot +} + +func (d *BitQiu) pageSize() int { + if size, err := strconv.Atoi(d.Addition.PageSize); err == nil && size > 0 { + return size + } + return 24 +} + +func (d *BitQiu) orderType() string { + if d.Addition.OrderType != "" { + return d.Addition.OrderType + } + return "updateTime" +} + +func (d *BitQiu) orderDesc() string { + if d.Addition.OrderDesc { + return "1" + } + return "0" +} + +var _ driver.Driver = (*BitQiu)(nil) +var _ driver.PutResult = (*BitQiu)(nil) diff --git a/drivers/bitqiu/meta.go b/drivers/bitqiu/meta.go new file mode 100644 index 00000000000..63cb03344c4 --- /dev/null +++ b/drivers/bitqiu/meta.go @@ -0,0 +1,28 @@ +package bitqiu + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + UserPlatform string `json:"user_platform" help:"Optional device identifier; auto-generated if empty."` + OrderType string `json:"order_type" type:"select" options:"updateTime,createTime,name,size" default:"updateTime"` + OrderDesc bool `json:"order_desc"` + PageSize string `json:"page_size" default:"24" help:"Number of entries to request per page."` + UserAgent string `json:"user_agent" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"` +} + +var config = driver.Config{ + Name: "BitQiu", + DefaultRoot: "0", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &BitQiu{} + }) +} diff --git a/drivers/bitqiu/types.go b/drivers/bitqiu/types.go new file mode 100644 index 00000000000..8fbec989135 --- /dev/null +++ b/drivers/bitqiu/types.go @@ -0,0 +1,107 @@ +package bitqiu + +import "encoding/json" + +type Response[T any] struct { + Code string `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type LoginData struct { + UserID int64 `json:"userId"` +} + +type ResourcePage struct { + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` + TotalCount int `json:"totalCount"` + TotalPageCount int `json:"totalPageCount"` + Data []Resource `json:"data"` + HasNext bool `json:"hasNext"` +} + +type Resource struct { + ResourceID string `json:"resourceId"` + ResourceUID string `json:"resourceUid"` + ResourceType int `json:"resourceType"` + ParentID string `json:"parentId"` + Name string `json:"name"` + ExtName string `json:"extName"` + Size *json.Number `json:"size"` + CreateTime *string `json:"createTime"` + UpdateTime *string `json:"updateTime"` + FileMD5 string `json:"fileMd5"` +} + +type DownloadData struct { + URL string `json:"url"` + MD5 string `json:"md5"` + Size int64 `json:"size"` +} + +type UserInfoData struct { + RootDirID string `json:"rootDirId"` +} + +type CreateDirData struct { + DirID string `json:"dirId"` + Name string `json:"name"` + ParentID string `json:"parentId"` +} + +type AsyncManagerData struct { + WaitTasks []AsyncTask `json:"waitTaskList"` + RunningTasks []AsyncTask `json:"runningTaskList"` + SuccessTasks []AsyncTask `json:"successTaskList"` + FailTasks []AsyncTask `json:"failTaskList"` + TaskList []AsyncTask `json:"taskList"` +} + +type AsyncTask struct { + TaskID string `json:"taskId"` + Status int `json:"status"` + ErrorMsg string `json:"errorMsg"` + Message string `json:"message"` + Result *AsyncTaskInfo `json:"result"` + TargetName string `json:"targetName"` + TargetDirID string `json:"parentId"` +} + +type AsyncTaskInfo struct { + Resource Resource `json:"resource"` + DirID string `json:"dirId"` + FileID string `json:"fileId"` + Name string `json:"name"` + ParentID string `json:"parentId"` +} + +func (t AsyncTask) ErrorMessage() string { + if t.ErrorMsg != "" { + return t.ErrorMsg + } + if t.Message != "" { + return t.Message + } + return "unknown error" +} + +type UploadInitData struct { + Name string `json:"name"` + Size int64 `json:"size"` + Token string `json:"token"` + FileUID string `json:"fileUid"` + FileSID string `json:"fileSid"` + ParentID string `json:"parentId"` + UserID int64 `json:"userId"` + SerialNumber string `json:"serialNumber"` + UploadURL string `json:"uploadUrl"` + AppID string `json:"appId"` +} + +type ChunkUploadResponse struct { + ErrCode int `json:"errCode"` + Offset int64 `json:"offset"` + Finished int `json:"finished"` + FinishedFlag string `json:"finishedFlag"` +} diff --git a/drivers/bitqiu/util.go b/drivers/bitqiu/util.go new file mode 100644 index 00000000000..bccd6815e9b --- /dev/null +++ b/drivers/bitqiu/util.go @@ -0,0 +1,102 @@ +package bitqiu + +import ( + "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Object struct { + model.Object + ParentID string +} + +func (r Resource) toObject(parentID, parentPath string) (model.Obj, error) { + id := r.ResourceID + if id == "" { + id = r.ResourceUID + } + obj := &Object{ + Object: model.Object{ + ID: id, + Name: r.Name, + IsFolder: r.ResourceType == 1, + }, + ParentID: parentID, + } + if r.Size != nil { + if size, err := (*r.Size).Int64(); err == nil { + obj.Size = size + } + } + if ct := parseBitQiuTime(r.CreateTime); !ct.IsZero() { + obj.Ctime = ct + } + if mt := parseBitQiuTime(r.UpdateTime); !mt.IsZero() { + obj.Modified = mt + } + if r.FileMD5 != "" { + obj.HashInfo = utils.NewHashInfo(utils.MD5, strings.ToLower(r.FileMD5)) + } + obj.SetPath(path.Join(parentPath, obj.Name)) + return obj, nil +} + +func parseBitQiuTime(value *string) time.Time { + if value == nil { + return time.Time{} + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return time.Time{} + } + if ts, err := time.ParseInLocation("2006-01-02 15:04:05", trimmed, time.Local); err == nil { + return ts + } + return time.Time{} +} + +func updateObjectName(obj model.Obj, newName string) model.Obj { + newPath := path.Join(parentPathOf(obj.GetPath()), newName) + + switch o := obj.(type) { + case *Object: + o.Name = newName + o.Object.Name = newName + o.SetPath(newPath) + return o + case *model.Object: + o.Name = newName + o.SetPath(newPath) + return o + } + + if setter, ok := obj.(model.SetPath); ok { + setter.SetPath(newPath) + } + + return &model.Object{ + ID: obj.GetID(), + Path: newPath, + Name: newName, + Size: obj.GetSize(), + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), + } +} + +func parentPathOf(p string) string { + if p == "" { + return "" + } + dir := path.Dir(p) + if dir == "." { + return "" + } + return dir +} diff --git a/drivers/chunker/driver.go b/drivers/chunker/driver.go new file mode 100644 index 00000000000..f05d4a9f89a --- /dev/null +++ b/drivers/chunker/driver.go @@ -0,0 +1,459 @@ +package chunker + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "hash" + "io" + "path" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Chunker) Config() driver.Config { + return config +} + +func (d *Chunker) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Chunker) Init(ctx context.Context) error { + if d.ChunkSize == 0 { + d.ChunkSize = defaultChunkSize + } + if d.StartFrom == 0 { + d.StartFrom = defaultStartFrom + } + d.NameFormat = utils.GetNoneEmpty(d.NameFormat, defaultChunkNameFmt) + d.MetaFormat = utils.GetNoneEmpty(d.MetaFormat, defaultMetaFormat) + d.HashType = utils.GetNoneEmpty(d.HashType, defaultHashType) + + if err := d.setChunkNameFormat(d.NameFormat); err != nil { + return fmt.Errorf("invalid name_format: %w", err) + } + if err := d.validateOptions(); err != nil { + return err + } + + targetPaths := d.configuredRemotePaths() + d.remoteTargets = make([]remoteTarget, 0, len(targetPaths)) + for _, targetPath := range targetPaths { + storage, err := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if err != nil { + return fmt.Errorf("can't find remote storage %q: %w", targetPath, err) + } + d.remoteTargets = append(d.remoteTargets, remoteTarget{ + MountPath: targetPath, + Storage: storage, + }) + } + if len(d.remoteTargets) == 0 { + return fmt.Errorf("can't find remote storage: %w", errs.ObjectNotFound) + } + d.remoteStorage = d.remoteTargets[0].Storage + return nil +} + +func (d *Chunker) Drop(ctx context.Context) error { + d.remoteStorage = nil + d.remoteTargets = nil + return nil +} + +func (d *Chunker) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.listDirObjects(ctx, dir.GetPath(), args.Refresh) +} + +func (d *Chunker) Get(ctx context.Context, pathStr string) (model.Obj, error) { + if utils.PathEqual(pathStr, "/") { + return &model.Object{ + Name: "Root", + Path: "/", + IsFolder: true, + }, nil + } + parent, name := path.Split(utils.FixAndCleanPath(pathStr)) + if parent == "" { + parent = "/" + } + objs, err := d.listDirObjects(ctx, parent, false) + if err != nil { + return nil, err + } + for _, obj := range objs { + if obj.GetName() == name { + return obj, nil + } + } + return nil, errs.ObjectNotFound +} + +func (d *Chunker) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj := d.linkedObject(file) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + actualPath, err := d.getActualPathForRemoteOnTarget(file.GetPath(), remoteIndex) + if err != nil { + return nil, fmt.Errorf("failed to convert path to remote path: %w", err) + } + link, _, err := op.Link(ctx, d.remoteTargets[remoteIndex].Storage, actualPath, args) + return link, err + } + + linkedParts := make([]linkedPart, 0, len(obj.Parts)) + baseClosers := utils.EmptyClosers() + for _, part := range obj.Parts { + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID, part.RemoteIndex) + if err != nil { + return nil, fmt.Errorf("failed to convert chunk path: %w", err) + } + link, _, err := op.Link(ctx, d.remoteTargets[part.RemoteIndex].Storage, actualPath, args) + if err != nil { + return nil, err + } + if link.MFile != nil { + baseClosers.Add(link.MFile) + } + if link.RangeReadCloser != nil { + baseClosers.Add(link.RangeReadCloser) + } + linkedParts = append(linkedParts, linkedPart{ + part: part, + link: link, + }) + } + + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{ + RangeReader: func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return d.openChunkReader(ctx, linkedParts, obj.GetSize(), httpRange) + }, + Closers: baseClosers, + }, + }, nil +} + +func (d *Chunker) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.ensureDirOnAllTargets(ctx, path.Join(parentDir.GetPath(), dirName)) +} + +func (d *Chunker) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj.IsDir() { + return d.moveDirAcrossTargets(ctx, srcObj.GetPath(), dstDir.GetPath()) + } + obj := d.linkedObject(srcObj) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDir.GetPath()); err != nil { + return err + } + srcRemoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Move(ctx, d.remoteTargets[remoteIndex].Storage, srcRemoteActualPath, dstRemoteActualPath) + } + + ensuredTargets := map[int]struct{}{} + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := ensuredTargets[location.RemoteIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, location.RemoteIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[location.RemoteIndex] = struct{}{} + } + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) + if err != nil { + return err + } + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), location.RemoteIndex) + if err != nil { + return err + } + if err := op.Move(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath, dstRemoteActualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if srcObj.IsDir() { + return d.renameDirAcrossTargets(ctx, srcObj.GetPath(), newName) + } + obj := d.linkedObject(srcObj) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + remoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Rename(ctx, d.remoteTargets[remoteIndex].Storage, remoteActualPath, newName) + } + + for _, part := range obj.Parts { + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID, part.RemoteIndex) + if err != nil { + return err + } + newChunkName := d.chunkPartBaseName(path.Join(path.Dir(obj.GetPath()), newName), part.No, part.XactID) + if err := op.Rename(ctx, d.remoteTargets[part.RemoteIndex].Storage, actualPath, newChunkName); err != nil { + return err + } + } + if obj.UsesMeta { + actualPath, err := d.getActualPathForRemoteOnTarget(obj.GetPath(), obj.MainRemoteIndex) + if err != nil { + return err + } + if err := op.Rename(ctx, d.remoteTargets[obj.MainRemoteIndex].Storage, actualPath, newName); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj.IsDir() { + return d.copyDirAcrossTargets(ctx, srcObj.GetPath(), dstDir.GetPath()) + } + obj := d.linkedObject(srcObj) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDir.GetPath()); err != nil { + return err + } + srcRemoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Copy(ctx, d.remoteTargets[remoteIndex].Storage, srcRemoteActualPath, dstRemoteActualPath) + } + + ensuredTargets := map[int]struct{}{} + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := ensuredTargets[location.RemoteIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, location.RemoteIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[location.RemoteIndex] = struct{}{} + } + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) + if err != nil { + return err + } + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), location.RemoteIndex) + if err != nil { + return err + } + if err := op.Copy(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath, dstRemoteActualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + return d.removeDirAcrossTargets(ctx, obj.GetPath()) + } + chunkedObj := d.linkedObject(obj) + if chunkedObj == nil || !chunkedObj.Chunked { + remoteIndex := 0 + if chunkedObj != nil { + remoteIndex = chunkedObj.MainRemoteIndex + } + remoteActualPath, err := d.getActualPathForRemoteOnTarget(obj.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, remoteActualPath) + } + + for _, location := range d.objectLocationsForObject(chunkedObj) { + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) + if err != nil { + return err + } + if err := op.Remove(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.FileStreamer, up driver.UpdateProgress) error { + primaryDirActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), 0) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + if err := d.ensureDirOnTarget(ctx, 0, dstDir.GetPath()); err != nil { + return err + } + + existing := d.linkedObject(streamer.GetExist()) + logicalPath := path.Join(dstDir.GetPath(), streamer.GetName()) + if streamer.GetSize() <= d.ChunkSize { + if err := op.Put(ctx, d.remoteTargets[0].Storage, primaryDirActualPath, streamer, up, false); err != nil { + return err + } + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(d.targetLocation(logicalPath, 0))) + } + + if up == nil { + up = func(float64) {} + } + + var ( + md5Hasher hash.Hash + sha1Hasher hash.Hash + writers []io.Writer + ) + switch d.HashType { + case "md5": + md5Hasher = md5.New() + writers = append(writers, md5Hasher) + case "sha1": + sha1Hasher = sha1.New() + writers = append(writers, sha1Hasher) + } + writers = append(writers, driver.NewProgress(streamer.GetSize(), up)) + + baseReader := io.TeeReader(streamer, io.MultiWriter(writers...)) + xactID := strconv.FormatInt(time.Now().UnixNano(), 36) + if len(xactID) > 9 { + xactID = xactID[len(xactID)-9:] + } + if len(xactID) < 4 { + xactID = fmt.Sprintf("%04s", xactID) + } + + chunkCount := 0 + remaining := streamer.GetSize() + keepLocations := make([]objectLocation, 0, len(d.remoteTargets)+1) + ensuredTargets := map[int]struct{}{0: {}} + for remaining > 0 { + chunkLen := utils.Min(remaining, d.ChunkSize) + targetIndex := d.chunkTargetIndex(chunkCount) + if _, ok := ensuredTargets[targetIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, targetIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[targetIndex] = struct{}{} + } + dstDirActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), targetIndex) + if err != nil { + return err + } + chunkName := d.chunkPartBaseName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) + chunkPath := d.makeChunkName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) + partReader := driver.NewLimitedUploadStream(ctx, &driver.ReaderWithCtx{ + Reader: io.LimitReader(baseReader, chunkLen), + Ctx: ctx, + }) + partStream := &stream.FileStream{ + Obj: &model.Object{ + Name: chunkName, + Size: chunkLen, + Modified: streamer.ModTime(), + Ctime: streamer.CreateTime(), + IsFolder: false, + }, + Reader: partReader, + Mimetype: "application/octet-stream", + WebPutAsTask: streamer.NeedStore(), + ForceStreamUpload: true, + } + if err := op.Put(ctx, d.remoteTargets[targetIndex].Storage, dstDirActualPath, partStream, nil, false); err != nil { + return err + } + keepLocations = append(keepLocations, d.targetLocation(chunkPath, targetIndex)) + remaining -= chunkLen + chunkCount++ + } + + if d.MetaFormat == "simplejson" { + md5Value := "" + if md5Hasher != nil { + md5Value = hex.EncodeToString(md5Hasher.Sum(nil)) + } + sha1Value := "" + if sha1Hasher != nil { + sha1Value = hex.EncodeToString(sha1Hasher.Sum(nil)) + } + txn := xactID + metaData, err := marshalMetadata(streamer.GetSize(), chunkCount, md5Value, sha1Value, txn) + if err != nil { + return err + } + metaStream := &stream.FileStream{ + Obj: &model.Object{ + Name: streamer.GetName(), + Size: int64(len(metaData)), + Modified: streamer.ModTime(), + Ctime: streamer.CreateTime(), + IsFolder: false, + }, + Reader: bytes.NewReader(metaData), + Mimetype: "application/json", + WebPutAsTask: false, + ForceStreamUpload: true, + } + if err := op.Put(ctx, d.remoteTargets[0].Storage, primaryDirActualPath, metaStream, nil, false); err != nil { + return err + } + keepLocations = append(keepLocations, d.targetLocation(logicalPath, 0)) + } else { + for remoteIndex := range d.remoteTargets { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) + if err == nil { + _ = op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, actualPath) + } + } + } + + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(keepLocations...)) +} + +func xactIDIfNeeded(metaFormat, xactID string) string { + if metaFormat == "simplejson" { + return xactID + } + return "" +} + +var _ driver.Driver = (*Chunker)(nil) diff --git a/drivers/chunker/meta.go b/drivers/chunker/meta.go new file mode 100644 index 00000000000..27265fe3033 --- /dev/null +++ b/drivers/chunker/meta.go @@ -0,0 +1,45 @@ +package chunker + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +const ( + defaultChunkSize int64 = 2147483648 + defaultChunkNameFmt = "{name}.rclone_chunk.{chunk:3}" + defaultMetaFormat = "simplejson" + defaultHashType = "md5" + defaultStartFrom = 1 +) + +type Addition struct { + RemotePath string `json:"remote_path" required:"true" help:"Primary AList mounted folder path used to store metadata and small files, e.g. /my-storage/chunks"` + RemotePaths string `json:"remote_paths" type:"text" help:"Additional AList mounted folder paths, one per line. Chunk files will be distributed across remote_path and these extra paths."` + StoreChunksInPrimary bool `json:"store_chunks_in_primary" type:"bool" default:"true" help:"When extra remote paths are configured, also store chunk files in remote_path"` + ChunkSize int64 `json:"chunk_size" type:"number" required:"true" default:"2147483648" help:"Files larger than this will be split into chunks"` + NameFormat string `json:"name_format" required:"true" default:"{name}.rclone_chunk.{chunk:3}" help:"Magic tokens: {name}, {chunk}, {chunk:N}. Name token must appear before chunk token."` + StartFrom int `json:"start_from" type:"number" required:"true" default:"1" help:"Chunk number base, usually 0 or 1"` + MetaFormat string `json:"meta_format" type:"select" required:"true" options:"simplejson,none" default:"simplejson" help:"simplejson is compatible with rclone chunker metadata"` + HashType string `json:"hash_type" type:"select" required:"true" options:"none,md5,sha1" default:"md5" help:"Hash stored in metadata for chunked files"` +} + +var config = driver.Config{ + Name: "Chunker", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: true, + NoCache: true, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Chunker{} + }) +} diff --git a/drivers/chunker/types.go b/drivers/chunker/types.go new file mode 100644 index 00000000000..ff0f6fc4d85 --- /dev/null +++ b/drivers/chunker/types.go @@ -0,0 +1,91 @@ +package chunker + +import ( + "regexp" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" +) + +const ( + ctrlTypeRegStr = `[a-z][a-z0-9]{2,6}` + tempSuffixFormat = `_%04s` + tempSuffixRegStr = `_([0-9a-z]{4,9})` + tempSuffixRegOld = `\.\.tmp_([0-9]{10,13})` + maxMetadataSizeRead = 1023 + maxMetadataSizeWrite = 255 + maxSafeChunkNumber = 10000000 + chunkerMetadataVerion = 2 +) + +var ctrlTypeRegexp = regexp.MustCompile(`^` + ctrlTypeRegStr + `$`) +var chunkTokenRegexp = regexp.MustCompile(`\{chunk(?::([0-9]+))?\}`) + +type Chunker struct { + model.Storage + Addition + remoteStorage driver.Driver + remoteTargets []remoteTarget + dataNameFmt string + nameRegexp *regexp.Regexp +} + +type remoteTarget struct { + MountPath string + Storage driver.Driver +} + +type locatedObj struct { + Obj model.Obj + RemoteIndex int +} + +type objectLocation struct { + LogicalPath string + RemoteIndex int +} + +type metadataJSON struct { + Version *int `json:"ver"` + Size *int64 `json:"size"` + ChunkNum *int `json:"nchunks"` + MD5 string `json:"md5,omitempty"` + SHA1 string `json:"sha1,omitempty"` + XactID string `json:"txn,omitempty"` +} + +type chunkMetadata struct { + Version int + Size int64 + NChunks int + MD5 string + SHA1 string + XactID string +} + +type chunkPart struct { + No int + Size int64 + XactID string + RemoteIndex int +} + +type groupInfo struct { + base *locatedObj + partsByXact map[string]map[int]chunkPart +} + +type Object struct { + model.Object + Main model.Obj + MainRemoteIndex int + Parts []chunkPart + Meta *chunkMetadata + Chunked bool + UsesMeta bool +} + +type linkedPart struct { + part chunkPart + link *model.Link +} diff --git a/drivers/chunker/util.go b/drivers/chunker/util.go new file mode 100644 index 00000000000..ebb740e8194 --- /dev/null +++ b/drivers/chunker/util.go @@ -0,0 +1,904 @@ +package chunker + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Chunker) validateOptions() error { + if strings.TrimSpace(d.RemotePath) == "" { + return errors.New("remote_path is required") + } + if d.ChunkSize <= 0 { + return errors.New("chunk_size must be positive") + } + switch d.MetaFormat { + case "simplejson", "none": + default: + return fmt.Errorf("unsupported meta_format: %s", d.MetaFormat) + } + switch d.HashType { + case "none", "md5", "sha1": + default: + return fmt.Errorf("unsupported hash_type: %s", d.HashType) + } + if d.MetaFormat == "none" && d.HashType != "none" { + return fmt.Errorf("hash_type %q requires meta_format=simplejson", d.HashType) + } + return nil +} + +func (d *Chunker) configuredRemotePaths() []string { + seen := map[string]struct{}{} + paths := make([]string, 0, 1) + addPath := func(p string) { + p = strings.TrimSpace(p) + if p == "" { + return + } + p = utils.FixAndCleanPath(p) + if _, ok := seen[p]; ok { + return + } + seen[p] = struct{}{} + paths = append(paths, p) + } + + addPath(d.RemotePath) + for _, line := range strings.Split(d.RemotePaths, "\n") { + addPath(line) + } + return paths +} + +func (d *Chunker) setChunkNameFormat(pattern string) error { + if dir, _ := path.Split(pattern); dir != "" { + return errors.New("directory separator prohibited") + } + + nameStart, nameEnd, err := parseNameToken(pattern) + if err != nil { + return err + } + chunkStart, chunkEnd, chunkWidth, err := parseChunkToken(pattern) + if err != nil { + return err + } + if nameStart > chunkStart { + return errors.New("name token must come before chunk token") + } + + reDigits := "[0-9]+" + if chunkWidth > 0 { + reDigits = fmt.Sprintf("[0-9]{%d,}", chunkWidth) + } + reDataOrCtrl := fmt.Sprintf("(?:(%s)|_(%s))", reDigits, ctrlTypeRegStr) + + beforeName := pattern[:nameStart] + between := pattern[nameEnd:chunkStart] + afterChunk := pattern[chunkEnd:] + + strRegex := fmt.Sprintf( + "^%s(.+?)%s%s%s(?:%s|%s)?$", + regexp.QuoteMeta(beforeName), + regexp.QuoteMeta(between), + reDataOrCtrl, + regexp.QuoteMeta(afterChunk), + tempSuffixRegStr, + tempSuffixRegOld, + ) + d.nameRegexp = regexp.MustCompile(strRegex) + + fmtDigits := "%d" + if chunkWidth > 0 { + fmtDigits = fmt.Sprintf("%%0%dd", chunkWidth) + } + d.dataNameFmt = strings.ReplaceAll(beforeName, "%", "%%") + + "%s" + + strings.ReplaceAll(between, "%", "%%") + + fmtDigits + + strings.ReplaceAll(afterChunk, "%", "%%") + return nil +} + +func parseNameToken(pattern string) (start, end int, err error) { + nameMagicCount := strings.Count(pattern, "{name}") + switch nameMagicCount { + case 0: + return 0, 0, errors.New("pattern must contain one name token: {name}") + case 1: + default: + return 0, 0, errors.New("pattern must contain exactly one name token: {name}") + } + start = strings.Index(pattern, "{name}") + return start, start + len("{name}"), nil +} + +func parseChunkToken(pattern string) (start, end, width int, err error) { + chunkMatches := chunkTokenRegexp.FindAllStringSubmatchIndex(pattern, -1) + switch len(chunkMatches) { + case 0: + return 0, 0, 0, errors.New("pattern must contain one chunk token: {chunk} or {chunk:N}") + case 1: + default: + return 0, 0, 0, errors.New("pattern must contain exactly one chunk token: {chunk} or {chunk:N}") + } + match := chunkMatches[0] + start = match[0] + end = match[1] + if match[2] >= 0 && match[3] >= 0 { + width, err = strconv.Atoi(pattern[match[2]:match[3]]) + if err != nil || width <= 0 { + return 0, 0, 0, errors.New("chunk width in {chunk:N} must be a positive integer") + } + } + return start, end, width, nil +} + +func (d *Chunker) makeChunkName(filePath string, chunkNo int, xactID string) string { + dir, baseName := path.Split(filePath) + name := fmt.Sprintf(d.dataNameFmt, baseName, chunkNo+d.StartFrom) + if xactID != "" { + name += fmt.Sprintf(tempSuffixFormat, xactID) + } + return dir + name +} + +func (d *Chunker) parseChunkName(filePath string) (parentPath string, chunkNo int, ctrlType, xactID string) { + dir, name := path.Split(filePath) + match := d.nameRegexp.FindStringSubmatch(name) + if match == nil || match[1] == "" { + return "", -1, "", "" + } + + chunkNo = -1 + if match[2] != "" { + n, err := strconv.Atoi(match[2]) + if err != nil { + return "", -1, "", "" + } + chunkNo = n - d.StartFrom + if chunkNo < 0 { + return "", -1, "", "" + } + } + + if match[4] != "" { + xactID = match[4] + } + if match[5] != "" { + oldNum, err := strconv.ParseInt(match[5], 10, 64) + if err != nil || oldNum < 0 { + return "", -1, "", "" + } + xactID = fmt.Sprintf(tempSuffixFormat, strconv.FormatInt(oldNum, 36))[1:] + } + + return dir + match[1], chunkNo, match[3], xactID +} + +func marshalMetadata(size int64, nChunks int, md5Value, sha1Value, xactID string) ([]byte, error) { + version := chunkerMetadataVerion + if xactID == "" && version == 2 { + version = 1 + } + meta := metadataJSON{ + Version: &version, + Size: &size, + ChunkNum: &nChunks, + MD5: md5Value, + SHA1: sha1Value, + XactID: xactID, + } + data, err := json.Marshal(&meta) + if err == nil && len(data) >= maxMetadataSizeWrite { + return nil, errors.New("metadata can't be this big") + } + return data, err +} + +func unmarshalMetadata(data []byte) (*chunkMetadata, error) { + if len(data) > maxMetadataSizeWrite { + return nil, errors.New("metadata is too large") + } + if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' { + return nil, errors.New("invalid json") + } + var meta metadataJSON + if err := json.Unmarshal(data, &meta); err != nil { + return nil, err + } + if meta.Version == nil || meta.Size == nil || meta.ChunkNum == nil { + return nil, errors.New("missing required field") + } + if *meta.Version < 1 { + return nil, errors.New("wrong version") + } + if *meta.Size < 0 { + return nil, errors.New("negative file size") + } + if *meta.ChunkNum < 1 || *meta.ChunkNum > maxSafeChunkNumber { + return nil, errors.New("wrong number of chunks") + } + if meta.MD5 != "" { + if _, err := hex.DecodeString(meta.MD5); err != nil || len(meta.MD5) != 32 { + return nil, errors.New("wrong md5 hash") + } + } + if meta.SHA1 != "" { + if _, err := hex.DecodeString(meta.SHA1); err != nil || len(meta.SHA1) != 40 { + return nil, errors.New("wrong sha1 hash") + } + } + if *meta.Version > chunkerMetadataVerion { + return nil, errors.New("unknown metadata version") + } + return &chunkMetadata{ + Version: *meta.Version, + Size: *meta.Size, + NChunks: *meta.ChunkNum, + MD5: meta.MD5, + SHA1: meta.SHA1, + XactID: meta.XactID, + }, nil +} + +func joinRemotePathWithBase(baseMountPath, logicalPath string) string { + logicalPath = utils.FixAndCleanPath(logicalPath) + if utils.PathEqual(logicalPath, "/") { + return utils.FixAndCleanPath(baseMountPath) + } + return path.Join(utils.FixAndCleanPath(baseMountPath), logicalPath) +} + +func (d *Chunker) joinRemotePath(logicalPath string) string { + return joinRemotePathWithBase(d.RemotePath, logicalPath) +} + +func (d *Chunker) joinRemotePathForTarget(logicalPath string, remoteIndex int) string { + target := d.remoteTargets[remoteIndex] + return joinRemotePathWithBase(target.MountPath, logicalPath) +} + +func (d *Chunker) getActualPathForRemote(logicalPath string) (string, error) { + return d.getActualPathForRemoteOnTarget(logicalPath, 0) +} + +func (d *Chunker) getActualPathForRemoteOnTarget(logicalPath string, remoteIndex int) (string, error) { + _, actualPath, err := op.GetStorageAndActualPath(d.joinRemotePathForTarget(logicalPath, remoteIndex)) + return actualPath, err +} + +func (d *Chunker) getActualChunkPath(filePath string, chunkNo int, xactID string, remoteIndex int) (string, error) { + return d.getActualPathForRemoteOnTarget(d.makeChunkName(filePath, chunkNo, xactID), remoteIndex) +} + +func (d *Chunker) chunkTargetIndex(chunkNo int) int { + targetIndexes := d.chunkTargetIndexes() + if len(targetIndexes) == 0 { + return 0 + } + if chunkNo < 0 { + return targetIndexes[0] + } + return targetIndexes[chunkNo%len(targetIndexes)] +} + +func (d *Chunker) chunkTargetIndexes() []int { + if len(d.remoteTargets) <= 1 { + return []int{0} + } + if d.StoreChunksInPrimary { + targets := make([]int, 0, len(d.remoteTargets)) + for i := range d.remoteTargets { + targets = append(targets, i) + } + return targets + } + targets := make([]int, 0, len(d.remoteTargets)-1) + for i := 1; i < len(d.remoteTargets); i++ { + targets = append(targets, i) + } + if len(targets) == 0 { + return []int{0} + } + return targets +} + +func (d *Chunker) targetLocation(logicalPath string, remoteIndex int) objectLocation { + return objectLocation{ + LogicalPath: utils.FixAndCleanPath(logicalPath), + RemoteIndex: remoteIndex, + } +} + +func (d *Chunker) chunkLocation(filePath string, part chunkPart) objectLocation { + return d.targetLocation(d.makeChunkName(filePath, part.No, part.XactID), part.RemoteIndex) +} + +func (d *Chunker) listDirObjects(ctx context.Context, dirPath string, refresh bool) ([]model.Obj, error) { + groups := map[string]*groupInfo{} + dirMap := map[string]model.Obj{} + found := false + + for remoteIndex := range d.remoteTargets { + remotePath := d.joinRemotePathForTarget(dirPath, remoteIndex) + entries, err := fsList(ctx, remotePath, refresh) + if err != nil { + if errs.IsObjectNotFound(err) { + continue + } + return nil, err + } + found = true + + for _, entry := range entries { + if entry.IsDir() { + if _, ok := dirMap[entry.GetName()]; !ok { + dirMap[entry.GetName()] = &model.Object{ + Name: entry.GetName(), + Path: path.Join(dirPath, entry.GetName()), + Size: 0, + Modified: entry.ModTime(), + Ctime: entry.CreateTime(), + IsFolder: true, + HashInfo: entry.GetHash(), + } + } + continue + } + + mainName, chunkNo, ctrlType, xactID := d.parseChunkName(entry.GetName()) + if mainName == "" { + g := groups[entry.GetName()] + if g == nil { + g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} + groups[entry.GetName()] = g + } + if g.base == nil || remoteIndex < g.base.RemoteIndex { + g.base = &locatedObj{ + Obj: entry, + RemoteIndex: remoteIndex, + } + } + continue + } + if chunkNo < 0 || ctrlType != "" { + continue + } + g := groups[mainName] + if g == nil { + g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} + groups[mainName] = g + } + if g.partsByXact[xactID] == nil { + g.partsByXact[xactID] = map[int]chunkPart{} + } + part := chunkPart{ + No: chunkNo, + Size: entry.GetSize(), + XactID: xactID, + RemoteIndex: remoteIndex, + } + existing, ok := g.partsByXact[xactID][chunkNo] + if !ok || part.RemoteIndex < existing.RemoteIndex { + g.partsByXact[xactID][chunkNo] = part + } + } + } + + if !found && !utils.PathEqual(dirPath, "/") { + return nil, errs.ObjectNotFound + } + + dirs := make([]model.Obj, 0, len(dirMap)) + for _, obj := range dirMap { + dirs = append(dirs, obj) + } + + result := make([]model.Obj, 0, len(dirs)+len(groups)) + result = append(result, dirs...) + for name, group := range groups { + obj, ok, err := d.buildListedObject(ctx, dirPath, name, group) + if err != nil { + return nil, err + } + if ok { + result = append(result, obj) + } + } + return result, nil +} + +func (d *Chunker) targetPathExists(ctx context.Context, remoteIndex int, logicalPath string) (bool, error) { + _, err := fs.Get(ctx, d.joinRemotePathForTarget(logicalPath, remoteIndex), &fs.GetArgs{NoLog: true}) + if err == nil { + return true, nil + } + if errs.IsObjectNotFound(err) { + return false, nil + } + return false, err +} + +func (d *Chunker) ensureDirOnTarget(ctx context.Context, remoteIndex int, logicalDirPath string) error { + logicalDirPath = utils.FixAndCleanPath(logicalDirPath) + if utils.PathEqual(logicalDirPath, "/") { + return nil + } + return fs.MakeDir(ctx, d.joinRemotePathForTarget(logicalDirPath, remoteIndex)) +} + +func (d *Chunker) ensureDirOnAllTargets(ctx context.Context, logicalDirPath string) error { + var errsList []error + for remoteIndex := range d.remoteTargets { + if err := d.ensureDirOnTarget(ctx, remoteIndex, logicalDirPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) existingLocationsForDir(ctx context.Context, logicalDirPath string) ([]int, error) { + locations := make([]int, 0, len(d.remoteTargets)) + for remoteIndex := range d.remoteTargets { + exists, err := d.targetPathExists(ctx, remoteIndex, logicalDirPath) + if err != nil { + return nil, err + } + if exists { + locations = append(locations, remoteIndex) + } + } + return locations, nil +} + +func (d *Chunker) dirLocationsOrAll(ctx context.Context, logicalDirPath string) ([]int, error) { + locations, err := d.existingLocationsForDir(ctx, logicalDirPath) + if err != nil { + return nil, err + } + if len(locations) > 0 { + return locations, nil + } + all := make([]int, 0, len(d.remoteTargets)) + for remoteIndex := range d.remoteTargets { + all = append(all, remoteIndex) + } + return all, nil +} + +func (d *Chunker) moveDirAcrossTargets(ctx context.Context, srcPath, dstDirPath string) error { + locations, err := d.existingLocationsForDir(ctx, srcPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDirPath); err != nil { + errsList = append(errsList, err) + continue + } + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + dstActualPath, err := d.getActualPathForRemoteOnTarget(dstDirPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Move(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, dstActualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) copyDirAcrossTargets(ctx context.Context, srcPath, dstDirPath string) error { + locations, err := d.dirLocationsOrAll(ctx, srcPath) + if err != nil { + return err + } + var errsList []error + for _, remoteIndex := range locations { + exists, err := d.targetPathExists(ctx, remoteIndex, srcPath) + if err != nil { + errsList = append(errsList, err) + continue + } + if !exists { + continue + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDirPath); err != nil { + errsList = append(errsList, err) + continue + } + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + dstActualPath, err := d.getActualPathForRemoteOnTarget(dstDirPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Copy(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, dstActualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) renameDirAcrossTargets(ctx context.Context, srcPath, newName string) error { + locations, err := d.existingLocationsForDir(ctx, srcPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Rename(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, newName); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) removeDirAcrossTargets(ctx context.Context, logicalPath string) error { + locations, err := d.existingLocationsForDir(ctx, logicalPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, actualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, group *groupInfo) (model.Obj, bool, error) { + var meta *chunkMetadata + var err error + if group.base != nil && group.base.Obj.GetSize() <= maxMetadataSizeRead && len(group.partsByXact) > 0 { + meta, err = d.readMetadata(ctx, path.Join(dirPath, name), group.base.Obj.GetSize(), group.base.RemoteIndex) + if err != nil { + meta = nil + } + } + + if meta == nil && group.base != nil && len(group.partsByXact) == 0 { + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: group.base.Obj.GetSize(), + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), + IsFolder: false, + HashInfo: group.base.Obj.GetHash(), + }, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, + }, true, nil + } + + selected := map[int]chunkPart{} + switch { + case meta != nil: + selected = group.partsByXact[meta.XactID] + case group.base == nil: + selected = group.partsByXact[""] + default: + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: group.base.Obj.GetSize(), + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), + IsFolder: false, + HashInfo: group.base.Obj.GetHash(), + }, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, + }, true, nil + } + + parts := sortChunkParts(selected) + if len(parts) == 0 { + if meta != nil { + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: meta.Size, + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), + IsFolder: false, + HashInfo: buildHashInfo(meta), + }, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, + Meta: meta, + Chunked: true, + UsesMeta: true, + }, true, nil + } + return nil, false, nil + } + + size := int64(0) + for _, part := range parts { + size += part.Size + } + modified := time.Time{} + ctime := time.Time{} + if group.base != nil { + modified = group.base.Obj.ModTime() + ctime = group.base.Obj.CreateTime() + } + if meta != nil { + size = meta.Size + } + + mainRemoteIndex := 0 + var mainObj model.Obj + if group.base != nil { + mainRemoteIndex = group.base.RemoteIndex + mainObj = group.base.Obj + } + + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: size, + Modified: modified, + Ctime: ctime, + IsFolder: false, + HashInfo: buildHashInfo(meta), + }, + Main: mainObj, + MainRemoteIndex: mainRemoteIndex, + Parts: parts, + Meta: meta, + Chunked: true, + UsesMeta: meta != nil, + }, true, nil +} + +func (d *Chunker) readMetadata(ctx context.Context, logicalPath string, size int64, remoteIndex int) (*chunkMetadata, error) { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) + if err != nil { + return nil, err + } + link, obj, err := op.Link(ctx, d.remoteTargets[remoteIndex].Storage, actualPath, model.LinkArgs{}) + if err != nil { + return nil, err + } + ss, err := stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: obj}, link) + if err != nil { + return nil, err + } + defer ss.Close() + + reader, err := ss.RangeRead(http_range.Range{Start: 0, Length: size}) + if err != nil { + return nil, err + } + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } + data, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return unmarshalMetadata(data) +} + +func sortChunkParts(parts map[int]chunkPart) []chunkPart { + result := make([]chunkPart, 0, len(parts)) + for _, part := range parts { + result = append(result, part) + } + sort.Slice(result, func(i, j int) bool { + return result[i].No < result[j].No + }) + return result +} + +func buildHashInfo(meta *chunkMetadata) utils.HashInfo { + if meta == nil { + return utils.HashInfo{} + } + hashes := map[*utils.HashType]string{} + if meta.MD5 != "" { + hashes[utils.MD5] = meta.MD5 + } + if meta.SHA1 != "" { + hashes[utils.SHA1] = meta.SHA1 + } + return utils.NewHashInfoByMap(hashes) +} + +func (d *Chunker) linkedObject(obj model.Obj) *Object { + if linked, ok := obj.(*Object); ok { + return linked + } + return nil +} + +func (d *Chunker) objectLocationsForObject(obj *Object) []objectLocation { + if obj == nil { + return nil + } + locations := make([]objectLocation, 0, len(obj.Parts)+1) + if obj.Chunked && obj.UsesMeta { + locations = append(locations, d.targetLocation(obj.GetPath(), obj.MainRemoteIndex)) + } + if !obj.Chunked { + locations = append(locations, d.targetLocation(obj.GetPath(), obj.MainRemoteIndex)) + return locations + } + for _, part := range obj.Parts { + locations = append(locations, d.chunkLocation(obj.GetPath(), part)) + } + return locations +} + +func (d *Chunker) cleanupReplacedObject(ctx context.Context, obj *Object, keep map[string]struct{}) error { + if obj == nil { + return nil + } + var errs []error + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := keep[d.keepKey(location)]; ok { + continue + } + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) + if err != nil { + errs = append(errs, err) + continue + } + if err := op.Remove(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (d *Chunker) keepKey(location objectLocation) string { + return fmt.Sprintf("%d:%s", location.RemoteIndex, utils.FixAndCleanPath(location.LogicalPath)) +} + +func (d *Chunker) buildKeepSet(locations ...objectLocation) map[string]struct{} { + keep := make(map[string]struct{}, len(locations)) + for _, location := range locations { + if location.LogicalPath == "" { + continue + } + keep[d.keepKey(location)] = struct{}{} + } + return keep +} + +func (d *Chunker) chunkPartBaseName(filePath string, chunkNo int, xactID string) string { + return path.Base(d.makeChunkName(filePath, chunkNo, xactID)) +} + +func (d *Chunker) openChunkReader(ctx context.Context, parts []linkedPart, totalSize int64, req http_range.Range) (io.ReadCloser, error) { + if req.Start < 0 || req.Start > totalSize { + return nil, fmt.Errorf("range start out of bound") + } + if req.Length < 0 || req.Start+req.Length > totalSize { + req.Length = totalSize - req.Start + } + if req.Length == 0 { + return io.NopCloser(strings.NewReader("")), nil + } + + var ( + readers []io.Reader + closers = utils.EmptyClosers() + offset int64 + remaining = req.Length + ) + for _, part := range parts { + partStart := offset + partEnd := offset + part.part.Size + offset = partEnd + if req.Start >= partEnd || remaining <= 0 { + continue + } + localStart := int64(0) + if req.Start > partStart { + localStart = req.Start - partStart + } + localLength := utils.Min(part.part.Size-localStart, remaining) + rc, err := d.openPartRange(ctx, part.link, part.part.Size, localStart, localLength) + if err != nil { + _ = closers.Close() + return nil, err + } + readers = append(readers, rc) + closers.Add(rc) + remaining -= localLength + } + if remaining > 0 { + _ = closers.Close() + return nil, errors.New("missing chunk data") + } + return utils.NewReadCloser(io.MultiReader(readers...), func() error { + return closers.Close() + }), nil +} + +func (d *Chunker) openPartRange(ctx context.Context, link *model.Link, size, offset, length int64) (io.ReadCloser, error) { + httpRange := http_range.Range{Start: offset, Length: length} + switch { + case link.MFile != nil: + return io.NopCloser(io.NewSectionReader(link.MFile, offset, length)), nil + case link.RangeReadCloser != nil: + return link.RangeReadCloser.RangeRead(ctx, httpRange) + case link.URL != "": + rrc, err := stream.GetRangeReadCloserFromLink(size, link) + if err != nil { + return nil, err + } + rc, err := rrc.RangeRead(ctx, httpRange) + if err != nil { + _ = rrc.Close() + return nil, err + } + return utils.NewReadCloser(rc, func() error { + return rrc.Close() + }), nil + default: + return nil, errors.New("chunk part has no readable link") + } +} + +func fsList(ctx context.Context, remotePath string, refresh bool) ([]model.Obj, error) { + return fs.List(ctx, remotePath, &fs.ListArgs{NoLog: true, Refresh: refresh}) +} diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 8c2321b8f40..dcde58c638d 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -18,6 +18,7 @@ import ( type Cloudreve struct { model.Storage Addition + ref *Cloudreve } func (d *Cloudreve) Config() driver.Config { @@ -37,8 +38,18 @@ func (d *Cloudreve) Init(ctx context.Context) error { return d.login() } +func (d *Cloudreve) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*Cloudreve) + if ok { + d.ref = refStorage + return nil + } + return errs.NotSupport +} + func (d *Cloudreve) Drop(ctx context.Context) error { d.Cookie = "" + d.ref = nil return nil } diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index 196d7303337..5054de6cb56 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "encoding/base64" + "encoding/json" "errors" "fmt" "io" "net/http" "strconv" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" @@ -19,7 +21,6 @@ import ( "github.com/alist-org/alist/v3/pkg/cookie" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" - json "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go" ) @@ -35,6 +36,9 @@ func (d *Cloudreve) getUA() string { } func (d *Cloudreve) request(method string, path string, callback base.ReqCallback, out interface{}) error { + if d.ref != nil { + return d.ref.request(method, path, callback, out) + } u := d.Address + "/api/v3" + path req := base.RestyClient.R() req.SetHeaders(map[string]string{ @@ -79,11 +83,11 @@ func (d *Cloudreve) request(method string, path string, callback base.ReqCallbac } if out != nil && r.Data != nil { var marshal []byte - marshal, err = json.Marshal(r.Data) + marshal, err = jsoniter.Marshal(r.Data) if err != nil { return err } - err = json.Unmarshal(marshal, out) + err = jsoniter.Unmarshal(marshal, out) if err != nil { return err } @@ -187,12 +191,9 @@ func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u Up if utils.IsCanceled(ctx) { return ctx.Err() } - utils.Log.Debugf("[Cloudreve-Local] upload: %d", finish) - var byteSize = DEFAULT left := stream.GetSize() - finish - if left < DEFAULT { - byteSize = left - } + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Cloudreve-Local] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) utils.Log.Debug(err, n) @@ -205,9 +206,26 @@ func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u Up req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) req.SetHeader("User-Agent", d.getUA()) req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + req.AddRetryCondition(func(r *resty.Response, err error) bool { + if err != nil { + return true + } + if r.IsError() { + return true + } + var retryResp Resp + jErr := base.RestyClient.JSONUnmarshal(r.Body(), &retryResp) + if jErr != nil { + return true + } + if retryResp.Code != 0 { + return true + } + return false + }) }, nil) if err != nil { - break + return err } finish += byteSize up(float64(finish) * 100 / float64(stream.GetSize())) @@ -222,16 +240,15 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U var finish int64 = 0 var chunk int = 0 DEFAULT := int64(u.ChunkSize) + retryCount := 0 + maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } - utils.Log.Debugf("[Cloudreve-Remote] upload: %d", finish) - var byteSize = DEFAULT left := stream.GetSize() - finish - if left < DEFAULT { - byteSize = left - } + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Cloudreve-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) utils.Log.Debug(err, n) @@ -248,14 +265,43 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Authorization", fmt.Sprint(credential)) req.Header.Set("User-Agent", d.getUA()) - finish += byteSize - res, err := base.HttpClient.Do(req) - if err != nil { - return err + err = func() error { + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return errors.New(res.Status) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }() + if err == nil { + retryCount = 0 + finish += byteSize + up(float64(finish) * 100 / float64(stream.GetSize())) + chunk++ + } else { + retryCount++ + if retryCount > maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< 0 { + src.Size = ds.FolderSummary.Size + } + } + var thumb model.Thumbnail + if d.EnableThumb && src.Type == 0 { + var t FileThumbResp + err := d.request(http.MethodGet, "/file/thumb", func(req *resty.Request) { + req.SetQueryParam("uri", src.Path) + }, &t) + if err == nil && t.URL != "" { + thumb = model.Thumbnail{ + Thumbnail: t.URL, + } + } + } + return &model.ObjThumb{ + Object: model.Object{ + ID: src.ID, + Path: src.Path, + Name: src.Name, + Size: src.Size, + Modified: src.UpdatedAt, + Ctime: src.CreatedAt, + IsFolder: src.Type == 1, + }, + Thumbnail: thumb, + }, nil + }) +} + +func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var url FileUrlResp + err := d.request(http.MethodPost, "/file/url", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{file.GetPath()}, + "download": true, + }) + }, &url) + if err != nil { + return nil, err + } + if len(url.Urls) == 0 { + return nil, errors.New("server returns no url") + } + exp := time.Until(url.Expires) + return &model.Link{ + URL: url.Urls[0].URL, + Expiration: &exp, + }, nil +} + +func (d *CloudreveV4) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "type": "folder", + "uri": parentDir.GetPath() + "/" + dirName, + "error_on_conflict": true, + }) + }, nil) +} + +func (d *CloudreveV4) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.request(http.MethodPost, "/file/move", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{srcObj.GetPath()}, + "dst": dstDir.GetPath(), + "copy": false, + }) + }, nil) +} + +func (d *CloudreveV4) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "new_name": newName, + "uri": srcObj.GetPath(), + }) + }, nil) + +} + +func (d *CloudreveV4) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.request(http.MethodPost, "/file/move", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{srcObj.GetPath()}, + "dst": dstDir.GetPath(), + "copy": true, + }) + }, nil) +} + +func (d *CloudreveV4) Remove(ctx context.Context, obj model.Obj) error { + return d.request(http.MethodDelete, "/file", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{obj.GetPath()}, + "unlink": false, + "skip_soft_delete": true, + }) + }, nil) +} + +func (d *CloudreveV4) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if file.GetSize() == 0 { + // 空文件使用新建文件方法，避免上传卡锁 + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "type": "file", + "uri": dstDir.GetPath() + "/" + file.GetName(), + "error_on_conflict": true, + }) + }, nil) + } + var p StoragePolicy + var r FileResp + var u FileUploadResp + var err error + params := map[string]string{ + "page_size": "10", + "uri": dstDir.GetPath(), + "order_by": "created_at", + "order_direction": "asc", + "page": "0", + } + err = d.request(http.MethodGet, "/file", func(req *resty.Request) { + req.SetQueryParams(params) + }, &r) + if err != nil { + return err + } + p = r.StoragePolicy + body := base.Json{ + "uri": dstDir.GetPath() + "/" + file.GetName(), + "size": file.GetSize(), + "policy_id": p.ID, + "last_modified": file.ModTime().UnixMilli(), + "mime_type": "", + } + if d.EnableVersionUpload { + body["entity_type"] = "version" + } + err = d.request(http.MethodPut, "/file/upload", func(req *resty.Request) { + req.SetBody(body) + }, &u) + if err != nil { + return err + } + if u.StoragePolicy.Relay { + err = d.upLocal(ctx, file, u, up) + } else { + switch u.StoragePolicy.Type { + case "local": + err = d.upLocal(ctx, file, u, up) + case "remote": + err = d.upRemote(ctx, file, u, up) + case "onedrive": + err = d.upOneDrive(ctx, file, u, up) + case "s3": + err = d.upS3(ctx, file, u, up) + default: + return errs.NotImplement + } + } + if err != nil { + // 删除失败的会话 + _ = d.request(http.MethodDelete, "/file/upload", func(req *resty.Request) { + req.SetBody(base.Json{ + "id": u.SessionID, + "uri": u.URI, + }) + }, nil) + return err + } + return nil +} + +func (d *CloudreveV4) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *CloudreveV4) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*CloudreveV4)(nil) diff --git a/drivers/cloudreve_v4/meta.go b/drivers/cloudreve_v4/meta.go new file mode 100644 index 00000000000..bfaa14f81e4 --- /dev/null +++ b/drivers/cloudreve_v4/meta.go @@ -0,0 +1,44 @@ +package cloudreve_v4 + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // driver.RootID + // define other + Address string `json:"address" required:"true"` + Username string `json:"username"` + Password string `json:"password"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + CustomUA string `json:"custom_ua"` + EnableFolderSize bool `json:"enable_folder_size"` + EnableThumb bool `json:"enable_thumb"` + EnableVersionUpload bool `json:"enable_version_upload"` + OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at" default:"name" required:"true"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc" required:"true"` +} + +var config = driver.Config{ + Name: "Cloudreve V4", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "cloudreve://my", + CheckStatus: true, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &CloudreveV4{} + }) +} diff --git a/drivers/cloudreve_v4/types.go b/drivers/cloudreve_v4/types.go new file mode 100644 index 00000000000..e81226d3da5 --- /dev/null +++ b/drivers/cloudreve_v4/types.go @@ -0,0 +1,164 @@ +package cloudreve_v4 + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Object struct { + model.Object + StoragePolicy StoragePolicy +} + +type Resp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data any `json:"data"` +} + +type BasicConfigResp struct { + InstanceID string `json:"instance_id"` + // Title string `json:"title"` + // Themes string `json:"themes"` + // DefaultTheme string `json:"default_theme"` + User struct { + ID string `json:"id"` + // Nickname string `json:"nickname"` + // CreatedAt time.Time `json:"created_at"` + // Anonymous bool `json:"anonymous"` + Group struct { + ID string `json:"id"` + Name string `json:"name"` + Permission string `json:"permission"` + } `json:"group"` + } `json:"user"` + // Logo string `json:"logo"` + // LogoLight string `json:"logo_light"` + // CaptchaReCaptchaKey string `json:"captcha_ReCaptchaKey"` + CaptchaType string `json:"captcha_type"` // support 'normal' only + // AppPromotion bool `json:"app_promotion"` +} + +type SiteLoginConfigResp struct { + LoginCaptcha bool `json:"login_captcha"` + Authn bool `json:"authn"` +} + +type PrepareLoginResp struct { + WebauthnEnabled bool `json:"webauthn_enabled"` + PasswordEnabled bool `json:"password_enabled"` +} + +type CaptchaResp struct { + Image string `json:"image"` + Ticket string `json:"ticket"` +} + +type Token struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + AccessExpires time.Time `json:"access_expires"` + RefreshExpires time.Time `json:"refresh_expires"` +} + +type TokenResponse struct { + User struct { + ID string `json:"id"` + // Email string `json:"email"` + // Nickname string `json:"nickname"` + Status string `json:"status"` + // CreatedAt time.Time `json:"created_at"` + Group struct { + ID string `json:"id"` + Name string `json:"name"` + Permission string `json:"permission"` + // DirectLinkBatchSize int `json:"direct_link_batch_size"` + // TrashRetention int `json:"trash_retention"` + } `json:"group"` + // Language string `json:"language"` + } `json:"user"` + Token Token `json:"token"` +} + +type File struct { + Type int `json:"type"` // 0: file, 1: folder + ID string `json:"id"` + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Size int64 `json:"size"` + Metadata interface{} `json:"metadata"` + Path string `json:"path"` + Capability string `json:"capability"` + Owned bool `json:"owned"` + PrimaryEntity string `json:"primary_entity"` +} + +type StoragePolicy struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + MaxSize int64 `json:"max_size"` + Relay bool `json:"relay,omitempty"` +} + +type Pagination struct { + Page int `json:"page"` + PageSize int `json:"page_size"` + IsCursor bool `json:"is_cursor"` + NextToken string `json:"next_token,omitempty"` +} + +type Props struct { + Capability string `json:"capability"` + MaxPageSize int `json:"max_page_size"` + OrderByOptions []string `json:"order_by_options"` + OrderDirectionOptions []string `json:"order_direction_options"` +} + +type FileResp struct { + Files []File `json:"files"` + Parent File `json:"parent"` + Pagination Pagination `json:"pagination"` + Props Props `json:"props"` + ContextHint string `json:"context_hint"` + MixedType bool `json:"mixed_type"` + StoragePolicy StoragePolicy `json:"storage_policy"` +} + +type FileUrlResp struct { + Urls []struct { + URL string `json:"url"` + } `json:"urls"` + Expires time.Time `json:"expires"` +} + +type FileUploadResp struct { + // UploadID string `json:"upload_id"` + SessionID string `json:"session_id"` + ChunkSize int64 `json:"chunk_size"` + Expires int64 `json:"expires"` + StoragePolicy StoragePolicy `json:"storage_policy"` + URI string `json:"uri"` + CompleteURL string `json:"completeURL,omitempty"` // for S3-like + CallbackSecret string `json:"callback_secret,omitempty"` // for S3-like, OneDrive + UploadUrls []string `json:"upload_urls,omitempty"` // for not-local + Credential string `json:"credential,omitempty"` // for local +} + +type FileThumbResp struct { + URL string `json:"url"` + Expires time.Time `json:"expires"` +} + +type FolderSummaryResp struct { + File + FolderSummary struct { + Size int64 `json:"size"` + Files int64 `json:"files"` + Folders int64 `json:"folders"` + Completed bool `json:"completed"` + CalculatedAt time.Time `json:"calculated_at"` + } `json:"folder_summary"` +} diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go new file mode 100644 index 00000000000..cf2337f279b --- /dev/null +++ b/drivers/cloudreve_v4/util.go @@ -0,0 +1,476 @@ +package cloudreve_v4 + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + jsoniter "github.com/json-iterator/go" +) + +// do others that not defined in Driver interface + +func (d *CloudreveV4) getUA() string { + if d.CustomUA != "" { + return d.CustomUA + } + return base.UserAgent +} + +func (d *CloudreveV4) request(method string, path string, callback base.ReqCallback, out any) error { + if d.ref != nil { + return d.ref.request(method, path, callback, out) + } + u := d.Address + "/api/v4" + path + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "User-Agent": d.getUA(), + }) + if d.AccessToken != "" { + req.SetHeader("Authorization", "Bearer "+d.AccessToken) + } + + var r Resp + req.SetResult(&r) + + if callback != nil { + callback(req) + } + + resp, err := req.Execute(method, u) + if err != nil { + return err + } + if !resp.IsSuccess() { + return errors.New(resp.String()) + } + + if r.Code != 0 { + if r.Code == 401 && d.RefreshToken != "" && path != "/session/token/refresh" { + // try to refresh token + err = d.refreshToken() + if err != nil { + return err + } + return d.request(method, path, callback, out) + } + return errors.New(r.Msg) + } + + if out != nil && r.Data != nil { + var marshal []byte + marshal, err = json.Marshal(r.Data) + if err != nil { + return err + } + err = json.Unmarshal(marshal, out) + if err != nil { + return err + } + } + + return nil +} + +func (d *CloudreveV4) login() error { + var siteConfig SiteLoginConfigResp + err := d.request(http.MethodGet, "/site/config/login", nil, &siteConfig) + if err != nil { + return err + } + if !siteConfig.Authn { + return errors.New("authn not support") + } + var prepareLogin PrepareLoginResp + err = d.request(http.MethodGet, "/session/prepare?email="+d.Addition.Username, nil, &prepareLogin) + if err != nil { + return err + } + if !prepareLogin.PasswordEnabled { + return errors.New("password not enabled") + } + if prepareLogin.WebauthnEnabled { + return errors.New("webauthn not support") + } + for range 5 { + err = d.doLogin(siteConfig.LoginCaptcha) + if err == nil { + break + } + if err.Error() != "CAPTCHA not match." { + break + } + } + return err +} + +func (d *CloudreveV4) doLogin(needCaptcha bool) error { + var err error + loginBody := base.Json{ + "email": d.Username, + "password": d.Password, + } + if needCaptcha { + var config BasicConfigResp + err = d.request(http.MethodGet, "/site/config/basic", nil, &config) + if err != nil { + return err + } + if config.CaptchaType != "normal" { + return fmt.Errorf("captcha type %s not support", config.CaptchaType) + } + var captcha CaptchaResp + err = d.request(http.MethodGet, "/site/captcha", nil, &captcha) + if err != nil { + return err + } + if !strings.HasPrefix(captcha.Image, "data:image/png;base64,") { + return errors.New("can not get captcha") + } + loginBody["ticket"] = captcha.Ticket + i := strings.Index(captcha.Image, ",") + dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(captcha.Image[i+1:])) + vRes, err := base.RestyClient.R().SetMultipartField( + "image", "validateCode.png", "image/png", dec). + Post(setting.GetStr(conf.OcrApi)) + if err != nil { + return err + } + if jsoniter.Get(vRes.Body(), "status").ToInt() != 200 { + return errors.New("ocr error:" + jsoniter.Get(vRes.Body(), "msg").ToString()) + } + captchaCode := jsoniter.Get(vRes.Body(), "result").ToString() + if captchaCode == "" { + return errors.New("ocr error: empty result") + } + loginBody["captcha"] = captchaCode + } + var token TokenResponse + err = d.request(http.MethodPost, "/session/token", func(req *resty.Request) { + req.SetBody(loginBody) + }, &token) + if err != nil { + return err + } + d.AccessToken, d.RefreshToken = token.Token.AccessToken, token.Token.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *CloudreveV4) refreshToken() error { + var token Token + if token.RefreshToken == "" { + if d.Username != "" { + err := d.login() + if err != nil { + return fmt.Errorf("cannot login to get refresh token, error: %s", err) + } + } + return nil + } + err := d.request(http.MethodPost, "/session/token/refresh", func(req *resty.Request) { + req.SetBody(base.Json{ + "refresh_token": d.RefreshToken, + }) + }, &token) + if err != nil { + return err + } + d.AccessToken, d.RefreshToken = token.AccessToken, token.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *CloudreveV4) upLocal(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + if DEFAULT == 0 { + // support relay + DEFAULT = file.GetSize() + } + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[CloudreveV4-Local] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { + req.SetHeader("Content-Type", "application/octet-stream") + req.SetContentLength(true) + req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) + req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + req.AddRetryCondition(func(r *resty.Response, err error) bool { + if err != nil { + return true + } + if r.IsError() { + return true + } + var retryResp Resp + jErr := base.RestyClient.JSONUnmarshal(r.Body(), &retryResp) + if jErr != nil { + return true + } + if retryResp.Code != 0 { + return true + } + return false + }) + }, nil) + if err != nil { + return err + } + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + chunk++ + } + return nil +} + +func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { + uploadUrl := u.UploadUrls[0] + credential := u.Credential + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + retryCount := 0 + maxRetries := 3 + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[CloudreveV4-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.ContentLength = byteSize + // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.Header.Set("Authorization", fmt.Sprint(credential)) + req.Header.Set("User-Agent", d.getUA()) + err = func() error { + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return errors.New(res.Status) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }() + if err == nil { + retryCount = 0 + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + chunk++ + } else { + retryCount++ + if retryCount > maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors", maxRetries) + } + backoff := time.Duration(1<") + for i, etag := range etags { + bodyBuilder.WriteString(fmt.Sprintf( + `%d%s`, + i+1, // PartNumber 从 1 开始 + etag, + )) + } + bodyBuilder.WriteString("") + req, err := http.NewRequest( + "POST", + u.CompleteURL, + strings.NewReader(bodyBuilder.String()), + ) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/xml") + req.Header.Set("User-Agent", d.getUA()) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + return fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(body)) + } + + // 上传成功发送回调请求 + return d.request(http.MethodPost, "/callback/s3/"+u.SessionID+"/"+u.CallbackSecret, func(req *resty.Request) { + req.SetBody("{}") + }, nil) +} diff --git a/drivers/crypt/meta.go b/drivers/crypt/meta.go index 180773a3f48..0878f63869f 100644 --- a/drivers/crypt/meta.go +++ b/drivers/crypt/meta.go @@ -13,16 +13,16 @@ type Addition struct { FileNameEnc string `json:"filename_encryption" type:"select" required:"true" options:"off,standard,obfuscate" default:"off"` DirNameEnc string `json:"directory_name_encryption" type:"select" required:"true" options:"false,true" default:"false"` - RemotePath string `json:"remote_path" required:"true" help:"This is where the encrypted data stores"` + RemotePath string `json:"remote_path" required:"true" help:"AList mounted folder path used to store encrypted data, e.g. /my-storage/secret"` Password string `json:"password" required:"true" confidential:"true" help:"the main password"` Salt string `json:"salt" confidential:"true" help:"If you don't know what is salt, treat it as a second password. Optional but recommended"` EncryptedSuffix string `json:"encrypted_suffix" required:"true" default:".bin" help:"for advanced user only! encrypted files will have this suffix"` FileNameEncoding string `json:"filename_encoding" type:"select" required:"true" options:"base64,base32,base32768" default:"base64" help:"for advanced user only!"` - Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` + Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` - ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` + ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` } var config = driver.Config{ diff --git a/drivers/darkibox/driver.go b/drivers/darkibox/driver.go new file mode 100644 index 00000000000..0e3f2e5065b --- /dev/null +++ b/drivers/darkibox/driver.go @@ -0,0 +1,299 @@ +package darkibox + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" +) + +type Darkibox struct { + model.Storage + Addition +} + +func (d *Darkibox) Config() driver.Config { + return config +} + +func (d *Darkibox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Darkibox) Init(ctx context.Context) error { + if d.APIKey == "" { + return fmt.Errorf("API key is required") + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + + // Verify API key by calling account/info + var account accountInfoResult + if err := d.callAPI(ctx, "/account/info", nil, &account); err != nil { + return fmt.Errorf("failed to verify API key: %w", err) + } + + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Darkibox) Drop(ctx context.Context) error { + return nil +} + +func (d *Darkibox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := d.RootFolderID + if dir.GetID() != "" { + folderID = folderIDFromObjID(dir.GetID()) + } + + var objects []model.Obj + + // List sub-folders via /api/folder/list + var folders folderListResult + if err := d.callAPI(ctx, "/folder/list", map[string]string{ + "fld_id": fldIDStr(folderID), + }, &folders); err != nil { + return nil, fmt.Errorf("list folders failed: %w", err) + } + for _, f := range folders.Folders { + objects = append(objects, &model.Object{ + ID: encodeFolderID(f.FldID), + Name: f.Name, + IsFolder: true, + }) + } + + // List files via /api/file/list (paginated) + page := 1 + for { + var files fileListResult + if err := d.callAPI(ctx, "/file/list", map[string]string{ + "fld_id": fldIDStr(folderID), + "per_page": "200", + "page": strconv.Itoa(page), + }, &files); err != nil { + return nil, fmt.Errorf("list files failed: %w", err) + } + + for _, f := range files.Files { + modified := time.Now() + if f.Uploaded != "" { + if t, err := time.Parse("2006-01-02 15:04:05", f.Uploaded); err == nil { + modified = t + } + } + name := f.Name + if name == "" { + name = f.Title + } + objects = append(objects, &model.Object{ + ID: encodeFileID(f.FileCode), + Name: name, + Size: f.Size, + Modified: modified, + IsFolder: false, + }) + } + + // Check if there are more pages + if len(files.Files) < 200 { + break + } + page++ + } + + return objects, nil +} + +func (d *Darkibox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + + fileCode := fileCodeFromObjID(file.GetID()) + if fileCode == "" { + return nil, fmt.Errorf("empty file code") + } + + var result directLinkResult + if err := d.callAPI(ctx, "/file/direct_link", map[string]string{ + "file_code": fileCode, + }, &result); err != nil { + return nil, fmt.Errorf("failed to get direct link: %w", err) + } + + // Find the original quality version, fall back to first available + var dlURL string + for _, v := range result.Versions { + if v.Name == "o" { + dlURL = v.URL + break + } + } + if dlURL == "" && len(result.Versions) > 0 { + dlURL = result.Versions[0].URL + } + if dlURL == "" { + return nil, fmt.Errorf("no download URL available for file %s", fileCode) + } + + return &model.Link{ + URL: dlURL, + }, nil +} + +func (d *Darkibox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + parentID := d.RootFolderID + if parentDir.GetID() != "" { + parentID = folderIDFromObjID(parentDir.GetID()) + } + + var result folderCreateResult + if err := d.callAPI(ctx, "/folder/create", map[string]string{ + "name": dirName, + "parent_id": fldIDStr(parentID), + }, &result); err != nil { + return nil, fmt.Errorf("create folder failed: %w", err) + } + + return &model.Object{ + ID: encodeFolderID(result.FldID), + Name: dirName, + IsFolder: true, + }, nil +} + +func (d *Darkibox) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, errs.NotImplement + } + + fileCode := fileCodeFromObjID(srcObj.GetID()) + if fileCode == "" { + return nil, fmt.Errorf("empty file code") + } + + dstFolderID := d.RootFolderID + if dstDir.GetID() != "" { + dstFolderID = folderIDFromObjID(dstDir.GetID()) + } + + if err := d.callAPI(ctx, "/file/move", map[string]string{ + "file_code": fileCode, + "to_folder": fldIDStr(dstFolderID), + }, nil); err != nil { + return nil, fmt.Errorf("move file failed: %w", err) + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: false, + }, nil +} + +func (d *Darkibox) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + folderID := folderIDFromObjID(obj.GetID()) + return d.callAPI(ctx, "/folder/delete", map[string]string{ + "fld_id": folderID, + }, nil) + } + + fileCode := fileCodeFromObjID(obj.GetID()) + return d.callAPI(ctx, "/file/delete", map[string]string{ + "file_code": fileCode, + }, nil) +} + +func (d *Darkibox) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + // Step 1: Get the upload server URL + var server uploadServerResult + if err := d.callAPI(ctx, "/upload/server", nil, &server); err != nil { + return nil, fmt.Errorf("get upload server failed: %w", err) + } + if server.URL == "" { + return nil, fmt.Errorf("no upload server URL returned") + } + + // Step 2: Upload the file to the upload server + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }) + + res, err := base.RestyClient.R(). + SetContext(ctx). + SetMultipartField("file", file.GetName(), "", reader). + SetMultipartFormData(map[string]string{ + "key": d.APIKey, + "fld_id": fldIDStr(folderID), + }). + Post(server.URL) + if err != nil { + return nil, fmt.Errorf("upload failed: %w", err) + } + if res.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("upload failed: http %d", res.StatusCode()) + } + + // Try to parse upload response to get the file code + var uploadResp uploadResult + if err := base.RestyClient.JSONUnmarshal(res.Body(), &uploadResp); err == nil && len(uploadResp.Files) > 0 { + uf := uploadResp.Files[0] + return &model.Object{ + ID: encodeFileID(uf.FileCode), + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil + } + + return &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +func (d *Darkibox) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*Darkibox)(nil) diff --git a/drivers/darkibox/meta.go b/drivers/darkibox/meta.go new file mode 100644 index 00000000000..a09707707fd --- /dev/null +++ b/drivers/darkibox/meta.go @@ -0,0 +1,27 @@ +package darkibox + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APIKey string `json:"api_key" required:"true" help:"API key from your Darkibox account"` +} + +var config = driver.Config{ + Name: "Darkibox", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: true, + NoCache: false, + NoUpload: false, + DefaultRoot: "0", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Darkibox{} + }) +} diff --git a/drivers/darkibox/types.go b/drivers/darkibox/types.go new file mode 100644 index 00000000000..9f193f97e4f --- /dev/null +++ b/drivers/darkibox/types.go @@ -0,0 +1,79 @@ +package darkibox + +import "encoding/json" + +// apiResponse is the common wrapper for all Darkibox API responses. +type apiResponse struct { + Msg string `json:"msg"` + Result json.RawMessage `json:"result"` + ServerTime string `json:"server_time"` + Status int `json:"status"` +} + +// accountInfoResult represents the result of /api/account/info +type accountInfoResult struct { + Email string `json:"email"` + Balance string `json:"balance"` + StorageUsed string `json:"storage_used"` +} + +// fileListResult represents the result of /api/file/list +type fileListResult struct { + Results int `json:"results"` + ResultsTotal int `json:"results_total"` + Files []fileItem `json:"files"` +} + +type fileItem struct { + FileCode string `json:"file_code"` + Name string `json:"name"` + Title string `json:"title"` + Size int64 `json:"size"` + Uploaded string `json:"uploaded"` + FldID int64 `json:"fld_id"` +} + +// folderListResult represents the result of /api/folder/list +type folderListResult struct { + Folders []folderItem `json:"folders"` +} + +type folderItem struct { + FldID int64 `json:"fld_id"` + Name string `json:"name"` + Code string `json:"code"` +} + +// directLinkResult represents the result of /api/file/direct_link +type directLinkResult struct { + Versions []directLinkVersion `json:"versions"` +} + +type directLinkVersion struct { + Name string `json:"name"` + URL string `json:"url"` +} + +// uploadServerResult represents the result of /api/upload/server +type uploadServerResult struct { + URL string `json:"url"` +} + +// uploadResult represents the response from the upload endpoint +type uploadResult struct { + Files []uploadedFile `json:"files"` +} + +type uploadedFile struct { + FileCode string `json:"filecode"` + URL string `json:"url"` + Name string `json:"name"` + Size int64 `json:"size"` + Status int `json:"status"` +} + +// folderCreateResult represents the result of /api/folder/create +type folderCreateResult struct { + FldID int64 `json:"fld_id"` + Name string `json:"name"` +} diff --git a/drivers/darkibox/util.go b/drivers/darkibox/util.go new file mode 100644 index 00000000000..0032abab106 --- /dev/null +++ b/drivers/darkibox/util.go @@ -0,0 +1,88 @@ +package darkibox + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" +) + +const apiBase = "https://darkibox.com/api" + +// callAPI makes a GET request to the Darkibox API with the given endpoint and params. +// It automatically injects the API key. The result JSON is unmarshalled into out if non-nil. +func (d *Darkibox) callAPI(ctx context.Context, endpoint string, params map[string]string, out any) error { + query := map[string]string{ + "key": d.APIKey, + } + for k, v := range params { + if strings.TrimSpace(v) == "" { + continue + } + query[k] = v + } + + var resp apiResponse + r, err := base.RestyClient.R(). + SetContext(ctx). + SetQueryParams(query). + SetResult(&resp). + Get(apiBase + endpoint) + if err != nil { + return err + } + if r.StatusCode() != http.StatusOK { + return fmt.Errorf("darkibox http error: %d", r.StatusCode()) + } + if resp.Status != 200 { + return fmt.Errorf("darkibox api error: status=%d msg=%s", resp.Status, resp.Msg) + } + if out == nil || len(resp.Result) == 0 || string(resp.Result) == "null" { + return nil + } + if err := json.Unmarshal(resp.Result, out); err != nil { + return fmt.Errorf("decode darkibox result failed: %w", err) + } + return nil +} + +// fldIDStr converts a folder ID (which may be the root "0") to a string suitable for API params. +func fldIDStr(id string) string { + if id == "" { + return "0" + } + return id +} + +// encodeFolderID prefixes a folder ID so we can distinguish folders from files. +func encodeFolderID(id int64) string { + return "d:" + strconv.FormatInt(id, 10) +} + +// encodeFileID prefixes a file code so we can distinguish files from folders. +func encodeFileID(code string) string { + return "f:" + code +} + +// folderIDFromObjID extracts the numeric folder ID string from an object ID. +func folderIDFromObjID(id string) string { + if strings.HasPrefix(id, "d:") { + return strings.TrimPrefix(id, "d:") + } + if id == "" || id == "/" { + return "0" + } + return id +} + +// fileCodeFromObjID extracts the file code from an object ID. +func fileCodeFromObjID(id string) string { + if strings.HasPrefix(id, "f:") { + return strings.TrimPrefix(id, "f:") + } + return id +} diff --git a/drivers/doubao_new/driver.go b/drivers/doubao_new/driver.go new file mode 100644 index 00000000000..af0f9dd2ac0 --- /dev/null +++ b/drivers/doubao_new/driver.go @@ -0,0 +1,598 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" +) + +type DoubaoNew struct { + model.Storage + Addition + TtLogid string +} + +func (d *DoubaoNew) Config() driver.Config { + return config +} + +func (d *DoubaoNew) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *DoubaoNew) Init(ctx context.Context) error { + // TODO login / refresh token + //op.MustSaveDriverStorage(d) + return nil +} + +func (d *DoubaoNew) Drop(ctx context.Context) error { + return nil +} + +func (d *DoubaoNew) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + nodes, err := d.listAllChildren(ctx, dir.GetID()) + if err != nil { + return nil, err + } + + objs := make([]model.Obj, 0, len(nodes)) + for _, node := range nodes { + size := parseSize(node.Extra.Size) + isFolder := node.Type == 0 + obj := &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: dir.GetID(), + Name: node.Name, + Size: size, + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: isFolder, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + } + objs = append(objs, obj) + } + + return objs, nil +} + +func (d *DoubaoNew) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj, ok := file.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + if obj.IsFolder { + return nil, errs.LinkIsDir + } + if args.Type == "preview" || args.Type == "thumb" { + if link, err := d.previewLink(ctx, obj, args); err == nil { + return link, nil + } + } + auth := d.resolveAuthorization() + dpop := d.resolveDpop() + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + + query := url.Values{} + query.Set("authorization", auth) + query.Set("dpop", dpop) + + downloadURL := DownloadBaseURL + "/space/api/box/stream/download/all/" + obj.ObjToken + "/?" + query.Encode() + + headers := http.Header{ + "Referer": []string{"https://www.doubao.com/"}, + "User-Agent": []string{base.UserAgent}, + } + + return &model.Link{ + URL: downloadURL, + Header: headers, + }, nil +} + +func (d *DoubaoNew) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + node, err := d.createFolder(ctx, parentDir.GetID(), dirName) + if err != nil { + return nil, err + } + return &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: parentDir.GetID(), + Name: node.Name, + Size: parseSize(node.Extra.Size), + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: true, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + }, nil +} + +func (d *DoubaoNew) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if dstDir == nil { + return nil, errors.New("nil destination dir") + } + srcToken := srcObj.GetID() + if srcToken == "" { + if obj, ok := srcObj.(*Object); ok { + srcToken = obj.ObjToken + } + } + if srcToken == "" { + return nil, errors.New("missing source token") + } + if err := d.moveObj(ctx, srcToken, dstDir.GetID()); err != nil { + return nil, err + } + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Path = dstDir.GetID() + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if srcObj.IsDir() { + if err := d.renameFolder(ctx, srcObj.GetID(), newName); err != nil { + return nil, err + } + } else { + fileToken := "" + if obj, ok := srcObj.(*Object); ok { + fileToken = obj.ObjToken + } + if fileToken == "" { + fileToken = srcObj.GetID() + } + if err := d.renameFile(ctx, fileToken, newName); err != nil { + return nil, err + } + } + + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Name = newName + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Remove(ctx context.Context, obj model.Obj) error { + if obj == nil { + return errors.New("nil object") + } + token := obj.GetID() + if token == "" { + if o, ok := obj.(*Object); ok { + token = o.ObjToken + } + } + if token == "" { + return errors.New("missing object token") + } + return d.removeObj(ctx, []string{token}) +} + +func (d *DoubaoNew) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if file == nil { + return nil, errors.New("nil file") + } + if file.GetSize() <= 0 { + return nil, errors.New("invalid file size") + } + + uploadPrep, err := d.prepareUpload(ctx, file.GetName(), file.GetSize(), dstDir.GetID()) + if err != nil { + return nil, err + } + if uploadPrep.BlockSize <= 0 { + return nil, errors.New("invalid block size from prepare") + } + + tmpFile, err := file.CacheFullInTempFile() + if err != nil { + return nil, err + } + defer tmpFile.Close() + + blockSize := uploadPrep.BlockSize + totalSize := file.GetSize() + numBlocks := int((totalSize + blockSize - 1) / blockSize) + blocks := make([]UploadBlock, 0, numBlocks) + blockMeta := make(map[int]UploadBlock, numBlocks) + + for seq := 0; seq < numBlocks; seq++ { + offset := int64(seq) * blockSize + length := blockSize + if remain := totalSize - offset; remain < length { + length = remain + } + buf := make([]byte, int(length)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + buf = buf[:n] + sum := sha256.Sum256(buf) + hash := base64.StdEncoding.EncodeToString(sum[:]) + checksum := adler32String(buf) + + block := UploadBlock{ + Hash: hash, + Seq: seq, + Size: int64(n), + Checksum: checksum, + IsUploaded: true, + } + blocks = append(blocks, block) + blockMeta[seq] = block + } + + needed, err := d.uploadBlocks(ctx, uploadPrep.UploadID, blocks, "explorer") + if err != nil { + return nil, err + } + + if len(needed.NeededUploadBlocks) > 0 { + sort.Slice(needed.NeededUploadBlocks, func(i, j int) bool { + return needed.NeededUploadBlocks[i].Seq < needed.NeededUploadBlocks[j].Seq + }) + const maxMergeBlockCount = 20 + var ( + groupSeqs []int + groupChecksums []string + groupSizes []int64 + groupRealSize int64 + groupExpectSum int64 + groupBuf bytes.Buffer + uploadedBytes int64 + ) + + flushGroup := func() error { + if len(groupSeqs) == 0 { + return nil + } + data := groupBuf.Bytes() + expectLen := groupExpectSum + if len(data) > 0 { + headLen := 32 + if len(data) < headLen { + headLen = len(data) + } + tailLen := 32 + if len(data) < tailLen { + tailLen = len(data) + } + } + if int64(len(data)) != expectLen { + return fmt.Errorf("[doubao_new] merge blocks invalid body len: got=%d expect=%d seqs=%v", len(data), expectLen, groupSeqs) + } + mergeResp, err := d.mergeUploadBlocks(ctx, uploadPrep.UploadID, groupSeqs, groupChecksums, groupSizes, blockSize, data) + if err != nil { + return err + } + if len(mergeResp.SuccessSeqList) != len(groupSeqs) { + return fmt.Errorf("[doubao_new] merge blocks incomplete: %v", mergeResp.SuccessSeqList) + } + success := make(map[int]bool, len(mergeResp.SuccessSeqList)) + for _, seq := range mergeResp.SuccessSeqList { + success[seq] = true + } + for _, seq := range groupSeqs { + if !success[seq] { + return fmt.Errorf("[doubao_new] merge blocks missing seq %d", seq) + } + } + + uploadedBytes += groupRealSize + groupSeqs = groupSeqs[:0] + groupChecksums = groupChecksums[:0] + groupSizes = groupSizes[:0] + groupRealSize = 0 + groupExpectSum = 0 + groupBuf.Reset() + if up != nil { + percent := float64(uploadedBytes) / float64(totalSize) * 100 + up(percent) + } + return nil + } + + for _, item := range needed.NeededUploadBlocks { + if _, ok := blockMeta[item.Seq]; !ok { + return nil, fmt.Errorf("[doubao_new] missing block meta for seq %d", item.Seq) + } + if item.Size <= 0 { + return nil, fmt.Errorf("[doubao_new] invalid block size from needed list: seq=%d size=%d", item.Seq, item.Size) + } + offset := int64(item.Seq) * blockSize + buf := make([]byte, int(item.Size)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + if n != len(buf) { + return nil, fmt.Errorf("[doubao_new] short read: seq=%d want=%d got=%d", item.Seq, len(buf), n) + } + buf = buf[:n] + realAdler := adler32String(buf) + if realAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] block checksum mismatch: seq=%d offset=%d adler32=%s step2=%s", item.Seq, offset, realAdler, item.Checksum) + } + payloadStart := groupBuf.Len() + groupBuf.Write(buf) + payloadEnd := groupBuf.Len() + payloadAdler := adler32String(groupBuf.Bytes()[payloadStart:payloadEnd]) + if payloadAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] payload checksum mismatch: seq=%d start=%d end=%d adler32=%s step2=%s", item.Seq, payloadStart, payloadEnd, payloadAdler, item.Checksum) + } + groupSeqs = append(groupSeqs, item.Seq) + groupChecksums = append(groupChecksums, item.Checksum) + groupSizes = append(groupSizes, item.Size) + groupRealSize += int64(n) + groupExpectSum += item.Size + if len(groupSeqs) >= maxMergeBlockCount { + if err := flushGroup(); err != nil { + return nil, err + } + } + } + + if err := flushGroup(); err != nil { + return nil, err + } + if up != nil { + up(100) + } + } else if up != nil { + up(100) + } + + numBlocksFinish := uploadPrep.NumBlocks + if numBlocksFinish <= 0 { + numBlocksFinish = numBlocks + } + finish, err := d.finishUpload(ctx, uploadPrep.UploadID, numBlocksFinish, "explorer") + if err != nil { + return nil, err + } + + nodeToken := finish.Extra.NodeToken + if nodeToken == "" { + nodeToken = finish.FileToken + } + now := time.Now() + return &Object{ + Object: model.Object{ + ID: nodeToken, + Path: dstDir.GetID(), + Name: file.GetName(), + Size: file.GetSize(), + Modified: now, + Ctime: now, + IsFolder: false, + }, + ObjToken: finish.FileToken, + }, nil +} + +func (d *DoubaoNew) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch args.Method { + case "doubao_preview", "preview": + obj, ok := args.Obj.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errs.NotSupport + } + + imgExt := ".webp" + pageNums := 1 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + if extra.PageNums > 0 { + pageNums = extra.PageNums + } + } + } + + return base.Json{ + "version": info.Version, + "img_ext": imgExt, + "page_nums": pageNums, + }, nil + default: + return nil, errs.NotSupport + } +} + +func (d *DoubaoNew) listAllChildren(ctx context.Context, parentToken string) ([]Node, error) { + nodes := make([]Node, 0, 50) + lastLabel := "" + for page := 0; page < 100; page++ { + data, err := d.listChildren(ctx, parentToken, lastLabel) + if err != nil { + return nil, err + } + + if len(data.NodeList) > 0 { + for _, token := range data.NodeList { + node, ok := data.Entities.Nodes[token] + if !ok { + continue + } + nodes = append(nodes, node) + } + } else { + for _, node := range data.Entities.Nodes { + nodes = append(nodes, node) + } + } + + if !data.HasMore || data.LastLabel == "" || data.LastLabel == lastLabel { + break + } + lastLabel = data.LastLabel + } + + if len(nodes) == 0 { + return nil, nil + } + return nodes, nil +} + +func (d *DoubaoNew) previewLink(ctx context.Context, obj *Object, args model.LinkArgs) (*model.Link, error) { + auth := d.resolveAuthorization() + dpop := d.resolveDpop() + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errors.New("preview not available") + } + + subID := "" + pageIndex := 0 + if args.HttpReq != nil { + query := args.HttpReq.URL.Query() + if v := query.Get("sub_id"); v != "" { + subID = v + } else if v := query.Get("page"); v != "" { + if p, err := strconv.Atoi(v); err == nil && p >= 0 { + pageIndex = p + } + } + } + if subID == "" { + imgExt := ".webp" + pageNums := 0 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + pageNums = extra.PageNums + } + } + if pageNums > 0 && pageIndex >= pageNums { + pageIndex = pageNums - 1 + } + subID = fmt.Sprintf("img_%d%s", pageIndex, imgExt) + } + + query := url.Values{} + query.Set("preview_type", "22") + query.Set("sub_id", subID) + if info.Version != "" { + query.Set("version", info.Version) + } + previewURL := fmt.Sprintf("%s/space/api/box/stream/download/preview_sub/%s?%s", BaseURL, obj.ObjToken, query.Encode()) + + headers := http.Header{ + "Referer": []string{"https://www.doubao.com/"}, + "User-Agent": []string{base.UserAgent}, + "Authorization": []string{auth}, + "Dpop": []string{dpop}, + } + + return &model.Link{ + URL: previewURL, + Header: headers, + }, nil +} + +func parseSize(size string) int64 { + if size == "" { + return 0 + } + val, err := strconv.ParseInt(size, 10, 64) + if err != nil { + return 0 + } + return val +} + +var _ driver.Driver = (*DoubaoNew)(nil) diff --git a/drivers/doubao_new/meta.go b/drivers/doubao_new/meta.go new file mode 100644 index 00000000000..5a8050a2653 --- /dev/null +++ b/drivers/doubao_new/meta.go @@ -0,0 +1,36 @@ +package doubao_new + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootID + // define other + Authorization string `json:"authorization" help:"DPoP access token (Authorization header value); optional if present in cookie"` + Dpop string `json:"dpop" help:"DPoP header value; optional if present in cookie"` + Cookie string `json:"cookie" help:"Optional cookie; only used to extract authorization/dpop tokens"` + Debug bool `json:"debug" help:"Enable debug logs for upload"` +} + +var config = driver.Config{ + Name: "DoubaoNew", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &DoubaoNew{} + }) +} diff --git a/drivers/doubao_new/types.go b/drivers/doubao_new/types.go new file mode 100644 index 00000000000..ea8acfc5b18 --- /dev/null +++ b/drivers/doubao_new/types.go @@ -0,0 +1,182 @@ +package doubao_new + +import "github.com/alist-org/alist/v3/internal/model" + +type BaseResp struct { + Code int `json:"code"` + Msg string `json:"msg,omitempty"` + Message string `json:"message,omitempty"` +} + +type ListResp struct { + BaseResp + Data ListData `json:"data"` +} + +type ListData struct { + HasMore bool `json:"has_more"` + LastLabel string `json:"last_label"` + NodeList []string `json:"node_list"` + Entities struct { + Nodes map[string]Node `json:"nodes"` + Users map[string]User `json:"users"` + } `json:"entities"` +} + +type Node struct { + Token string `json:"token"` + NodeToken string `json:"node_token"` + ObjToken string `json:"obj_token"` + Name string `json:"name"` + Type int `json:"type"` + NodeType int `json:"node_type"` + OwnerID string `json:"owner_id"` + EditUID string `json:"edit_uid"` + CreateTime int64 `json:"create_time"` + EditTime int64 `json:"edit_time"` + URL string `json:"url"` + Extra struct { + Size string `json:"size"` + } `json:"extra"` +} + +type User struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type Object struct { + model.Object + ObjToken string + NodeType int + ObjType int + URL string +} + +type CreateFolderResp struct { + BaseResp + Data struct { + Entities struct { + Nodes map[string]Node `json:"nodes"` + } `json:"entities"` + NodeList []string `json:"node_list"` + } `json:"data"` +} + +type FileInfoResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data FileInfo `json:"data"` +} + +type FileInfo struct { + Name string `json:"name"` + NumBlocks int `json:"num_blocks"` + Version string `json:"version"` + MimeType string `json:"mime_type"` + MountPoint string `json:"mount_point"` + PreviewMeta PreviewMeta `json:"preview_meta"` +} + +type PreviewMeta struct { + Data map[string]PreviewMetaEntry `json:"data"` +} + +type PreviewMetaEntry struct { + Status int `json:"status"` + Extra string `json:"extra"` + PreviewFileSize int64 `json:"preview_file_size"` +} + +type PreviewImageExtra struct { + ImgExt string `json:"img_ext"` + PageNums int `json:"page_nums"` +} + +type UserStorageResp struct { + BaseResp + Data UserStorageData `json:"data"` +} + +type UserStorageData struct { + ShowSizeLimit bool `json:"show_size_limit"` + TotalSizeLimitBytes int64 `json:"total_size_limit_bytes"` + UsedSizeBytes int64 `json:"used_size_bytes"` +} + +type UploadPrepareResp struct { + BaseResp + Data UploadPrepareData `json:"data"` +} + +type UploadPrepareData struct { + BlockSize int64 `json:"block_size"` + NumBlocks int `json:"num_blocks"` + OptionBlockSize int64 `json:"option_block_size"` + DedupeSupport bool `json:"dedupe_support"` + UploadID string `json:"upload_id"` +} + +type UploadBlock struct { + Hash string `json:"hash"` + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + IsUploaded bool `json:"isUploaded"` +} + +type UploadBlocksResp struct { + BaseResp + Data UploadBlocksData `json:"data"` +} + +type UploadBlocksData struct { + NeededUploadBlocks []UploadBlockNeed `json:"needed_upload_blocks"` +} + +type UploadBlockNeed struct { + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + Hash string `json:"hash"` +} + +type UploadMergeResp struct { + BaseResp + Data UploadMergeData `json:"data"` +} + +type UploadMergeData struct { + SuccessSeqList []int `json:"success_seq_list"` +} + +type UploadFinishResp struct { + BaseResp + Data UploadFinishData `json:"data"` +} + +type UploadFinishData struct { + Version string `json:"version"` + DataVersion string `json:"data_version"` + Extra struct { + NodeToken string `json:"node_token"` + } `json:"extra"` + FileToken string `json:"file_token"` +} + +type RemoveResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + } `json:"data"` +} + +type TaskStatusResp struct { + BaseResp + Data TaskStatusData `json:"data"` +} + +type TaskStatusData struct { + IsFinish bool `json:"is_finish"` + IsFail bool `json:"is_fail"` +} diff --git a/drivers/doubao_new/util.go b/drivers/doubao_new/util.go new file mode 100644 index 00000000000..5c21ca090db --- /dev/null +++ b/drivers/doubao_new/util.go @@ -0,0 +1,909 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "hash/adler32" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/go-resty/resty/v2" +) + +const ( + BaseURL = "https://my.feishu.cn" + DownloadBaseURL = "https://internal-api-drive-stream.feishu.cn" +) + +var defaultObjTypes = []string{"124", "0", "12", "30", "123", "22"} + +func (d *DoubaoNew) request(ctx context.Context, path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + + if callback != nil { + callback(req) + } + + res, err := req.Execute(method, BaseURL+path) + if err != nil { + return nil, err + } + if res != nil { + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + } + + body := res.Body() + var common BaseResp + if err = json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return body, fmt.Errorf(msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return body, fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + if resp != nil { + if err = json.Unmarshal(body, resp); err != nil { + return body, err + } + } + + return body, nil +} + +func getCookieValue(cookie, name string) string { + parts := strings.Split(cookie, ";") + prefix := name + "=" + for _, part := range parts { + part = strings.TrimSpace(part) + if strings.HasPrefix(part, prefix) { + return strings.TrimPrefix(part, prefix) + } + } + return "" +} + +func adler32String(data []byte) string { + sum := adler32.Checksum(data) + return strconv.FormatUint(uint64(sum), 10) +} + +func buildCommaHeader(items []string) string { + return strings.Join(items, ",") +} + +func joinIntComma(items []int) string { + if len(items) == 0 { + return "" + } + var sb strings.Builder + for i, v := range items { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(strconv.Itoa(v)) + } + return sb.String() +} + +func previewList(items []string, n int) string { + if n <= 0 || len(items) == 0 { + return "" + } + if len(items) < n { + n = len(items) + } + return strings.Join(items[:n], ",") +} + +func (d *DoubaoNew) resolveAuthorization() string { + auth := strings.TrimSpace(d.Authorization) + if auth == "" && d.Cookie != "" { + if token := getCookieValue(d.Cookie, "LARK_SUITE_ACCESS_TOKEN"); token != "" { + auth = token + } + } + if auth == "" { + return "" + } + if !strings.HasPrefix(auth, "DPoP ") && !strings.HasPrefix(auth, "dpop ") { + auth = "DPoP " + auth + } + return auth +} + +func (d *DoubaoNew) resolveDpop() string { + dpop := strings.TrimSpace(d.Dpop) + if dpop == "" && d.Cookie != "" { + dpop = getCookieValue(d.Cookie, "LARK_SUITE_DPOP") + } + return dpop +} + +func (d *DoubaoNew) listChildren(ctx context.Context, parentToken string, lastLabel string) (ListData, error) { + var resp ListResp + _, err := d.request(ctx, "/space/api/explorer/doubao/children/list/", http.MethodGet, func(req *resty.Request) { + values := url.Values{} + for _, t := range defaultObjTypes { + values.Add("obj_type", t) + } + values.Set("length", "50") + values.Set("rank", "0") + values.Set("asc", "0") + values.Set("min_length", "40") + values.Set("thumbnail_width", "1028") + values.Set("thumbnail_height", "1028") + values.Set("thumbnail_policy", "4") + if parentToken != "" { + values.Set("token", parentToken) + } + if lastLabel != "" { + values.Set("last_label", lastLabel) + } + req.SetQueryParamsFromValues(values) + }, &resp) + if err != nil { + return ListData{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) getFileInfo(ctx context.Context, fileToken string) (FileInfo, error) { + var resp FileInfoResp + _, err := d.request(ctx, "/space/api/box/file/info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "caller": "explorer", + "file_token": fileToken, + "mount_point": "explorer", + "option_params": []string{"preview_meta", "check_cipher"}, + }) + }, &resp) + if err != nil { + return FileInfo{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) createFolder(ctx context.Context, parentToken, name string) (Node, error) { + data := url.Values{} + data.Set("name", name) + data.Set("source", "0") + if parentToken != "" { + data.Set("parent_token", parentToken) + } + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/create/folder/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return Node{}, err + } + if err := decodeBaseResp(body, res); err != nil { + return Node{}, err + } + + var resp CreateFolderResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return Node{}, fmt.Errorf(msg) + } + + var node Node + if len(resp.Data.NodeList) > 0 { + if n, ok := resp.Data.Entities.Nodes[resp.Data.NodeList[0]]; ok { + node = n + } + } + if node.Token == "" { + for _, n := range resp.Data.Entities.Nodes { + node = n + break + } + } + if node.Token == "" && node.ObjToken == "" && node.NodeToken == "" { + return Node{}, fmt.Errorf("[doubao_new] create folder failed: empty response") + } + if node.NodeToken == "" { + if node.Token != "" { + node.NodeToken = node.Token + } else if node.ObjToken != "" { + node.NodeToken = node.ObjToken + } + } + if node.ObjToken == "" && node.Token != "" { + node.ObjToken = node.Token + } + return node, nil +} + +func (d *DoubaoNew) renameFolder(ctx context.Context, token, name string) error { + if token == "" { + return fmt.Errorf("[doubao_new] rename folder missing token") + } + data := url.Values{} + data.Set("token", token) + data.Set("name", name) + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/rename/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func isCsrfTokenError(body []byte, res *resty.Response) bool { + if len(body) == 0 { + return false + } + if strings.Contains(strings.ToLower(string(body)), "csrf token error") { + return true + } + if res != nil && res.StatusCode() == http.StatusForbidden { + return true + } + return false +} + +func doRequestWithCsrf(doRequest func(csrfToken string) (*resty.Response, []byte, error)) (*resty.Response, []byte, error) { + res, body, err := doRequest("") + if err != nil { + return res, body, err + } + if isCsrfTokenError(body, res) { + csrfToken := extractCsrfTokenFromResponse(res) + if csrfToken != "" { + return doRequest(csrfToken) + } + } + return res, body, err +} + +func extractCsrfTokenFromResponse(res *resty.Response) string { + if res == nil || res.Request == nil { + return "" + } + if res.Request.RawRequest != nil { + if csrf := getCookieValue(res.Request.RawRequest.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + } + if csrf := getCookieValue(res.Request.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + for _, c := range res.Cookies() { + if c.Name == "_csrf_token" { + return c.Value + } + } + return "" +} + +func decodeBaseResp(body []byte, res *resty.Response) error { + var common BaseResp + if err := json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf(msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + return nil +} + +func (d *DoubaoNew) renameFile(ctx context.Context, fileToken, name string) error { + if fileToken == "" { + return fmt.Errorf("[doubao_new] rename file missing file token") + } + _, err := d.request(ctx, "/space/api/box/file/update_info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "file_token": fileToken, + "name": name, + }) + }, nil) + return err +} + +func (d *DoubaoNew) moveObj(ctx context.Context, srcToken, destToken string) error { + if srcToken == "" { + return fmt.Errorf("[doubao_new] move missing src token") + } + data := url.Values{} + data.Set("src_token", srcToken) + if destToken != "" { + data.Set("dest_token", destToken) + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/move/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func (d *DoubaoNew) removeObj(ctx context.Context, tokens []string) error { + if len(tokens) == 0 { + return fmt.Errorf("[doubao_new] remove missing tokens") + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "tokens": tokens, + "apply": 1, + }) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v3/remove/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + var resp RemoveResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + if resp.Data.TaskID == "" { + return nil + } + return d.waitTask(ctx, resp.Data.TaskID) +} + +func (d *DoubaoNew) getUserStorage(ctx context.Context) (UserStorageData, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("agw-js-conv", "str") + req.SetHeader("content-type", "application/json") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if d.Cookie != "" { + req.SetHeader("cookie", d.Cookie) + } + req.SetBody(base.Json{}) + + res, err := req.Execute(http.MethodPost, "https://www.doubao.com/alice/aispace/facade/get_user_storage") + if err != nil { + return UserStorageData{}, err + } + + body := res.Body() + var resp UserStorageResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UserStorageData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UserStorageData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) waitTask(ctx context.Context, taskID string) error { + const ( + taskPollInterval = time.Second + taskPollMaxAttempts = 120 + ) + var lastErr error + for attempt := 0; attempt < taskPollMaxAttempts; attempt++ { + if attempt > 0 { + if err := waitWithContext(ctx, taskPollInterval); err != nil { + return err + } + } + status, err := d.getTaskStatus(ctx, taskID) + if err != nil { + lastErr = err + continue + } + if status.IsFail { + return fmt.Errorf("[doubao_new] remove task failed: %s", taskID) + } + if status.IsFinish { + return nil + } + } + if lastErr != nil { + return lastErr + } + return fmt.Errorf("[doubao_new] remove task timed out: %s", taskID) +} + +func (d *DoubaoNew) getTaskStatus(ctx context.Context, taskID string) (TaskStatusData, error) { + if taskID == "" { + return TaskStatusData{}, fmt.Errorf("[doubao_new] task status missing task_id") + } + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + req.SetQueryParam("task_id", taskID) + res, err := req.Execute(http.MethodGet, BaseURL+"/space/api/explorer/v2/task/") + if err != nil { + return TaskStatusData{}, err + } + body := res.Body() + var resp TaskStatusResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return TaskStatusData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return TaskStatusData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + return resp.Data, nil +} + +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (d *DoubaoNew) prepareUpload(ctx context.Context, name string, size int64, mountNodeToken string) (UploadPrepareData, error) { + var resp UploadPrepareResp + _, err := d.request(ctx, "/space/api/box/upload/prepare/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.prepare") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + body := base.Json{ + "mount_point": "explorer", + "mount_node_token": "", + "name": name, + "size": size, + "size_checker": true, + } + if mountNodeToken != "" { + body["mount_node_token"] = mountNodeToken + } + req.SetBody(body) + }, &resp) + if err != nil { + return UploadPrepareData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlocks(ctx context.Context, uploadID string, blocks []UploadBlock, mountPoint string) (UploadBlocksData, error) { + if uploadID == "" { + return UploadBlocksData{}, fmt.Errorf("[doubao_new] upload blocks missing upload_id") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadBlocksResp + _, err := d.request(ctx, "/space/api/box/upload/blocks/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.blocks") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetBody(base.Json{ + "blocks": blocks, + "upload_id": uploadID, + "mount_point": mountPoint, + }) + }, &resp) + if err != nil { + return UploadBlocksData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) mergeUploadBlocks(ctx context.Context, uploadID string, seqList []int, checksumList []string, sizeList []int64, blockOriginSize int64, data []byte) (UploadMergeData, error) { + if uploadID == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing upload_id") + } + if len(seqList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty seq list") + } + if len(checksumList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty checksum list") + } + if len(sizeList) != len(seqList) { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks size list mismatch") + } + if blockOriginSize <= 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks invalid block origin size") + } + if len(data) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty data") + } + + seqHeader := joinIntComma(seqList) + checksumHeader := buildCommaHeader(checksumList) + + client := base.NewRestyClient() + client.SetCookieJar(nil) + req := client.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("content-type", "application/octet-stream") + req.Header.Set("x-block-list-checksum", checksumHeader) + req.Header.Set("x-seq-list", seqHeader) + req.SetHeader("x-block-origin-size", strconv.FormatInt(blockOriginSize, 10)) + req.SetHeader("x-command", "space.api.box.stream.upload.merge_block") + req.SetHeader("x-csrftoken", "") + reqID := "" + if buf := make([]byte, 16); true { + if _, err := rand.Read(buf); err == nil { + reqID = hex.EncodeToString(buf) + } + } + if reqID != "" { + req.SetHeader("x-request-id", reqID) + } + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + req.Header.Del("Cookie") + req.Header.Del("cookie") + if req.Header.Get("x-command") == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing x-command header") + } + req.SetBody(data) + + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + urlStr := "https://internal-api-drive-stream.feishu.cn/space/api/box/stream/upload/merge_block/?" + values.Encode() + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return UploadMergeData{}, err + } + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + body := res.Body() + var resp UploadMergeResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UploadMergeData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + if res != nil && res.StatusCode() == http.StatusBadRequest && resp.Code == 2 { + success := make([]int, 0, len(seqList)) + offset := 0 + for i, seq := range seqList { + size := sizeList[i] + if size <= 0 { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback invalid size: seq=%d size=%d", seq, size) + } + if offset+int(size) > len(data) { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback payload out of range: seq=%d offset=%d size=%d total=%d", seq, offset, size, len(data)) + } + payload := data[offset : offset+int(size)] + block := UploadBlockNeed{ + Seq: seq, + Size: size, + Checksum: checksumList[i], + } + if err := d.uploadBlockV3(ctx, uploadID, block, payload); err != nil { + return UploadMergeData{SuccessSeqList: success}, err + } + success = append(success, seq) + offset += int(size) + } + return UploadMergeData{SuccessSeqList: success}, nil + } + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UploadMergeData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlockV3(ctx context.Context, uploadID string, block UploadBlockNeed, data []byte) error { + if uploadID == "" { + return fmt.Errorf("[doubao_new] upload v3 block missing upload_id") + } + if block.Seq < 0 { + return fmt.Errorf("[doubao_new] upload v3 block invalid seq") + } + if len(data) == 0 { + return fmt.Errorf("[doubao_new] upload v3 block empty data") + } + + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("x-block-seq", strconv.Itoa(block.Seq)) + req.SetHeader("x-block-checksum", block.Checksum) + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + + req.SetMultipartFormData(map[string]string{ + "upload_id": uploadID, + "size": strconv.FormatInt(int64(len(data)), 10), + }) + req.SetMultipartField("file", "blob", "application/octet-stream", bytes.NewReader(data)) + + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("seq", strconv.Itoa(block.Seq)) + values.Set("size", strconv.FormatInt(int64(len(data)), 10)) + values.Set("checksum", block.Checksum) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + urlStr := "https://internal-api-drive-stream.feishu.cn/space/api/box/stream/upload/v3/block/?" + values.Encode() + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return err + } + body := res.Body() + if err := decodeBaseResp(body, res); err != nil { + return err + } + return nil +} + +func (d *DoubaoNew) finishUpload(ctx context.Context, uploadID string, numBlocks int, mountPoint string) (UploadFinishData, error) { + if uploadID == "" { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload missing upload_id") + } + if numBlocks <= 0 { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload invalid num_blocks") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadFinishResp + _, err := d.request(ctx, "/space/api/box/upload/finish/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.finish") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetHeader("biz-scene", "file_upload") + req.SetHeader("biz-ua-type", "Web") + req.SetBody(base.Json{ + "upload_id": uploadID, + "num_blocks": numBlocks, + "mount_point": mountPoint, + "push_open_history_record": 1, + }) + }, &resp) + if err != nil { + return UploadFinishData{}, err + } + return resp.Data, nil +} diff --git a/drivers/ftps/driver.go b/drivers/ftps/driver.go new file mode 100644 index 00000000000..4467380f09c --- /dev/null +++ b/drivers/ftps/driver.go @@ -0,0 +1,127 @@ +package ftps + +import ( + "context" + stdpath "path" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/jlaffaye/ftp" +) + +type FTPS struct { + model.Storage + Addition + conn *ftp.ServerConn +} + +func (d *FTPS) Config() driver.Config { + return config +} + +func (d *FTPS) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *FTPS) Init(ctx context.Context) error { + return d.login() +} + +func (d *FTPS) Drop(ctx context.Context) error { + if d.conn != nil { + _ = d.conn.Logout() + } + return nil +} + +func (d *FTPS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.login(); err != nil { + return nil, err + } + entries, err := d.conn.List(encode(dir.GetPath(), d.Encoding)) + if err != nil { + return nil, err + } + res := make([]model.Obj, 0) + for _, entry := range entries { + if entry.Name == "." || entry.Name == ".." { + continue + } + f := model.Object{ + Name: decode(entry.Name, d.Encoding), + Size: int64(entry.Size), + Modified: entry.Time, + IsFolder: entry.Type == ftp.EntryTypeFolder, + } + res = append(res, &f) + } + return res, nil +} + +func (d *FTPS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.login(); err != nil { + return nil, err + } + r := NewFileReader(d.conn, encode(file.GetPath(), d.Encoding), file.GetSize()) + link := &model.Link{ + MFile: r, + } + return link, nil +} + +func (d *FTPS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.login(); err != nil { + return err + } + return d.conn.MakeDir(encode(stdpath.Join(parentDir.GetPath(), dirName), d.Encoding)) +} + +func (d *FTPS) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.login(); err != nil { + return err + } + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(dstDir.GetPath(), srcObj.GetName()), d.Encoding), + ) +} + +func (d *FTPS) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.login(); err != nil { + return err + } + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), d.Encoding), + ) +} + +func (d *FTPS) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *FTPS) Remove(ctx context.Context, obj model.Obj) error { + if err := d.login(); err != nil { + return err + } + path := encode(obj.GetPath(), d.Encoding) + if obj.IsDir() { + return d.conn.RemoveDirRecur(path) + } else { + return d.conn.Delete(path) + } +} + +func (d *FTPS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + if err := d.login(); err != nil { + return err + } + path := stdpath.Join(dstDir.GetPath(), s.GetName()) + return d.conn.Stor(encode(path, d.Encoding), driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + })) +} + +var _ driver.Driver = (*FTPS)(nil) diff --git a/drivers/ftps/meta.go b/drivers/ftps/meta.go new file mode 100644 index 00000000000..a752ec01aa6 --- /dev/null +++ b/drivers/ftps/meta.go @@ -0,0 +1,46 @@ +package ftps + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" + "github.com/axgle/mahonia" +) + +func encode(str string, encoding string) string { + if encoding == "" { + return str + } + encoder := mahonia.NewEncoder(encoding) + return encoder.ConvertString(str) +} + +func decode(str string, encoding string) string { + if encoding == "" { + return str + } + decoder := mahonia.NewDecoder(encoding) + return decoder.ConvertString(str) +} + +type Addition struct { + Address string `json:"address" required:"true"` + Encoding string `json:"encoding" required:"false"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + TLSMode string `json:"tls_mode" type:"select" options:"Explicit,Implicit" default:"Explicit" required:"true" help:"Explicit: STARTTLS on port 21; Implicit: direct TLS on port 990"` + TLSInsecureSkipVerify bool `json:"tls_insecure_skip_verify" default:"false" help:"Allow insecure TLS connections (e.g. self-signed certificates)"` + driver.RootPath +} + +var config = driver.Config{ + Name: "FTPS", + LocalSort: true, + OnlyLocal: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &FTPS{} + }) +} diff --git a/drivers/ftps/types.go b/drivers/ftps/types.go new file mode 100644 index 00000000000..d01dc417a16 --- /dev/null +++ b/drivers/ftps/types.go @@ -0,0 +1 @@ +package ftps diff --git a/drivers/ftps/util.go b/drivers/ftps/util.go new file mode 100644 index 00000000000..10680fa8b80 --- /dev/null +++ b/drivers/ftps/util.go @@ -0,0 +1,139 @@ +package ftps + +import ( + "crypto/tls" + "io" + "net" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/jlaffaye/ftp" +) + +func (d *FTPS) login() error { + if d.conn != nil { + _, err := d.conn.CurrentDir() + if err == nil { + return nil + } + } + + host, _, err := net.SplitHostPort(d.Address) + if err != nil { + host = d.Address + } + + tlsConfig := &tls.Config{ + ServerName: host, + InsecureSkipVerify: d.TLSInsecureSkipVerify, + } + + opts := []ftp.DialOption{ + ftp.DialWithShutTimeout(10 * time.Second), + } + if d.TLSMode == "Implicit" { + opts = append(opts, ftp.DialWithTLS(tlsConfig)) + } else { + opts = append(opts, ftp.DialWithExplicitTLS(tlsConfig)) + } + + conn, err := ftp.Dial(d.Address, opts...) + if err != nil { + return err + } + err = conn.Login(d.Username, d.Password) + if err != nil { + _ = conn.Quit() + return err + } + d.conn = conn + return nil +} + +type FileReader struct { + conn *ftp.ServerConn + resp *ftp.Response + offset atomic.Int64 + readAtOffset int64 + mu sync.Mutex + path string + size int64 +} + +func NewFileReader(conn *ftp.ServerConn, path string, size int64) *FileReader { + return &FileReader{ + conn: conn, + path: path, + size: size, + } +} + +func (r *FileReader) Read(buf []byte) (n int, err error) { + r.mu.Lock() + defer r.mu.Unlock() + off := r.offset.Load() + n, err = r.readAtLocked(buf, off) + r.offset.Add(int64(n)) + return +} + +func (r *FileReader) ReadAt(buf []byte, off int64) (n int, err error) { + if off < 0 { + return 0, os.ErrInvalid + } + r.mu.Lock() + defer r.mu.Unlock() + return r.readAtLocked(buf, off) +} + +func (r *FileReader) readAtLocked(buf []byte, off int64) (n int, err error) { + if r.resp != nil && off != r.readAtOffset { + _ = r.resp.Close() + r.resp = nil + } + + if r.resp == nil { + r.resp, err = r.conn.RetrFrom(r.path, uint64(off)) + r.readAtOffset = off + if err != nil { + return 0, err + } + } + + n, err = r.resp.Read(buf) + r.readAtOffset += int64(n) + return +} + +func (r *FileReader) Seek(offset int64, whence int) (int64, error) { + oldOffset := r.offset.Load() + var newOffset int64 + switch whence { + case io.SeekStart: + newOffset = offset + case io.SeekCurrent: + newOffset = oldOffset + offset + case io.SeekEnd: + newOffset = r.size + offset + default: + return -1, os.ErrInvalid + } + + if newOffset < 0 { + return oldOffset, os.ErrInvalid + } + if newOffset == oldOffset { + return oldOffset, nil + } + r.offset.Store(newOffset) + return newOffset, nil +} + +func (r *FileReader) Close() error { + if r.resp != nil { + return r.resp.Close() + } + return nil +} diff --git a/drivers/gitee/driver.go b/drivers/gitee/driver.go new file mode 100644 index 00000000000..78a400941b9 --- /dev/null +++ b/drivers/gitee/driver.go @@ -0,0 +1,224 @@ +package gitee + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type Gitee struct { + model.Storage + Addition + client *resty.Client +} + +func (d *Gitee) Config() driver.Config { + return config +} + +func (d *Gitee) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Gitee) Init(ctx context.Context) error { + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + d.Endpoint = strings.TrimSpace(d.Endpoint) + if d.Endpoint == "" { + d.Endpoint = "https://gitee.com/api/v5" + } + d.Endpoint = strings.TrimSuffix(d.Endpoint, "/") + d.Owner = strings.TrimSpace(d.Owner) + d.Repo = strings.TrimSpace(d.Repo) + d.Token = strings.TrimSpace(d.Token) + d.DownloadProxy = strings.TrimSpace(d.DownloadProxy) + if d.Owner == "" || d.Repo == "" { + return errors.New("owner and repo are required") + } + d.client = base.NewRestyClient(). + SetBaseURL(d.Endpoint). + SetHeader("Accept", "application/json") + repo, err := d.getRepo() + if err != nil { + return err + } + d.Ref = strings.TrimSpace(d.Ref) + if d.Ref == "" { + d.Ref = repo.DefaultBranch + } + return nil +} + +func (d *Gitee) Drop(ctx context.Context) error { + return nil +} + +func (d *Gitee) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + relPath := d.relativePath(dir.GetPath()) + contents, err := d.listContents(relPath) + if err != nil { + return nil, err + } + objs := make([]model.Obj, 0, len(contents)) + for i := range contents { + objs = append(objs, contents[i].toModelObj()) + } + return objs, nil +} + +func (d *Gitee) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var downloadURL string + if obj, ok := file.(*Object); ok { + downloadURL = obj.DownloadURL + if downloadURL == "" { + relPath := d.relativePath(file.GetPath()) + content, err := d.getContent(relPath) + if err != nil { + return nil, err + } + if content.DownloadURL == "" { + return nil, errors.New("empty download url") + } + obj.DownloadURL = content.DownloadURL + downloadURL = content.DownloadURL + } + } else { + relPath := d.relativePath(file.GetPath()) + content, err := d.getContent(relPath) + if err != nil { + return nil, err + } + if content.DownloadURL == "" { + return nil, errors.New("empty download url") + } + downloadURL = content.DownloadURL + } + url := d.applyProxy(downloadURL) + return &model.Link{ + URL: url, + Header: http.Header{ + "Cookie": {d.Cookie}, + }, + }, nil +} + +func (d *Gitee) newRequest() *resty.Request { + req := d.client.R() + if d.Token != "" { + req.SetQueryParam("access_token", d.Token) + } + if d.Ref != "" { + req.SetQueryParam("ref", d.Ref) + } + return req +} + +func (d *Gitee) apiPath(path string) string { + escapedOwner := url.PathEscape(d.Owner) + escapedRepo := url.PathEscape(d.Repo) + if path == "" { + return fmt.Sprintf("/repos/%s/%s/contents", escapedOwner, escapedRepo) + } + return fmt.Sprintf("/repos/%s/%s/contents/%s", escapedOwner, escapedRepo, encodePath(path)) +} + +func (d *Gitee) listContents(path string) ([]Content, error) { + res, err := d.newRequest().Get(d.apiPath(path)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var contents []Content + if err := utils.Json.Unmarshal(res.Body(), &contents); err != nil { + var single Content + if err2 := utils.Json.Unmarshal(res.Body(), &single); err2 == nil && single.Type != "" { + if single.Type != "dir" { + return nil, errs.NotFolder + } + return []Content{}, nil + } + return nil, err + } + for i := range contents { + contents[i].Path = joinPath(path, contents[i].Name) + } + return contents, nil +} + +func (d *Gitee) getContent(path string) (*Content, error) { + res, err := d.newRequest().Get(d.apiPath(path)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var content Content + if err := utils.Json.Unmarshal(res.Body(), &content); err != nil { + return nil, err + } + if content.Type == "" { + return nil, errors.New("invalid response") + } + if content.Path == "" { + content.Path = path + } + return &content, nil +} + +func (d *Gitee) relativePath(full string) string { + full = utils.FixAndCleanPath(full) + root := utils.FixAndCleanPath(d.RootFolderPath) + if root == "/" { + return strings.TrimPrefix(full, "/") + } + if utils.PathEqual(full, root) { + return "" + } + prefix := utils.PathAddSeparatorSuffix(root) + if strings.HasPrefix(full, prefix) { + return strings.TrimPrefix(full, prefix) + } + return strings.TrimPrefix(full, "/") +} + +func (d *Gitee) applyProxy(raw string) string { + if raw == "" || d.DownloadProxy == "" { + return raw + } + proxy := d.DownloadProxy + if !strings.HasSuffix(proxy, "/") { + proxy += "/" + } + return proxy + strings.TrimLeft(raw, "/") +} + +func encodePath(p string) string { + if p == "" { + return "" + } + parts := strings.Split(p, "/") + for i, part := range parts { + parts[i] = url.PathEscape(part) + } + return strings.Join(parts, "/") +} + +func joinPath(base, name string) string { + if base == "" { + return name + } + return strings.TrimPrefix(stdpath.Join(base, name), "./") +} diff --git a/drivers/gitee/meta.go b/drivers/gitee/meta.go new file mode 100644 index 00000000000..2f926d635f3 --- /dev/null +++ b/drivers/gitee/meta.go @@ -0,0 +1,29 @@ +package gitee + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Endpoint string `json:"endpoint" type:"string" help:"Gitee API endpoint, default https://gitee.com/api/v5"` + Token string `json:"token" type:"string"` + Owner string `json:"owner" type:"string" required:"true"` + Repo string `json:"repo" type:"string" required:"true"` + Ref string `json:"ref" type:"string" help:"Branch, tag or commit SHA, defaults to repository default branch"` + DownloadProxy string `json:"download_proxy" type:"string" help:"Prefix added before download URLs, e.g. https://mirror.example.com/"` + Cookie string `json:"cookie" type:"string" help:"Cookie returned from user info request"` +} + +var config = driver.Config{ + Name: "Gitee", + LocalSort: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Gitee{} + }) +} diff --git a/drivers/gitee/types.go b/drivers/gitee/types.go new file mode 100644 index 00000000000..c10536a5d10 --- /dev/null +++ b/drivers/gitee/types.go @@ -0,0 +1,60 @@ +package gitee + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Links struct { + Self string `json:"self"` + Html string `json:"html"` +} + +type Content struct { + Type string `json:"type"` + Size *int64 `json:"size"` + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + URL string `json:"url"` + HtmlURL string `json:"html_url"` + DownloadURL string `json:"download_url"` + Links Links `json:"_links"` +} + +func (c Content) toModelObj() model.Obj { + size := int64(0) + if c.Size != nil { + size = *c.Size + } + return &Object{ + Object: model.Object{ + ID: c.Path, + Name: c.Name, + Size: size, + Modified: time.Unix(0, 0), + IsFolder: c.Type == "dir", + }, + DownloadURL: c.DownloadURL, + HtmlURL: c.HtmlURL, + } +} + +type Object struct { + model.Object + DownloadURL string + HtmlURL string +} + +func (o *Object) URL() string { + return o.DownloadURL +} + +type Repo struct { + DefaultBranch string `json:"default_branch"` +} + +type ErrResp struct { + Message string `json:"message"` +} diff --git a/drivers/gitee/util.go b/drivers/gitee/util.go new file mode 100644 index 00000000000..fbef972ad3d --- /dev/null +++ b/drivers/gitee/util.go @@ -0,0 +1,44 @@ +package gitee + +import ( + "fmt" + "net/url" + + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +func (d *Gitee) getRepo() (*Repo, error) { + req := d.client.R() + if d.Token != "" { + req.SetQueryParam("access_token", d.Token) + } + if d.Cookie != "" { + req.SetHeader("Cookie", d.Cookie) + } + escapedOwner := url.PathEscape(d.Owner) + escapedRepo := url.PathEscape(d.Repo) + res, err := req.Get(fmt.Sprintf("/repos/%s/%s", escapedOwner, escapedRepo)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var repo Repo + if err := utils.Json.Unmarshal(res.Body(), &repo); err != nil { + return nil, err + } + if repo.DefaultBranch == "" { + return nil, fmt.Errorf("failed to fetch default branch") + } + return &repo, nil +} + +func toErr(res *resty.Response) error { + var errMsg ErrResp + if err := utils.Json.Unmarshal(res.Body(), &errMsg); err == nil && errMsg.Message != "" { + return fmt.Errorf("%s: %s", res.Status(), errMsg.Message) + } + return fmt.Errorf(res.Status()) +} diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go index b35aa57a41c..3268dc2fd88 100644 --- a/drivers/github_releases/driver.go +++ b/drivers/github_releases/driver.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "sync" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -36,88 +37,130 @@ func (d *GithubReleases) Drop(ctx context.Context) error { return nil } -func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files := make([]File, 0) - path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) +// processPoint 处理单个挂载点的文件列表 +func (d *GithubReleases) processPoint(point *MountPoint, path string, args model.ListArgs) []File { + var pointFiles []File - for i := range d.points { - point := &d.points[i] + if !d.Addition.ShowAllVersion { // latest + point.RequestLatestRelease(d.GetRequest, args.Refresh) + pointFiles = d.processLatestVersion(point, path) + } else { // all version + point.RequestReleases(d.GetRequest, args.Refresh) + pointFiles = d.processAllVersions(point, path) + } - if !d.Addition.ShowAllVersion { // latest - point.RequestRelease(d.GetRequest, args.Refresh) + return pointFiles +} - if point.Point == path { // 与仓库路径相同 - files = append(files, point.GetLatestRelease()...) - if d.Addition.ShowReadme { - files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) - } - } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 - nextDir := GetNextDir(point.Point, path) - if nextDir == "" { - continue - } +// processLatestVersion 处理最新版本的逻辑 +func (d *GithubReleases) processLatestVersion(point *MountPoint, path string) []File { + var pointFiles []File - hasSameDir := false - for index := range files { - if files[index].GetName() == nextDir { - hasSameDir = true - files[index].Size += point.GetLatestSize() - break - } - } - if !hasSameDir { - files = append(files, File{ - Path: path + "/" + nextDir, - FileName: nextDir, - Size: point.GetLatestSize(), - UpdateAt: point.Release.PublishedAt, - CreateAt: point.Release.CreatedAt, - Type: "dir", - Url: "", - }) - } + if point.Point == path { // 与仓库路径相同 + pointFiles = append(pointFiles, point.GetLatestRelease()...) + if d.Addition.ShowReadme { + files := point.GetOtherFile(d.GetRequest, false) + pointFiles = append(pointFiles, files...) + } + } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + nextDir := GetNextDir(point.Point, path) + if nextDir != "" { + dirFile := File{ + Path: path + "/" + nextDir, + FileName: nextDir, + Size: point.GetLatestSize(), + UpdateAt: point.Release.PublishedAt, + CreateAt: point.Release.CreatedAt, + Type: "dir", + Url: "", } - } else { // all version - point.RequestReleases(d.GetRequest, args.Refresh) + pointFiles = append(pointFiles, dirFile) + } + } - if point.Point == path { // 与仓库路径相同 - files = append(files, point.GetAllVersion()...) - if d.Addition.ShowReadme { - files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) - } - } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 - nextDir := GetNextDir(point.Point, path) - if nextDir == "" { - continue - } + return pointFiles +} - hasSameDir := false - for index := range files { - if files[index].GetName() == nextDir { - hasSameDir = true - files[index].Size += point.GetAllVersionSize() - break - } - } - if !hasSameDir { - files = append(files, File{ - FileName: nextDir, - Path: path + "/" + nextDir, - Size: point.GetAllVersionSize(), - UpdateAt: (*point.Releases)[0].PublishedAt, - CreateAt: (*point.Releases)[0].CreatedAt, - Type: "dir", - Url: "", - }) - } - } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 - tagName := GetNextDir(path, point.Point) - if tagName == "" { - continue - } +// processAllVersions 处理所有版本的逻辑 +func (d *GithubReleases) processAllVersions(point *MountPoint, path string) []File { + var pointFiles []File - files = append(files, point.GetReleaseByTagName(tagName)...) + if point.Point == path { // 与仓库路径相同 + pointFiles = append(pointFiles, point.GetAllVersion()...) + if d.Addition.ShowReadme { + files := point.GetOtherFile(d.GetRequest, false) + pointFiles = append(pointFiles, files...) + } + } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + nextDir := GetNextDir(point.Point, path) + if nextDir != "" { + dirFile := File{ + FileName: nextDir, + Path: path + "/" + nextDir, + Size: point.GetAllVersionSize(), + UpdateAt: (*point.Releases)[0].PublishedAt, + CreateAt: (*point.Releases)[0].CreatedAt, + Type: "dir", + Url: "", } + pointFiles = append(pointFiles, dirFile) + } + } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 + tagName := GetNextDir(path, point.Point) + if tagName != "" { + pointFiles = append(pointFiles, point.GetReleaseByTagName(tagName)...) + } + } + + return pointFiles +} + +// mergeFiles 合并文件列表，处理重复目录 +func (d *GithubReleases) mergeFiles(files *[]File, newFiles []File) { + for _, newFile := range newFiles { + if newFile.Type == "dir" { + hasSameDir := false + for index := range *files { + if (*files)[index].GetName() == newFile.GetName() && (*files)[index].Type == "dir" { + hasSameDir = true + (*files)[index].Size += newFile.Size + break + } + } + if !hasSameDir { + *files = append(*files, newFile) + } + } else { + *files = append(*files, newFile) + } + } +} + +func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files := make([]File, 0) + path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) + + if d.Addition.ConcurrentRequests && d.Addition.Token != "" { // 并发处理 + var mu sync.Mutex + var wg sync.WaitGroup + + for i := range d.points { + wg.Add(1) + go func(point *MountPoint) { + defer wg.Done() + pointFiles := d.processPoint(point, path, args) + + mu.Lock() + d.mergeFiles(&files, pointFiles) + mu.Unlock() + }(&d.points[i]) + } + wg.Wait() + } else { // 串行处理 + for i := range d.points { + point := &d.points[i] + pointFiles := d.processPoint(point, path, args) + d.mergeFiles(&files, pointFiles) } } diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go index 47b84d37927..b54cb3cc608 100644 --- a/drivers/github_releases/meta.go +++ b/drivers/github_releases/meta.go @@ -7,11 +7,12 @@ import ( type Addition struct { driver.RootID - RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"` - ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` - Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` - ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` - GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` + RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"` + ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` + Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` + ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` + ConcurrentRequests bool `json:"concurrent_requests" type:"bool" default:"false" help:"To concurrently request the GitHub API, you must enter a GitHub token"` + GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` } var config = driver.Config{ diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go index b0a9ee619e0..b4562056185 100644 --- a/drivers/github_releases/types.go +++ b/drivers/github_releases/types.go @@ -18,7 +18,7 @@ type MountPoint struct { } // 请求最新版本 -func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) { +func (m *MountPoint) RequestLatestRelease(get func(url string) (*resty.Response, error), refresh bool) { if m.Repo == "" { return } diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go index df846e8a109..097295bf408 100644 --- a/drivers/github_releases/util.go +++ b/drivers/github_releases/util.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" - log "github.com/sirupsen/logrus" ) // 发送 GET 请求 @@ -23,7 +23,7 @@ func (d *GithubReleases) GetRequest(url string) (*resty.Response, error) { return nil, err } if res.StatusCode() != 200 { - log.Warn("failed to get request: ", res.StatusCode(), res.String()) + utils.Log.Warnf("failed to get request: %s %d %s", url, res.StatusCode(), res.String()) } return res, nil } diff --git a/drivers/gofile/driver.go b/drivers/gofile/driver.go new file mode 100644 index 00000000000..8046bd163fb --- /dev/null +++ b/drivers/gofile/driver.go @@ -0,0 +1,271 @@ +package gofile + +import ( + "context" + "fmt" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" +) + +type Gofile struct { + model.Storage + Addition + + accountId string +} + +func (d *Gofile) Config() driver.Config { + return config +} + +func (d *Gofile) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Gofile) Init(ctx context.Context) error { + if d.APIToken == "" { + return fmt.Errorf("API token is required") + } + + // Get account ID + accountId, err := d.getAccountId(ctx) + if err != nil { + return fmt.Errorf("failed to get account ID: %w", err) + } + d.accountId = accountId + + // Get account info to set root folder if not specified + if d.RootFolderID == "" { + accountInfo, err := d.getAccountInfo(ctx, accountId) + if err != nil { + return fmt.Errorf("failed to get account info: %w", err) + } + d.RootFolderID = accountInfo.Data.RootFolder + } + + // Save driver storage + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Gofile) Drop(ctx context.Context) error { + return nil +} + +func (d *Gofile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var folderId string + if dir.GetID() == "" { + folderId = d.GetRootId() + } else { + folderId = dir.GetID() + } + + endpoint := fmt.Sprintf("/contents/%s", folderId) + + var response ContentsResponse + err := d.getJSON(ctx, endpoint, &response) + if err != nil { + return nil, err + } + + var objects []model.Obj + + // Process children or contents + contents := response.Data.Children + if contents == nil { + contents = response.Data.Contents + } + + for _, content := range contents { + objects = append(objects, d.convertContentToObj(content)) + } + + return objects, nil +} + +func (d *Gofile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + + // Create a direct link for the file + directLink, err := d.createDirectLink(ctx, file.GetID()) + if err != nil { + return nil, fmt.Errorf("failed to create direct link: %w", err) + } + + // Configure cache expiration based on user setting + link := &model.Link{ + URL: directLink, + } + + // Only set expiration if LinkExpiry > 0 (0 means no caching) + if d.LinkExpiry > 0 { + expiration := time.Duration(d.LinkExpiry) * 24 * time.Hour + link.Expiration = &expiration + } + + return link, nil +} + +func (d *Gofile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var parentId string + if parentDir.GetID() == "" { + parentId = d.GetRootId() + } else { + parentId = parentDir.GetID() + } + + data := map[string]interface{}{ + "parentFolderId": parentId, + "folderName": dirName, + } + + var response CreateFolderResponse + err := d.postJSON(ctx, "/contents/createFolder", data, &response) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: response.Data.ID, + Name: response.Data.Name, + IsFolder: true, + }, nil +} + +func (d *Gofile) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var dstId string + if dstDir.GetID() == "" { + dstId = d.GetRootId() + } else { + dstId = dstDir.GetID() + } + + data := map[string]interface{}{ + "contentsId": srcObj.GetID(), + "folderId": dstId, + } + + err := d.putJSON(ctx, "/contents/move", data, nil) + if err != nil { + return nil, err + } + + // Return updated object + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + data := map[string]interface{}{ + "attribute": "name", + "attributeValue": newName, + } + + var response UpdateResponse + err := d.putJSON(ctx, fmt.Sprintf("/contents/%s/update", srcObj.GetID()), data, &response) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var dstId string + if dstDir.GetID() == "" { + dstId = d.GetRootId() + } else { + dstId = dstDir.GetID() + } + + data := map[string]interface{}{ + "contentsId": srcObj.GetID(), + "folderId": dstId, + } + + var response CopyResponse + err := d.postJSON(ctx, "/contents/copy", data, &response) + if err != nil { + return nil, err + } + + // Get the new ID from the response + newId := srcObj.GetID() + if response.Data.CopiedContents != nil { + if id, ok := response.Data.CopiedContents[srcObj.GetID()]; ok { + newId = id + } + } + + return &model.Object{ + ID: newId, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Remove(ctx context.Context, obj model.Obj) error { + data := map[string]interface{}{ + "contentsId": obj.GetID(), + } + + return d.deleteJSON(ctx, "/contents", data) +} + +func (d *Gofile) Put(ctx context.Context, dstDir model.Obj, fileStreamer model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var folderId string + if dstDir.GetID() == "" { + folderId = d.GetRootId() + } else { + folderId = dstDir.GetID() + } + + response, err := d.uploadFile(ctx, folderId, fileStreamer, up) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: response.Data.FileId, + Name: response.Data.FileName, + Size: fileStreamer.GetSize(), + IsFolder: false, + }, nil +} + +func (d *Gofile) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*Gofile)(nil) diff --git a/drivers/gofile/meta.go b/drivers/gofile/meta.go new file mode 100644 index 00000000000..00656025770 --- /dev/null +++ b/drivers/gofile/meta.go @@ -0,0 +1,28 @@ +package gofile + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APIToken string `json:"api_token" required:"true" help:"Get your API token from your Gofile profile page"` + LinkExpiry int `json:"link_expiry" type:"number" default:"30" help:"Direct link cache duration in days. Set to 0 to disable caching"` + DirectLinkExpiry int `json:"direct_link_expiry" type:"number" default:"0" help:"Direct link expiration time in hours on Gofile server. Set to 0 for no expiration"` +} + +var config = driver.Config{ + Name: "Gofile", + DefaultRoot: "", + LocalSort: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Gofile{} + }) +} diff --git a/drivers/gofile/types.go b/drivers/gofile/types.go new file mode 100644 index 00000000000..be307347081 --- /dev/null +++ b/drivers/gofile/types.go @@ -0,0 +1,124 @@ +package gofile + +import "time" + +type APIResponse struct { + Status string `json:"status"` + Data interface{} `json:"data"` +} + +type AccountResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + } `json:"data"` +} + +type AccountInfoResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Email string `json:"email"` + RootFolder string `json:"rootFolder"` + } `json:"data"` +} + +type Content struct { + ID string `json:"id"` + Type string `json:"type"` // "file" or "folder" + Name string `json:"name"` + Size int64 `json:"size,omitempty"` + CreateTime int64 `json:"createTime"` + ModTime int64 `json:"modTime,omitempty"` + DirectLink string `json:"directLink,omitempty"` + Children map[string]Content `json:"children,omitempty"` + ParentFolder string `json:"parentFolder,omitempty"` + MD5 string `json:"md5,omitempty"` + MimeType string `json:"mimeType,omitempty"` + Link string `json:"link,omitempty"` +} + +type ContentsResponse struct { + Status string `json:"status"` + Data struct { + IsOwner bool `json:"isOwner"` + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + ParentFolder string `json:"parentFolder"` + CreateTime int64 `json:"createTime"` + ChildrenList []string `json:"childrenList,omitempty"` + Children map[string]Content `json:"children,omitempty"` + Contents map[string]Content `json:"contents,omitempty"` + Public bool `json:"public,omitempty"` + Description string `json:"description,omitempty"` + Tags string `json:"tags,omitempty"` + Expiry int64 `json:"expiry,omitempty"` + } `json:"data"` +} + +type UploadResponse struct { + Status string `json:"status"` + Data struct { + DownloadPage string `json:"downloadPage"` + Code string `json:"code"` + ParentFolder string `json:"parentFolder"` + FileId string `json:"fileId"` + FileName string `json:"fileName"` + GuestToken string `json:"guestToken,omitempty"` + } `json:"data"` +} + +type DirectLinkResponse struct { + Status string `json:"status"` + Data struct { + DirectLink string `json:"directLink"` + ID string `json:"id"` + } `json:"data"` +} + +type CreateFolderResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + ParentFolder string `json:"parentFolder"` + CreateTime int64 `json:"createTime"` + } `json:"data"` +} + +type CopyResponse struct { + Status string `json:"status"` + Data struct { + CopiedContents map[string]string `json:"copiedContents"` // oldId -> newId mapping + } `json:"data"` +} + +type UpdateResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"data"` +} + +type ErrorResponse struct { + Status string `json:"status"` + Error struct { + Message string `json:"message"` + Code string `json:"code"` + } `json:"error"` +} + +func (c *Content) ModifiedTime() time.Time { + if c.ModTime > 0 { + return time.Unix(c.ModTime, 0) + } + return time.Unix(c.CreateTime, 0) +} + +func (c *Content) IsDir() bool { + return c.Type == "folder" +} diff --git a/drivers/gofile/util.go b/drivers/gofile/util.go new file mode 100644 index 00000000000..1dd6229a773 --- /dev/null +++ b/drivers/gofile/util.go @@ -0,0 +1,265 @@ +package gofile + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + log "github.com/sirupsen/logrus" +) + +const ( + baseAPI = "https://api.gofile.io" + uploadAPI = "https://upload.gofile.io" +) + +func (d *Gofile) request(ctx context.Context, method, endpoint string, body io.Reader, headers map[string]string) (*http.Response, error) { + var url string + if strings.HasPrefix(endpoint, "http") { + url = endpoint + } else { + url = baseAPI + endpoint + } + + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return nil, err + } + + req.Header.Set("Authorization", "Bearer "+d.APIToken) + req.Header.Set("User-Agent", "AList/3.0") + + for k, v := range headers { + req.Header.Set(k, v) + } + + return base.HttpClient.Do(req) +} + +func (d *Gofile) getJSON(ctx context.Context, endpoint string, result interface{}) error { + resp, err := d.request(ctx, "GET", endpoint, nil, nil) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + return json.NewDecoder(resp.Body).Decode(result) +} + +func (d *Gofile) postJSON(ctx context.Context, endpoint string, data interface{}, result interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "POST", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + if result != nil { + return json.NewDecoder(resp.Body).Decode(result) + } + + return nil +} + +func (d *Gofile) putJSON(ctx context.Context, endpoint string, data interface{}, result interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "PUT", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + if result != nil { + return json.NewDecoder(resp.Body).Decode(result) + } + + return nil +} + +func (d *Gofile) deleteJSON(ctx context.Context, endpoint string, data interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "DELETE", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + return nil +} + +func (d *Gofile) handleError(resp *http.Response) error { + body, _ := io.ReadAll(resp.Body) + log.Debugf("Gofile API error (HTTP %d): %s", resp.StatusCode, string(body)) + + var errorResp ErrorResponse + if err := json.Unmarshal(body, &errorResp); err == nil && errorResp.Status == "error" { + return fmt.Errorf("gofile API error: %s (code: %s)", errorResp.Error.Message, errorResp.Error.Code) + } + + return fmt.Errorf("gofile API error: HTTP %d - %s", resp.StatusCode, string(body)) +} + +func (d *Gofile) uploadFile(ctx context.Context, folderId string, file model.FileStreamer, up driver.UpdateProgress) (*UploadResponse, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + if folderId != "" { + writer.WriteField("folderId", folderId) + } + + part, err := writer.CreateFormFile("file", filepath.Base(file.GetName())) + if err != nil { + return nil, err + } + + // Copy with progress tracking if available + if up != nil { + reader := &progressReader{ + reader: file, + total: file.GetSize(), + up: up, + } + _, err = io.Copy(part, reader) + } else { + _, err = io.Copy(part, file) + } + + if err != nil { + return nil, err + } + + writer.Close() + + headers := map[string]string{ + "Content-Type": writer.FormDataContentType(), + } + + resp, err := d.request(ctx, "POST", uploadAPI+"/uploadfile", &body, headers) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, d.handleError(resp) + } + + var result UploadResponse + err = json.NewDecoder(resp.Body).Decode(&result) + return &result, err +} + +func (d *Gofile) createDirectLink(ctx context.Context, contentId string) (string, error) { + data := map[string]interface{}{} + + if d.DirectLinkExpiry > 0 { + expireTime := time.Now().Add(time.Duration(d.DirectLinkExpiry) * time.Hour).Unix() + data["expireTime"] = expireTime + } + + var result DirectLinkResponse + err := d.postJSON(ctx, fmt.Sprintf("/contents/%s/directlinks", contentId), data, &result) + if err != nil { + return "", err + } + + return result.Data.DirectLink, nil +} + +func (d *Gofile) convertContentToObj(content Content) model.Obj { + return &model.ObjThumb{ + Object: model.Object{ + ID: content.ID, + Name: content.Name, + Size: content.Size, + Modified: content.ModifiedTime(), + IsFolder: content.IsDir(), + }, + } +} + +func (d *Gofile) getAccountId(ctx context.Context) (string, error) { + var result AccountResponse + err := d.getJSON(ctx, "/accounts/getid", &result) + if err != nil { + return "", err + } + return result.Data.ID, nil +} + +func (d *Gofile) getAccountInfo(ctx context.Context, accountId string) (*AccountInfoResponse, error) { + var result AccountInfoResponse + err := d.getJSON(ctx, fmt.Sprintf("/accounts/%s", accountId), &result) + if err != nil { + return nil, err + } + return &result, nil +} + +// progressReader wraps an io.Reader to track upload progress +type progressReader struct { + reader io.Reader + total int64 + read int64 + up driver.UpdateProgress +} + +func (pr *progressReader) Read(p []byte) (n int, err error) { + n, err = pr.reader.Read(p) + pr.read += int64(n) + if pr.up != nil && pr.total > 0 { + progress := float64(pr.read) * 100 / float64(pr.total) + pr.up(progress) + } + return n, err +} diff --git a/drivers/guangyapan/driver.go b/drivers/guangyapan/driver.go new file mode 100644 index 00000000000..e18d1b795ca --- /dev/null +++ b/drivers/guangyapan/driver.go @@ -0,0 +1,1051 @@ +package guangyapan + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +const ( + accountBaseURL = "https://account.guangyapan.com" + apiBaseURL = "https://api.guangyapan.com" + defaultClient = "aMe-8VSlkrbQXpUR" +) + +type GuangYaPan struct { + model.Storage + Addition + + accountClient *resty.Client + apiClient *resty.Client + + resolvedRootFolderID string + rootFolderResolved bool +} + +func (d *GuangYaPan) Config() driver.Config { + return config +} + +func (d *GuangYaPan) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *GuangYaPan) Init(ctx context.Context) error { + d.ClientID = strings.TrimSpace(d.ClientID) + if d.ClientID == "" { + d.ClientID = defaultClient + } + d.DeviceID = normalizeDeviceID(d.DeviceID) + if d.DeviceID == "" { + d.DeviceID = randomDeviceID() + } + if d.PageSize <= 0 { + d.PageSize = 100 + } + if d.OrderBy < 0 { + d.OrderBy = 3 + } + if d.SortType != 0 && d.SortType != 1 { + d.SortType = 1 + } + + d.RootPath = strings.TrimSpace(d.RootPath) + d.AccessToken = strings.TrimSpace(d.AccessToken) + d.RefreshToken = strings.TrimSpace(d.RefreshToken) + d.PhoneNumber = strings.TrimSpace(d.PhoneNumber) + d.VerifyCode = strings.TrimSpace(d.VerifyCode) + d.CaptchaToken = strings.TrimSpace(d.CaptchaToken) + d.VerificationID = strings.TrimSpace(d.VerificationID) + d.resolvedRootFolderID = "" + d.rootFolderResolved = false + + d.accountClient = base.NewRestyClient(). + SetBaseURL(accountBaseURL). + SetHeader("Accept", "application/json, text/plain, */*"). + SetHeader("Content-Type", "application/json"). + SetHeader("X-Device-Model", "chrome%2F147.0.0.0"). + SetHeader("X-Device-Name", "PC-Chrome"). + SetHeader("X-Device-Sign", "wdi10."+d.DeviceID+"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"). + SetHeader("X-Net-Work-Type", "NONE"). + SetHeader("X-OS-Version", "MacIntel"). + SetHeader("X-Platform-Version", "1"). + SetHeader("X-Protocol-Version", "301"). + SetHeader("X-Provider-Name", "NONE"). + SetHeader("X-SDK-Version", "9.0.2"). + SetHeader("X-Client-Id", d.ClientID). + SetHeader("X-Client-Version", "0.0.1"). + SetHeader("X-Device-Id", d.DeviceID) + if d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + } + + d.apiClient = base.NewRestyClient(). + SetBaseURL(apiBaseURL). + SetHeader("Accept", "application/json, text/plain, */*"). + SetHeader("Content-Type", "application/json"). + SetHeader("Did", d.DeviceID). + SetHeader("Dt", "4") + + // Priority: access_token -> refresh_token -> sms login. + if d.AccessToken != "" { + if err := d.validateToken(ctx); err == nil { + return d.prepareRootFolder(ctx) + } + d.AccessToken = "" + } + if d.RefreshToken != "" { + if err := d.refreshToken(ctx); err == nil { + if err2 := d.validateToken(ctx); err2 == nil { + return d.prepareRootFolder(ctx) + } + } + } + // Two-stage SMS flow: + // 1) phone only + send_code=true: send code and cache verification_id (do not fail init). + // 2) phone + verify_code: complete login and save tokens. + if d.PhoneNumber != "" { + if d.canSMSLogin() { + if err := d.loginBySMSCode(ctx); err != nil { + return err + } + if err := d.validateToken(ctx); err != nil { + return err + } + return d.prepareRootFolder(ctx) + } + if d.SendCode { + d.setTempStatus("SMS sending in progress...") + if err := d.prepareSMSCode(ctx); err != nil { + d.setTempStatus(fmt.Sprintf("SMS send failed: %v. Please check captcha/meta and set send_code=true to retry.", err)) + log.Warnf("guangyapan: prepare sms code failed: %v", err) + } else { + d.setTempStatus("SMS sent successfully. Please fill verify_code and save to complete login.") + } + } + return nil + } + return errors.New("login failed: provide a valid access_token, or refresh_token, or phone_number + verify_code + captcha_token") +} + +func (d *GuangYaPan) Drop(ctx context.Context) error { + return nil +} + +func (d *GuangYaPan) GetRoot(ctx context.Context) (model.Obj, error) { + rootID, err := d.getRootFolderID(ctx) + if err != nil { + return nil, err + } + return &model.Object{ + ID: rootID, + Path: "/", + Name: "root", + Size: 0, + Modified: d.Modified, + IsFolder: true, + }, nil +} + +func (d *GuangYaPan) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + parentID := dir.GetID() + + res := make([]model.Obj, 0, d.PageSize) + for page := 0; ; page++ { + var resp listResp + body := map[string]any{ + "parentId": parentID, + "page": page, + "pageSize": d.PageSize, + "orderBy": d.OrderBy, + "sortType": d.SortType, + "fileTypes": []int{}, + } + if err := d.postAPI(ctx, "/userres/v1/file/get_file_list", body, &resp); err != nil { + return nil, err + } + for _, item := range resp.Data.List { + res = append(res, &model.Object{ + ID: item.FileID, + Path: parentID, + Name: item.FileName, + Size: item.FileSize, + Modified: unixOrZero(item.UTime), + Ctime: unixOrZero(item.CTime), + IsFolder: item.ResType == 2, + }) + } + if len(resp.Data.List) < d.PageSize { + break + } + if resp.Data.Total > 0 && len(res) >= resp.Data.Total { + break + } + } + return res, nil +} + +func (d *GuangYaPan) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + var resp downloadResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_download_url", map[string]any{ + "fileId": file.GetID(), + }, &resp); err != nil { + return nil, err + } + + url := strings.TrimSpace(resp.Data.SignedURL) + if url == "" { + url = strings.TrimSpace(resp.Data.DownloadURL) + } + if url == "" { + return nil, errors.New("empty download url") + } + return &model.Link{URL: url}, nil +} + +func (d *GuangYaPan) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + name := strings.TrimSpace(dirName) + if name == "" { + return errors.New("dir name is empty") + } + + parentID := parentDir.GetID() + + var out createDirResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/create_dir", map[string]any{ + "parentId": parentID, + "dirName": name, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("make dir failed: %s", strings.TrimSpace(out.Msg)) + } + return nil +} + +func (d *GuangYaPan) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + name := strings.TrimSpace(newName) + if name == "" { + return errors.New("new name is empty") + } + + var out commonResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/rename", map[string]any{ + "fileId": fileID, + "newName": name, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("rename failed: %s", strings.TrimSpace(out.Msg)) + } + return nil +} + +func (d *GuangYaPan) Remove(ctx context.Context, obj model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(obj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + + var del deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/delete_file", map[string]any{ + "fileIds": []string{fileID}, + }, &del); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(del.Msg), "success") { + return fmt.Errorf("delete failed: %s", strings.TrimSpace(del.Msg)) + } + + taskID := strings.TrimSpace(del.Data.TaskID) + if taskID == "" { + // Some backends may apply deletion synchronously. + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + parentID := dstDir.GetID() + + var out deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/move_file", map[string]any{ + "fileIds": []string{fileID}, + "parentId": parentID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("move failed: %s", strings.TrimSpace(out.Msg)) + } + taskID := strings.TrimSpace(out.Data.TaskID) + if taskID == "" { + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + parentID := dstDir.GetID() + + var out deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/copy_file", map[string]any{ + "fileIds": []string{fileID}, + "parentId": parentID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("copy failed: %s", strings.TrimSpace(out.Msg)) + } + taskID := strings.TrimSpace(out.Data.TaskID) + if taskID == "" { + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + if file == nil { + return errors.New("file is nil") + } + if file.GetSize() < 0 { + return errors.New("invalid file size") + } + name := strings.TrimSpace(file.GetName()) + if name == "" { + return errors.New("file name is empty") + } + + parentID := dstDir.GetID() + + token, code, err := d.getUploadToken(ctx, parentID, name, file.GetSize()) + if err != nil { + return err + } + taskID := strings.TrimSpace(token.TaskID) + if code == 156 { + if taskID == "" { + return errors.New("instant upload returns empty task id") + } + return d.waitUploadTaskInfo(ctx, taskID) + } + + if token.ObjectPath == "" || token.BucketName == "" || token.EndPoint == "" || token.AccessKeyID == "" || token.SecretAccessKey == "" { + return errors.New("upload token is incomplete") + } + + ossEndpoint := normalizeOSSEndpoint(token.EndPoint, token.BucketName) + client, err := oss.New(ossEndpoint, token.AccessKeyID, token.SecretAccessKey, oss.SecurityToken(token.SessionToken)) + if err != nil { + return fmt.Errorf("create oss client failed: %w", err) + } + bucket, err := client.Bucket(token.BucketName) + if err != nil { + return fmt.Errorf("create oss bucket failed: %w", err) + } + + if file.GetSize() == 0 { + if err := bucket.PutObject(token.ObjectPath, strings.NewReader("")); err != nil { + return err + } + } else { + if err := d.multipartUploadToOSS(ctx, bucket, token.ObjectPath, file, up); err != nil { + return err + } + } + + if taskID == "" { + return nil + } + return d.waitUploadTaskInfo(ctx, taskID) +} + +func (d *GuangYaPan) getRootFolderID(ctx context.Context) (string, error) { + if d.rootFolderResolved { + return d.resolvedRootFolderID, nil + } + if err := d.ensureAccessToken(ctx); err != nil { + return "", err + } + if err := d.prepareRootFolder(ctx); err != nil { + return "", err + } + return d.resolvedRootFolderID, nil +} + +func (d *GuangYaPan) prepareRootFolder(ctx context.Context) error { + rootID, err := d.resolveConfiguredRootFolderID(ctx) + if err != nil { + return err + } + d.resolvedRootFolderID = rootID + d.rootFolderResolved = true + return nil +} + +func (d *GuangYaPan) resolveConfiguredRootFolderID(ctx context.Context) (string, error) { + root := strings.TrimSpace(d.RootPath) + if root == "" { + return "", nil + } + return d.resolveFolderPath(ctx, root) +} + +func (d *GuangYaPan) resolveFolderPath(ctx context.Context, rootPath string) (string, error) { + cleanPath := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rootPath), "\\", "/"), "/") + if cleanPath == "" { + return "", nil + } + + parentID := "" + for _, name := range strings.Split(cleanPath, "/") { + if name == "" { + continue + } + childID, err := d.findChildFolderID(ctx, parentID, name) + if err != nil { + return "", err + } + parentID = childID + } + return parentID, nil +} + +func (d *GuangYaPan) findChildFolderID(ctx context.Context, parentID, name string) (string, error) { + pageSize := d.PageSize + if pageSize <= 0 { + pageSize = 100 + } + + seen := 0 + for page := 0; ; page++ { + var resp listResp + body := map[string]any{ + "parentId": parentID, + "page": page, + "pageSize": pageSize, + "orderBy": d.OrderBy, + "sortType": d.SortType, + "fileTypes": []int{}, + } + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_file_list", body, &resp); err != nil { + return "", err + } + for _, item := range resp.Data.List { + seen++ + if item.ResType == 2 && item.FileName == name { + return item.FileID, nil + } + } + if len(resp.Data.List) < pageSize { + break + } + if resp.Data.Total > 0 && seen >= resp.Data.Total { + break + } + } + + if parentID == "" { + return "", fmt.Errorf("resolve root folder path failed: folder %q not found under /", name) + } + return "", fmt.Errorf("resolve root folder path failed: folder %q not found under parent %s", name, parentID) +} + +func (d *GuangYaPan) ensureAccessToken(ctx context.Context) error { + if strings.TrimSpace(d.AccessToken) != "" { + return nil + } + if strings.TrimSpace(d.RefreshToken) == "" { + if d.canSMSLogin() { + return d.loginBySMSCode(ctx) + } + if d.PhoneNumber != "" { + return errors.New("not logged in yet: please fill verify_code and save storage to finish SMS login") + } + return errors.New("access token is empty") + } + return d.refreshToken(ctx) +} + +func (d *GuangYaPan) validateToken(ctx context.Context) error { + var me userMeResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetResult(&me). + Get("/v1/user/me") + if err != nil { + return err + } + if resp.IsError() { + return fmt.Errorf("validate token failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + if strings.TrimSpace(me.Sub) == "" { + return errors.New("validate token failed: empty user sub") + } + return nil +} + +func (d *GuangYaPan) refreshToken(ctx context.Context) error { + if strings.TrimSpace(d.RefreshToken) == "" { + return errors.New("refresh_token is empty") + } + + var out tokenResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "client_id": d.ClientID, + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + }). + SetResult(&out). + Post("/v1/auth/token") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" { + errMsg := strings.TrimSpace(out.ErrorDesc) + if errMsg == "" { + errMsg = strings.TrimSpace(out.Error) + } + if errMsg == "" { + errMsg = strings.TrimSpace(resp.String()) + } + if errMsg == "" { + errMsg = fmt.Sprintf("status=%d", resp.StatusCode()) + } + return fmt.Errorf("refresh token failed: %s", errMsg) + } + + d.AccessToken = strings.TrimSpace(out.AccessToken) + if strings.TrimSpace(out.RefreshToken) != "" { + d.RefreshToken = strings.TrimSpace(out.RefreshToken) + } + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) canSMSLogin() bool { + return d.PhoneNumber != "" && d.VerifyCode != "" +} + +func (d *GuangYaPan) loginBySMSCode(ctx context.Context) error { + verificationID := strings.TrimSpace(d.VerificationID) + if verificationID == "" { + var err error + verificationID, err = d.requestVerificationID(ctx) + if err != nil { + return err + } + } + + var step2 verifyResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "verification_id": verificationID, + "verification_code": d.VerifyCode, + "client_id": d.ClientID, + }). + SetResult(&step2). + Post("/v1/auth/verification/verify") + if err != nil { + return err + } + if resp.IsError() || step2.Error != "" || strings.TrimSpace(step2.VerificationToken) == "" { + return fmt.Errorf("verify code failed: %s", d.accountErr(step2.ErrorDesc, step2.Error, resp)) + } + + var out tokenResp + resp, err = d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "verification_code": d.VerifyCode, + "verification_token": step2.VerificationToken, + "username": normalizePhoneE164(d.PhoneNumber), + "client_id": d.ClientID, + }). + SetResult(&out). + Post("/v1/auth/signin") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" { + return fmt.Errorf("signin failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp)) + } + + d.AccessToken = strings.TrimSpace(out.AccessToken) + d.RefreshToken = strings.TrimSpace(out.RefreshToken) + d.VerificationID = "" + // One-time SMS code should not be reused after successful login. + d.VerifyCode = "" + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) prepareSMSCode(ctx context.Context) error { + // Explicit send action should always refresh verification_id. + d.VerificationID = "" + if err := d.ensureCaptchaToken(ctx, false); err != nil { + return err + } + verificationID, err := d.requestVerificationID(ctx) + if err != nil { + return err + } + d.VerificationID = verificationID + d.SendCode = false + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) requestVerificationID(ctx context.Context) (string, error) { + if d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + } + + var step1 verificationResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "phone_number": normalizePhoneE164(d.PhoneNumber), + "target": "ANY", + "client_id": d.ClientID, + }). + SetResult(&step1). + Post("/v1/auth/verification") + if err != nil { + return "", err + } + if resp.IsError() || step1.Error != "" || strings.TrimSpace(step1.VerificationID) == "" { + // If captcha token is expired/invalid, refresh it once and retry. + if strings.Contains(step1.Error, "captcha_invalid") || strings.Contains(step1.ErrorDesc, "captcha_token expired") { + if err := d.ensureCaptchaToken(ctx, true); err == nil { + return d.requestVerificationID(ctx) + } + } + return "", fmt.Errorf("request verification failed: %s", d.accountErr(step1.ErrorDesc, step1.Error, resp)) + } + return strings.TrimSpace(step1.VerificationID), nil +} + +func (d *GuangYaPan) ensureCaptchaToken(ctx context.Context, force bool) error { + if !force && d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + return nil + } + + var out captchaInitResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "client_id": d.ClientID, + "action": "POST:/v1/auth/verification", + "device_id": d.DeviceID, + "meta": map[string]any{ + "username": normalizePhoneE164(d.PhoneNumber), + "phone_number": normalizePhoneE164(d.PhoneNumber), + "VERIFICATION_PHONE": normalizePhoneE164(d.PhoneNumber), + }, + }). + SetResult(&out). + Post("/v1/shield/captcha/init") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.CaptchaToken) == "" { + return fmt.Errorf("init captcha token failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp)) + } + d.CaptchaToken = strings.TrimSpace(out.CaptchaToken) + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + op.MustSaveDriverStorage(d) + return nil +} + +func normalizeCaptchaUsername(phone string) string { + p := strings.TrimSpace(phone) + p = strings.ReplaceAll(p, " ", "") + p = strings.TrimPrefix(p, "+") + // Keep only digits. + b := make([]rune, 0, len(p)) + for _, ch := range p { + if ch >= '0' && ch <= '9' { + b = append(b, ch) + } + } + digits := string(b) + // Mainland number normalization: +86xxxxxxxxxxx -> xxxxxxxxxxx + if strings.HasPrefix(digits, "86") && len(digits) > 11 { + digits = digits[2:] + } + return digits +} + +func normalizePhoneE164(phone string) string { + p := strings.TrimSpace(phone) + if p == "" { + return "" + } + p = strings.ReplaceAll(p, " ", "") + if strings.HasPrefix(p, "+") { + // Format as "+86 1xxxxxxxxxx" to match browser payload expectations. + if strings.HasPrefix(p, "+86") && len(p) > 3 { + rest := strings.TrimPrefix(p, "+86") + return "+86 " + rest + } + return p + } + // If raw mainland number is provided, normalize with +86 prefix. + digits := normalizeCaptchaUsername(p) + if len(digits) == 11 { + return "+86 " + digits + } + return p +} + +func (d *GuangYaPan) setTempStatus(status string) { + // initStorage sets status to WORK after Init returns, so we update it shortly after. + time.AfterFunc(200*time.Millisecond, func() { + d.GetStorage().SetStatus(status) + op.MustSaveDriverStorage(d) + }) +} + +func (d *GuangYaPan) accountErr(desc, short string, resp *resty.Response) string { + msg := strings.TrimSpace(desc) + if msg == "" { + msg = strings.TrimSpace(short) + } + if msg == "" && resp != nil { + msg = strings.TrimSpace(resp.String()) + } + if msg == "" && resp != nil { + msg = fmt.Sprintf("status=%d", resp.StatusCode()) + } + if msg == "" { + msg = "unknown error" + } + return msg +} + +func (d *GuangYaPan) postAPI(ctx context.Context, path string, body any, out any) error { + if strings.TrimSpace(d.AccessToken) == "" { + return errors.New("access token is empty") + } + resp, err := d.apiClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetBody(body). + SetResult(out). + Post(path) + if err != nil { + return err + } + if resp.StatusCode() == 401 || resp.StatusCode() == 403 { + if strings.TrimSpace(d.RefreshToken) == "" { + return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + if err := d.refreshToken(ctx); err != nil { + return err + } + resp, err = d.apiClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetBody(body). + SetResult(out). + Post(path) + if err != nil { + return err + } + } + if resp.IsError() { + return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + return nil +} + +func (d *GuangYaPan) waitTaskDone(ctx context.Context, taskID string) error { + const ( + maxTry = 30 + interval = 300 * time.Millisecond + ) + for i := 0; i < maxTry; i++ { + var out taskStatusResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_task_status", map[string]any{ + "taskId": taskID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("get task status failed: %s", strings.TrimSpace(out.Msg)) + } + switch out.Data.Status { + case 2: + return nil + case -1, 3: + return fmt.Errorf("task %s failed with status=%d", taskID, out.Data.Status) + } + if i == maxTry-1 { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } + return fmt.Errorf("task %s timeout", taskID) +} + +func (d *GuangYaPan) getUploadToken(ctx context.Context, parentID, name string, size int64) (*uploadTokenData, int, error) { + var out uploadTokenResp + err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_center_token", map[string]any{ + "capacity": 2, + "name": name, + "parentId": parentID, + "res": map[string]any{ + "fileSize": size, + }, + }, &out) + if err != nil { + return nil, 0, err + } + msg := strings.TrimSpace(out.Msg) + if msg != "" && !strings.EqualFold(msg, "success") { + return nil, out.Code, fmt.Errorf("get upload token failed: %s", msg) + } + if out.Data.TaskID == "" { + return nil, out.Code, errors.New("get upload token failed: empty task id") + } + if out.Data.AccessKeyID == "" { + out.Data.AccessKeyID = out.Data.Creds.AccessKeyID + } + if out.Data.SecretAccessKey == "" { + out.Data.SecretAccessKey = out.Data.Creds.SecretAccessKey + } + if out.Data.SessionToken == "" { + out.Data.SessionToken = out.Data.Creds.SessionToken + } + if strings.TrimSpace(out.Data.EndPoint) == "" { + out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint) + } + if strings.TrimSpace(out.Data.EndPoint) != "" && !strings.HasPrefix(out.Data.EndPoint, "http://") && !strings.HasPrefix(out.Data.EndPoint, "https://") { + if strings.TrimSpace(out.Data.FullEndPoint) != "" { + out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint) + } else if strings.TrimSpace(out.Data.BucketName) != "" { + host := strings.TrimSpace(out.Data.EndPoint) + prefix := strings.TrimSpace(out.Data.BucketName) + "." + if strings.HasPrefix(host, prefix) { + out.Data.EndPoint = "https://" + host + } else { + out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.BucketName) + "." + host + } + } else { + out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.EndPoint) + } + } + return &out.Data, out.Code, nil +} + +func (d *GuangYaPan) waitUploadTaskInfo(ctx context.Context, taskID string) error { + const ( + maxTry = 300 + interval = 1 * time.Second + ) + for i := 0; i < maxTry; i++ { + var out taskInfoResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_info_by_task_id", map[string]any{ + "taskId": taskID, + }, &out); err != nil { + return err + } + if out.Data.FileID != "" { + return nil + } + switch out.Code { + case 145, 146, 147, 155, 163, 0: + // uploading/verifying/processing + default: + if strings.TrimSpace(out.Msg) != "" { + return fmt.Errorf("upload task failed: code=%d msg=%s", out.Code, strings.TrimSpace(out.Msg)) + } + } + if i == maxTry-1 { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } + return fmt.Errorf("upload task %s timeout", taskID) +} + +func (d *GuangYaPan) multipartUploadToOSS(ctx context.Context, bucket *oss.Bucket, objectPath string, file model.FileStreamer, up driver.UpdateProgress) error { + partSize := calcUploadPartSize(file.GetSize()) + imur, err := bucket.InitiateMultipartUpload(objectPath, oss.Sequential()) + if err != nil { + return err + } + + total := file.GetSize() + partCount := int((total + partSize - 1) / partSize) + parts := make([]oss.UploadPart, 0, partCount) + var uploaded int64 + partNumber := 1 + + for uploaded < total { + if err := ctx.Err(); err != nil { + return err + } + curPartSize := partSize + left := total - uploaded + if left < curPartSize { + curPartSize = left + } + + reader := io.LimitReader(file, curPartSize) + part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, reader), curPartSize, partNumber) + if err != nil { + return err + } + parts = append(parts, part) + uploaded += curPartSize + partNumber++ + if total > 0 { + up(100 * float64(uploaded) / float64(total)) + } + } + + _, err = bucket.CompleteMultipartUpload(imur, parts) + return err +} + +func calcUploadPartSize(size int64) int64 { + const ( + mb = int64(1024 * 1024) + gb = int64(1024 * 1024 * 1024) + ) + switch { + case size <= 100*mb: + return 1 * mb + case size <= 16*gb: + return 2 * mb + case size <= 160*gb: + return 4 * mb + default: + return 8 * mb + } +} + +func normalizeOSSEndpoint(endpoint, bucket string) string { + ep := strings.TrimSpace(endpoint) + if ep == "" { + return ep + } + if !strings.HasPrefix(ep, "http://") && !strings.HasPrefix(ep, "https://") { + ep = "https://" + ep + } + u, err := url.Parse(ep) + if err != nil || u.Host == "" { + return ep + } + host := u.Host + prefix := strings.TrimSpace(bucket) + if prefix != "" && strings.HasPrefix(host, prefix+".") { + host = strings.TrimPrefix(host, prefix+".") + } + u.Host = host + return u.String() +} + +func normalizeDeviceID(v string) string { + v = strings.ToLower(strings.TrimSpace(v)) + v = strings.ReplaceAll(v, "-", "") + if len(v) != 32 { + return "" + } + for _, ch := range v { + if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { + return "" + } + } + return v +} + +func randomDeviceID() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "0123456789abcdef0123456789abcdef" + } + return hex.EncodeToString(b) +} + +var _ driver.Driver = (*GuangYaPan)(nil) +var _ driver.GetRooter = (*GuangYaPan)(nil) \ No newline at end of file diff --git a/drivers/guangyapan/meta.go b/drivers/guangyapan/meta.go new file mode 100644 index 00000000000..7d5f8fbcb51 --- /dev/null +++ b/drivers/guangyapan/meta.go @@ -0,0 +1,42 @@ +package guangyapan + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + RootPath string `json:"root_path" help:"光鸭云盘中的完整路径"` + PhoneNumber string `json:"phone_number" type:"text" help:"Phone number for SMS login, e.g. +86 13800000000"` + CaptchaToken string `json:"captcha_token" type:"text" help:"Captcha token required by /v1/auth/verification"` + SendCode bool `json:"send_code" type:"bool" help:"Set true and save to send SMS code, it auto-resets to false after sending"` + VerifyCode string `json:"verify_code" type:"text" help:"SMS verification code used with phone_number; fill then save to finish login"` + VerificationID string `json:"verification_id" type:"text" help:"Auto-generated after sending SMS code; do not edit manually"` + AccessToken string `json:"access_token" type:"text" help:"Bearer access token (optional if refresh_token is provided)"` + RefreshToken string `json:"refresh_token" type:"text" help:"Refresh token for auto-login/auto-refresh"` + ClientID string `json:"client_id" default:"aMe-8VSlkrbQXpUR"` + DeviceID string `json:"device_id" help:"Optional custom device id (32 hex chars), auto-generated when empty"` + PageSize int `json:"page_size" type:"number" default:"100"` + OrderBy int `json:"order_by" type:"number" options:"0,1,2,3,4" default:"3" help:"Sort field used by the file list"` + SortType int `json:"sort_type" type:"number" options:"0,1" default:"1" help:"Sort direction used by the file list"` +} + +var config = driver.Config{ + Name: "GuangYaPan", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: true, + Alert: "info|Two-stage SMS login: (1) fill phone_number (+ captcha_token if needed), set send_code=true and save; (2) fill verify_code and save to finish login and auto-save access_token/refresh_token.", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &GuangYaPan{} + }) +} \ No newline at end of file diff --git a/drivers/guangyapan/offline.go b/drivers/guangyapan/offline.go new file mode 100644 index 00000000000..a53b8adc33f --- /dev/null +++ b/drivers/guangyapan/offline.go @@ -0,0 +1,169 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/model" +) + +func (d *GuangYaPan) ResolveOfflineResource(ctx context.Context, fileURL string) (*OfflineResolveData, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + fileURL = strings.TrimSpace(fileURL) + if fileURL == "" { + return nil, errors.New("offline url is empty") + } + + var resp offlineResolveResp + if err := d.postAPI(ctx, "/cloudcollection/v1/resolve_res", map[string]any{ + "url": fileURL, + }, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("resolve offline resource failed: %s", strings.TrimSpace(resp.Msg)) + } + return &resp.Data, nil +} + +func (d *GuangYaPan) OfflineDownload(ctx context.Context, fileURL string, parentDir model.Obj, fileName string) (*OfflineTask, error) { + resolved, err := d.ResolveOfflineResource(ctx, fileURL) + if err != nil { + return nil, err + } + + parentID := parentDir.GetID() + rootID, err := d.getRootFolderID(ctx) + if err != nil { + return nil, err + } + if parentID == rootID { + parentID = "" + } + + taskURL := strings.TrimSpace(resolved.URL) + if taskURL == "" { + taskURL = strings.TrimSpace(fileURL) + } + name := strings.TrimSpace(fileName) + if name == "" { + name = resolved.defaultName(taskURL) + } + + body := map[string]any{ + "url": taskURL, + "parentId": parentID, + "newName": name, + } + if indexes := resolved.fileIndexes(); len(indexes) > 0 { + body["fileIndexes"] = indexes + } + + var resp offlineCreateResp + if err := d.postAPI(ctx, "/cloudcollection/v1/create_task", body, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("create offline task failed: %s", strings.TrimSpace(resp.Msg)) + } + taskID := strings.TrimSpace(resp.Data.TaskID) + if taskID == "" { + return nil, errors.New("create offline task failed: empty task id") + } + return &OfflineTask{ + TaskID: taskID, + FileName: name, + Res: taskURL, + }, nil +} + +func (d *GuangYaPan) OfflineList(ctx context.Context, taskIDs []string, statuses []int, cursor string, pageSize int) ([]OfflineTask, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + body := map[string]any{} + if len(taskIDs) > 0 { + body["taskIds"] = taskIDs + } + if len(statuses) > 0 { + body["status"] = statuses + } + if cursor = strings.TrimSpace(cursor); cursor != "" { + body["cursor"] = cursor + } + if pageSize > 0 { + body["pageSize"] = pageSize + } + + var resp offlineListResp + if err := d.postAPI(ctx, "/cloudcollection/v1/list_task", body, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("list offline tasks failed: %s", strings.TrimSpace(resp.Msg)) + } + return resp.Data.List, nil +} + +func (d *GuangYaPan) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + if len(taskIDs) == 0 { + return nil + } + + var resp offlineDeleteResp + if err := d.postAPI(ctx, "/cloudcollection/v2/delete_task", map[string]any{ + "taskIds": taskIDs, + }, &resp); err != nil { + return err + } + if !isSuccessMsg(resp.Msg) { + return fmt.Errorf("delete offline tasks failed: %s", strings.TrimSpace(resp.Msg)) + } + return nil +} + +func (d OfflineResolveData) defaultName(fileURL string) string { + if d.BTResInfo != nil && strings.TrimSpace(d.BTResInfo.FileName) != "" { + return strings.TrimSpace(d.BTResInfo.FileName) + } + u, err := url.Parse(fileURL) + if err == nil { + name := strings.TrimSpace(stdpath.Base(u.Path)) + if name != "" && name != "." && name != "/" { + if decoded, err := url.PathUnescape(name); err == nil { + name = decoded + } + return name + } + } + return "offline_download" +} + +func (d OfflineResolveData) fileIndexes() []int { + if d.BTResInfo == nil || len(d.BTResInfo.Subfiles) == 0 { + return nil + } + indexes := make([]int, 0, len(d.BTResInfo.Subfiles)) + for i, file := range d.BTResInfo.Subfiles { + if file.FileIndex != nil { + indexes = append(indexes, *file.FileIndex) + continue + } + indexes = append(indexes, i) + } + return indexes +} + +func isSuccessMsg(msg string) bool { + msg = strings.TrimSpace(msg) + return msg == "" || strings.EqualFold(msg, "success") +} diff --git a/drivers/guangyapan/types.go b/drivers/guangyapan/types.go new file mode 100644 index 00000000000..1a1c129c06a --- /dev/null +++ b/drivers/guangyapan/types.go @@ -0,0 +1,214 @@ +package guangyapan + +import "time" + +type tokenResp struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + TokenType string `json:"token_type"` + ExpiresIn int64 `json:"expires_in"` + Sub string `json:"sub"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type verificationResp struct { + VerificationID string `json:"verification_id"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type captchaInitResp struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type verifyResp struct { + VerificationToken string `json:"verification_token"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type userMeResp struct { + Sub string `json:"sub"` +} + +type listResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Total int `json:"total"` + List []fileItem `json:"list"` + } `json:"data"` +} + +type fileItem struct { + FileID string `json:"fileId"` + ParentID string `json:"parentId"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + ResType int `json:"resType"` + CTime int64 `json:"ctime"` + UTime int64 `json:"utime"` +} + +type downloadResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + SignedURL string `json:"signedURL"` + DownloadURL string `json:"downloadUrl"` + } `json:"data"` +} + +type createDirResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileID string `json:"fileId"` + FileName string `json:"fileName"` + ResType int `json:"resType"` + CTime int64 `json:"ctime"` + UTime int64 `json:"utime"` + } `json:"data"` +} + +type commonResp struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +type deleteResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID string `json:"taskId"` + } `json:"data"` +} + +type taskStatusResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Status int `json:"status"` + } `json:"data"` +} + +type uploadTokenResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data uploadTokenData `json:"data"` +} + +type uploadTokenData struct { + TaskID string `json:"taskId"` + ObjectPath string `json:"objectPath"` + Provider any `json:"provider"` + Region string `json:"region"` + BucketName string `json:"bucketName"` + EndPoint string `json:"endPoint"` + FullEndPoint string `json:"fullEndPoint"` + CallbackVar string `json:"callbackVar"` + AccessKeyID string `json:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` + Creds struct { + AccessKeyID string `json:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` + } `json:"creds"` +} + +type taskInfoResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileID string `json:"fileId"` + } `json:"data"` +} + +type offlineResolveResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data OfflineResolveData `json:"data"` +} + +type OfflineResolveData struct { + ResType int `json:"resType"` + BTResInfo *OfflineBTResInfo `json:"btResInfo"` + URL string `json:"url"` +} + +type OfflineBTResInfo struct { + InfoHash string `json:"infoHash"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + SubfilesNum int `json:"subfilesNum"` + Subfiles []OfflineSubfile `json:"subfiles"` + CreateTime int64 `json:"createTime"` + ExcludeIndices []int `json:"excludeIndices"` +} + +type OfflineSubfile struct { + FileName string `json:"fileName"` + FileIndex *int `json:"fileIndex"` + FileSize int64 `json:"fileSize"` +} + +type offlineCreateResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID string `json:"taskId"` + URL string `json:"url"` + } `json:"data"` +} + +type offlineDeleteResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskIDs []string `json:"taskIds"` + } `json:"data"` +} + +type offlineListResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + StatusCounts []struct { + Status int `json:"status"` + Count int `json:"count"` + } `json:"statusCounts"` + Cursor string `json:"cursor"` + List []OfflineTask `json:"list"` + Total int `json:"total"` + } `json:"data"` +} + +type OfflineTask struct { + TaskID string `json:"taskId"` + FileName string `json:"fileName"` + TotalSize int64 `json:"totalSize"` + Status int `json:"status"` + CreateTime int64 `json:"createTime"` + Res string `json:"res"` + ResType int `json:"resType"` + Progress int `json:"progress"` + FileID string `json:"fileId"` + IsDir bool `json:"isDir"` + Exist bool `json:"exist"` +} + +func unixOrZero(v int64) time.Time { + if v <= 0 { + return time.Time{} + } + return time.Unix(v, 0) +} diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index c3f5c6bb5bc..b3d69006791 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -94,6 +94,7 @@ func RemoveJSComment(data string) string { } if inComment && v == '*' && i+1 < len(data) && data[i+1] == '/' { inComment = false + i++ continue } if v == '/' && i+1 < len(data) { @@ -108,6 +109,9 @@ func RemoveJSComment(data string) string { continue } } + if inComment || inSingleLineComment { + continue + } result.WriteByte(v) } diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index e66252bcc79..be53963c157 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -430,17 +430,35 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( file.Time = timeFindReg.FindString(sharePageData) // 重定向获取真实链接 - res, err := base.NoRedirectClient.R().SetHeaders(map[string]string{ + headers := map[string]string{ "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6", - }).Get(downloadUrl) + } + res, err := base.NoRedirectClient.R().SetHeaders(headers).Get(downloadUrl) if err != nil { return nil, err } + rPageData := res.String() + if findAcwScV2Reg.MatchString(rPageData) { + log.Debug("lanzou: detected acw_sc__v2 challenge, recalculating cookie") + acwScV2, err := CalcAcwScV2(rPageData) + if err != nil { + return nil, err + } + // retry with calculated cookie to bypass anti-crawler validation + res, err = base.NoRedirectClient.R(). + SetHeaders(headers). + SetCookie(&http.Cookie{Name: "acw_sc__v2", Value: acwScV2}). + Get(downloadUrl) + if err != nil { + return nil, err + } + rPageData = res.String() + } + file.Url = res.Header().Get("location") // 触发验证 - rPageData := res.String() if res.StatusCode() != 302 { param, err = htmlJsonToMap(rPageData) if err != nil { diff --git a/drivers/lark/driver.go b/drivers/lark/driver.go index fbf7529afe3..ca704a2dbf3 100644 --- a/drivers/lark/driver.go +++ b/drivers/lark/driver.go @@ -8,14 +8,18 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" lark "github.com/larksuite/oapi-sdk-go/v3" larkcore "github.com/larksuite/oapi-sdk-go/v3/core" larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" + larkext "github.com/larksuite/oapi-sdk-go/v3/service/ext" + log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -25,8 +29,12 @@ type Lark struct { client *lark.Client rootFolderToken string + tokenMu sync.Mutex } +const larkListPageSize = 200 +const larkTokenRefreshSkew = 5 * time.Minute + func (c *Lark) Config() driver.Config { return config } @@ -41,34 +49,28 @@ func (c *Lark) Init(ctx context.Context) error { paths := strings.Split(c.RootFolderPath, "/") token := "" - var ok bool - var file *larkdrive.File for _, p := range paths { if p == "" { token = "" continue } - resp, err := c.client.Drive.File.ListByIterator(ctx, larkdrive.NewListFileReqBuilder().FolderToken(token).Build()) + files, err := c.listFiles(ctx, token) if err != nil { return err } - for { - ok, file, err = resp.Next() - if !ok { - return errs.ObjectNotFound - } - - if err != nil { - return err - } - + found := false + for _, file := range files { if *file.Type == "folder" && *file.Name == p { token = *file.Token + found = true break } } + if !found { + return errs.ObjectNotFound + } } c.rootFolderToken = token @@ -90,41 +92,262 @@ func (c *Lark) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] return nil, nil } - resp, err := c.client.Drive.File.ListByIterator(ctx, larkdrive.NewListFileReqBuilder().FolderToken(token).Build()) + files, err := c.listFiles(ctx, token) if err != nil { return nil, err } - ok = false - var file *larkdrive.File var res []model.Obj + for _, file := range files { + res = append(res, larkFileToObj(c.RootFolderPath, dir.GetPath(), file)) + } + + return res, nil +} + +func larkFileToObj(rootFolderPath, dirPath string, file *larkdrive.File) model.Obj { + name := larkString(file.Name) + fileType := larkString(file.Type) + modifiedUnix, _ := strconv.ParseInt(larkString(file.ModifiedTime), 10, 64) + createdUnix, _ := strconv.ParseInt(larkString(file.CreatedTime), 10, 64) + obj := model.Object{ + ID: larkString(file.Token), + Path: strings.Join([]string{rootFolderPath, dirPath, name}, "/"), + Name: larkDisplayName(name, fileType), + Size: 0, + Modified: time.Unix(modifiedUnix, 0), + Ctime: time.Unix(createdUnix, 0), + IsFolder: fileType == "folder", + } + if file.Url == nil || *file.Url == "" || obj.IsFolder || !isLarkNativeDocType(fileType) { + return &obj + } + return &model.ObjectURL{ + Object: obj, + Url: model.Url{Url: *file.Url}, + } +} + +func larkDisplayName(name, fileType string) string { + if isLarkCloudDocName(name) { + return name + } + switch fileType { + case "doc": + return name + ".lark-doc" + case "docx": + return name + ".lark-docx" + case "sheet": + return name + ".lark-sheet" + case "bitable": + return name + ".lark-bitable" + case "mindnote": + return name + ".lark-mindnote" + case "slides": + return name + ".lark-slides" + default: + return name + } +} + +func isLarkNativeDocType(fileType string) bool { + switch fileType { + case "doc", "docx", "sheet", "bitable", "mindnote", "slides": + return true + default: + return false + } +} + +func larkString(s *string) string { + if s == nil { + return "" + } + return *s +} + +func (c *Lark) requestOpts(ctx context.Context) ([]larkcore.RequestOptionFunc, error) { + userAccessToken, err := c.ensureUserAccessToken(ctx, false) + if err != nil { + return nil, err + } + if userAccessToken == "" { + return nil, nil + } + return []larkcore.RequestOptionFunc{larkcore.WithUserAccessToken(userAccessToken)}, nil +} + +func (c *Lark) ensureUserAccessToken(ctx context.Context, forceRefresh bool) (string, error) { + if strings.TrimSpace(c.RefreshToken) == "" { + return strings.TrimSpace(c.UserAccessToken), nil + } + if token := strings.TrimSpace(c.UserAccessToken); !forceRefresh && token != "" && !c.userAccessTokenExpired() { + return token, nil + } + + c.tokenMu.Lock() + defer c.tokenMu.Unlock() + + if token := strings.TrimSpace(c.UserAccessToken); !forceRefresh && token != "" && !c.userAccessTokenExpired() { + return token, nil + } + if c.RefreshTokenExpiresAt > 0 && time.Now().After(time.Unix(c.RefreshTokenExpiresAt, 0)) { + return "", errors.New("lark refresh token expired") + } + + resp, err := c.client.Ext.Authen.RefreshAuthenAccessToken(ctx, + larkext.NewRefreshAuthenAccessTokenReqBuilder(). + Body(larkext.NewRefreshAuthenAccessTokenReqBodyBuilder(). + GrantType(larkext.GrantTypeRefreshCode). + RefreshToken(strings.TrimSpace(c.RefreshToken)). + Build()). + Build()) + if err != nil { + return "", err + } + if !resp.Success() { + return "", errors.New(resp.Error()) + } + if resp.Data == nil || resp.Data.AccessToken == "" { + return "", errors.New("lark refresh token response missing access token") + } + + now := time.Now() + c.UserAccessToken = resp.Data.AccessToken + c.UserAccessTokenExpiresAt = now.Add(time.Duration(resp.Data.ExpiresIn) * time.Second).Unix() + if resp.Data.RefreshToken != "" { + c.RefreshToken = resp.Data.RefreshToken + } + if resp.Data.RefreshExpiresIn > 0 { + c.RefreshTokenExpiresAt = now.Add(time.Duration(resp.Data.RefreshExpiresIn) * time.Second).Unix() + } + op.MustSaveDriverStorage(c) + + return c.UserAccessToken, nil +} + +func (c *Lark) forceRefreshUserAccessToken(ctx context.Context) error { + if strings.TrimSpace(c.RefreshToken) == "" { + return nil + } + _, err := c.ensureUserAccessToken(ctx, true) + return err +} + +func (c *Lark) userAccessTokenExpired() bool { + if c.UserAccessTokenExpiresAt <= 0 { + return true + } + return time.Now().Add(larkTokenRefreshSkew).After(time.Unix(c.UserAccessTokenExpiresAt, 0)) +} + +func (c *Lark) listFiles(ctx context.Context, folderToken string) ([]*larkdrive.File, error) { + var files []*larkdrive.File + pageToken := "" + for { - ok, file, err = resp.Next() - if !ok { - break + builder := larkdrive.NewListFileReqBuilder(). + FolderToken(folderToken). + OrderBy("EditedTime"). + Direction("DESC") + if folderToken != "" { + builder.PageSize(larkListPageSize) + if pageToken != "" { + builder.PageToken(pageToken) + } } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.ListFileResp, error) { + return c.client.Drive.V1.File.List(ctx, builder.Build(), opts...) + }) if err != nil { return nil, err } + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + if resp.Data == nil { + return files, nil + } - modifiedUnix, _ := strconv.ParseInt(*file.ModifiedTime, 10, 64) - createdUnix, _ := strconv.ParseInt(*file.CreatedTime, 10, 64) - - f := model.Object{ - ID: *file.Token, - Path: strings.Join([]string{c.RootFolderPath, dir.GetPath(), *file.Name}, "/"), - Name: *file.Name, - Size: 0, - Modified: time.Unix(modifiedUnix, 0), - Ctime: time.Unix(createdUnix, 0), - IsFolder: *file.Type == "folder", + files = append(files, resp.Data.Files...) + if folderToken == "" || resp.Data.HasMore == nil || !*resp.Data.HasMore || + resp.Data.NextPageToken == nil || *resp.Data.NextPageToken == "" { + break } - res = append(res, &f) + pageToken = *resp.Data.NextPageToken } - return res, nil + return files, nil +} + +type larkResp interface { + Success() bool + Error() string +} + +func doDrive[T larkResp](ctx context.Context, c *Lark, call func(...larkcore.RequestOptionFunc) (T, error)) (T, error) { + opts, err := c.requestOpts(ctx) + if err != nil { + var zero T + return zero, err + } + + resp, err := call(opts...) + if err != nil { + var zero T + return zero, err + } + if !isLarkAuthFailed(resp) || strings.TrimSpace(c.RefreshToken) == "" { + return resp, nil + } + + log.WithField("mount_path", c.MountPath).Warn("lark user access token auth failed, refreshing and retrying once") + if err = c.forceRefreshUserAccessToken(ctx); err != nil { + return resp, nil + } + opts, err = c.requestOpts(ctx) + if err != nil { + var zero T + return zero, err + } + return call(opts...) +} + +func isLarkAuthFailed(resp larkResp) bool { + if resp == nil || resp.Success() { + return false + } + switch v := any(resp).(type) { + case *larkdrive.ListFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.CreateFolderFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.MoveFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.CopyFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.DeleteFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadPrepareFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadPartFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadFinishFileResp: + return isLarkAuthFailedCode(v.Code) + default: + return strings.Contains(resp.Error(), "1061005") || + strings.Contains(strings.ToLower(resp.Error()), "auth") + } +} + +func isLarkAuthFailedCode(code int) bool { + return code == 1061005 || code == 99991663 || code == 99991664 || code == 99991668 +} + +func isHTTPAuthFailed(statusCode int) bool { + return statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden } func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -133,17 +356,23 @@ func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* return nil, errs.ObjectNotFound } - resp, err := c.client.GetTenantAccessTokenBySelfBuiltApp(ctx, &larkcore.SelfBuiltTenantAccessTokenReq{ - AppID: c.AppId, - AppSecret: c.AppSecret, - }) + if isLarkCloudDocName(file.GetName()) { + return &model.Link{ + URL: c.filePreviewURL(token), + }, nil + } - if err != nil { - return nil, err + if !c.WebProxy || c.ExternalMode { + return &model.Link{ + URL: c.filePreviewURL(token), + }, nil } - if !c.ExternalMode { - accessToken := resp.TenantAccessToken + if c.WebProxy { + accessToken, err := c.downloadAccessToken(ctx, false) + if err != nil { + return nil, err + } url := fmt.Sprintf("https://open.feishu.cn/open-apis/drive/v1/files/%s/download", token) @@ -159,6 +388,20 @@ func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* if err != nil { return nil, err } + _ = ar.Body.Close() + + if isHTTPAuthFailed(ar.StatusCode) && strings.TrimSpace(c.RefreshToken) != "" { + accessToken, err = c.downloadAccessToken(ctx, true) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", accessToken)) + ar, err = http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + _ = ar.Body.Close() + } if ar.StatusCode != http.StatusPartialContent { return nil, errors.New("failed to get download link") @@ -170,13 +413,46 @@ func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* "Authorization": []string{fmt.Sprintf("Bearer %s", accessToken)}, }, }, nil - } else { - url := strings.Join([]string{c.TenantUrlPrefix, "file", token}, "/") + } - return &model.Link{ - URL: url, - }, nil + return nil, errors.New("lark download requires web proxy") +} + +func (c *Lark) filePreviewURL(token string) string { + prefix := strings.TrimRight(strings.TrimSpace(c.TenantUrlPrefix), "/") + if prefix == "" { + prefix = "https://www.feishu.cn" + } + return prefix + "/file/" + token +} + +func (c *Lark) downloadAccessToken(ctx context.Context, forceRefresh bool) (string, error) { + var accessToken string + var err error + if strings.TrimSpace(c.RefreshToken) != "" || strings.TrimSpace(c.UserAccessToken) != "" { + accessToken, err = c.ensureUserAccessToken(ctx, forceRefresh) + if err != nil { + return "", err + } } + if accessToken != "" { + return accessToken, nil + } + + resp, err := c.client.GetTenantAccessTokenBySelfBuiltApp(ctx, &larkcore.SelfBuiltTenantAccessTokenReq{ + AppID: c.AppId, + AppSecret: c.AppSecret, + }) + if err != nil { + return "", err + } + if !resp.Success() { + return "", errors.New(resp.Error()) + } + if resp.TenantAccessToken == "" { + return "", errors.New("lark tenant access token is empty") + } + return resp.TenantAccessToken, nil } func (c *Lark) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { @@ -190,8 +466,10 @@ func (c *Lark) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) return nil, err } - resp, err := c.client.Drive.File.CreateFolder(ctx, - larkdrive.NewCreateFolderFileReqBuilder().Body(body).Build()) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CreateFolderFileResp, error) { + return c.client.Drive.File.CreateFolder(ctx, + larkdrive.NewCreateFolderFileReqBuilder().Body(body).Build(), opts...) + }) if err != nil { return nil, err } @@ -228,7 +506,9 @@ func (c *Lark) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, e Build() // 发起请求 - resp, err := c.client.Drive.File.Move(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.MoveFileResp, error) { + return c.client.Drive.File.Move(ctx, req, opts...) + }) if err != nil { return nil, err } @@ -265,7 +545,9 @@ func (c *Lark) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, e Build() // 发起请求 - resp, err := c.client.Drive.File.Copy(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CopyFileResp, error) { + return c.client.Drive.File.Copy(ctx, req, opts...) + }) if err != nil { return nil, err } @@ -289,7 +571,9 @@ func (c *Lark) Remove(ctx context.Context, obj model.Obj) error { Build() // 发起请求 - resp, err := c.client.Drive.File.Delete(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.DeleteFileResp, error) { + return c.client.Drive.File.Delete(ctx, req, opts...) + }) if err != nil { return err } @@ -324,7 +608,9 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return nil, err } - resp, err := c.client.Drive.File.UploadPrepare(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadPrepareFileResp, error) { + return c.client.Drive.File.UploadPrepare(ctx, req, opts...) + }) if err != nil { return nil, err } @@ -360,7 +646,9 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return nil, err } - resp, err := c.client.Drive.File.UploadPart(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadPartFileResp, error) { + return c.client.Drive.File.UploadPart(ctx, req, opts...) + }) if err != nil { return nil, err @@ -382,7 +670,9 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea Build() // 发起请求 - closeResp, err := c.client.Drive.File.UploadFinish(ctx, closeReq) + closeResp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadFinishFileResp, error) { + return c.client.Drive.File.UploadFinish(ctx, closeReq, opts...) + }) if err != nil { return nil, err } diff --git a/drivers/lark/meta.go b/drivers/lark/meta.go index 221345e222c..fe6149d99a4 100644 --- a/drivers/lark/meta.go +++ b/drivers/lark/meta.go @@ -9,10 +9,14 @@ type Addition struct { // Usually one of two driver.RootPath // define other - AppId string `json:"app_id" type:"text" help:"app id"` - AppSecret string `json:"app_secret" type:"text" help:"app secret"` - ExternalMode bool `json:"external_mode" type:"bool" help:"external mode"` - TenantUrlPrefix string `json:"tenant_url_prefix" type:"text" help:"tenant url prefix"` + AppId string `json:"app_id" type:"text" help:"app id"` + AppSecret string `json:"app_secret" type:"text" help:"app secret"` + UserAccessToken string `json:"user_access_token" type:"text" help:"optional cached user access token for personal drive access"` + RefreshToken string `json:"refresh_token" type:"text" help:"optional refresh token for user access token auto refresh"` + UserAccessTokenExpiresAt int64 `json:"user_access_token_expires_at" type:"number" help:"user access token expires at unix timestamp"` + RefreshTokenExpiresAt int64 `json:"refresh_token_expires_at" type:"number" help:"refresh token expires at unix timestamp"` + ExternalMode bool `json:"external_mode" type:"bool" help:"external mode"` + TenantUrlPrefix string `json:"tenant_url_prefix" type:"text" help:"tenant url prefix"` } var config = driver.Config{ diff --git a/drivers/lark/other.go b/drivers/lark/other.go new file mode 100644 index 00000000000..f0217b72621 --- /dev/null +++ b/drivers/lark/other.go @@ -0,0 +1,433 @@ +package lark + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + larkcore "github.com/larksuite/oapi-sdk-go/v3/core" + larkbitable "github.com/larksuite/oapi-sdk-go/v3/service/bitable/v1" + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" + larksheets "github.com/larksuite/oapi-sdk-go/v3/service/sheets/v3" + "github.com/pkg/errors" +) + +const ( + larkExportOptionsMethod = "lark_export_options" + larkExportCreateMethod = "lark_export_create" + larkExportStatusMethod = "lark_export_status" + + larkExportStatusPending = "pending" + larkExportStatusProcessing = "processing" + larkExportStatusSuccess = "success" + larkExportStatusFailed = "failed" +) + +type larkExportCreateReq struct { + Format string `json:"format"` + SubID string `json:"sub_id"` +} + +type larkExportStatusReq struct { + Ticket string `json:"ticket"` +} + +type LarkExportCreateResp struct { + Ticket string `json:"ticket"` + Token string `json:"token"` + Type string `json:"type"` + Format string `json:"format"` + SubID string `json:"sub_id,omitempty"` +} + +type LarkExportOption struct { + Value string `json:"value"` + Label string `json:"label"` + RequiresSubID bool `json:"requires_sub_id"` +} + +type LarkExportSubResource struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` +} + +type LarkExportOptionsResp struct { + Type string `json:"type"` + Formats []LarkExportOption `json:"formats"` + SubResources []LarkExportSubResource `json:"sub_resources,omitempty"` + SubResourceError string `json:"sub_resource_error,omitempty"` +} + +type LarkExportStatusResp struct { + Status string `json:"status"` + FileToken string `json:"file_token,omitempty"` + FileSize int `json:"file_size,omitempty"` + JobStatus int `json:"job_status,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ErrorDetail string `json:"error_detail,omitempty"` +} + +type larkAPIErrorResp interface { + Error() string + ErrorResp() string +} + +func (c *Lark) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch strings.ToLower(strings.TrimSpace(args.Method)) { + case larkExportOptionsMethod: + return c.getExportOptions(ctx, args.Obj) + case larkExportCreateMethod: + var req larkExportCreateReq + if err := decodeOtherData(args.Data, &req); err != nil { + return nil, err + } + return c.createExportTask(ctx, args.Obj, req) + case larkExportStatusMethod: + var req larkExportStatusReq + if err := decodeOtherData(args.Data, &req); err != nil { + return nil, err + } + return c.getExportTask(ctx, args.Obj, req) + default: + return nil, fmt.Errorf("unsupported lark method: %s", args.Method) + } +} + +func decodeOtherData(data interface{}, v interface{}) error { + b, err := json.Marshal(data) + if err != nil { + return errors.WithMessage(err, "failed to encode request data") + } + if err = json.Unmarshal(b, v); err != nil { + return errors.WithMessage(err, "failed to decode request data") + } + return nil +} + +func (c *Lark) createExportTask(ctx context.Context, obj model.Obj, req larkExportCreateReq) (*LarkExportCreateResp, error) { + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + docType, err := larkExportType(obj.GetName()) + if err != nil { + return nil, err + } + format := strings.ToLower(strings.TrimSpace(req.Format)) + if !larkExportFormatAllowed(docType, format) { + return nil, fmt.Errorf("unsupported export format %q for lark type %q", format, docType) + } + subID := strings.TrimSpace(req.SubID) + if larkExportFormatRequiresSubID(docType, format) && subID == "" { + return nil, fmt.Errorf("sub_id is required when exporting lark type %q as %q", docType, format) + } + + builder := larkdrive.NewExportTaskBuilder(). + Token(token). + Type(docType). + FileExtension(format). + FileName(larkExportBaseName(obj.GetName())) + if subID != "" { + builder.SubId(subID) + } + exportTask := builder.Build() + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CreateExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Create(ctx, + larkdrive.NewCreateExportTaskReqBuilder().ExportTask(exportTask).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil || resp.Data.Ticket == nil || *resp.Data.Ticket == "" { + return nil, errors.New("lark export task response missing ticket") + } + return &LarkExportCreateResp{ + Ticket: *resp.Data.Ticket, + Token: token, + Type: docType, + Format: format, + SubID: subID, + }, nil +} + +func (c *Lark) getExportOptions(ctx context.Context, obj model.Obj) (*LarkExportOptionsResp, error) { + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + docType, err := larkExportType(obj.GetName()) + if err != nil { + return nil, err + } + out := &LarkExportOptionsResp{ + Type: docType, + Formats: larkExportOptions(docType), + } + switch docType { + case larkdrive.TypeSheet: + out.SubResources, err = c.listSheetSubResources(ctx, token) + case larkdrive.TypeBitable: + out.SubResources, err = c.listBitableSubResources(ctx, token) + } + if err != nil { + out.SubResourceError = err.Error() + } + return out, nil +} + +func (c *Lark) getExportTask(ctx context.Context, obj model.Obj, req larkExportStatusReq) (*LarkExportStatusResp, error) { + ticket := strings.TrimSpace(req.Ticket) + if ticket == "" { + return nil, errors.New("ticket is required") + } + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.GetExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Get(ctx, + larkdrive.NewGetExportTaskReqBuilder().Ticket(ticket).Token(token).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil || resp.Data.Result == nil { + return nil, errors.New("lark export task response missing result") + } + result := resp.Data.Result + out := &LarkExportStatusResp{ + Status: larkExportJobStatus(result), + } + if result.FileToken != nil { + out.FileToken = *result.FileToken + } + if result.FileSize != nil { + out.FileSize = *result.FileSize + } + if result.JobStatus != nil { + out.JobStatus = *result.JobStatus + } + if out.Status == larkExportStatusFailed { + out.ErrorMessage = larkExportTaskErrorMessage(result) + out.ErrorDetail = larkExportTaskErrorDetail(result) + } else if result.JobErrorMsg != nil { + out.ErrorMessage = strings.TrimSpace(*result.JobErrorMsg) + } + return out, nil +} + +func (c *Lark) DownloadExportFile(ctx context.Context, fileToken string) (io.Reader, string, error) { + fileToken = strings.TrimSpace(fileToken) + if fileToken == "" { + return nil, "", errors.New("file_token is required") + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.DownloadExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Download(ctx, + larkdrive.NewDownloadExportTaskReqBuilder().FileToken(fileToken).Build(), opts...) + }) + if err != nil { + return nil, "", err + } + if !resp.Success() { + return nil, "", larkAPIError(resp) + } + if resp.File == nil { + return nil, "", errors.New("lark export download response missing file") + } + return resp.File, resp.FileName, nil +} + +func larkExportType(name string) (string, error) { + switch { + case strings.HasSuffix(name, ".lark-doc"): + return larkdrive.TypeDoc, nil + case strings.HasSuffix(name, ".lark-docx"): + return larkdrive.TypeDocx, nil + case strings.HasSuffix(name, ".lark-sheet"): + return larkdrive.TypeSheet, nil + case strings.HasSuffix(name, ".lark-bitable"): + return larkdrive.TypeBitable, nil + default: + return "", fmt.Errorf("unsupported lark export file type: %s", name) + } +} + +func larkExportFormatAllowed(docType, format string) bool { + switch docType { + case larkdrive.TypeDoc, larkdrive.TypeDocx: + return format == larkdrive.FileExtensionPdf || format == larkdrive.FileExtensionDocx + case larkdrive.TypeSheet, larkdrive.TypeBitable: + return format == larkdrive.FileExtensionXlsx || format == larkdrive.FileExtensionCsv + default: + return false + } +} + +func larkExportFormatRequiresSubID(docType, format string) bool { + return (docType == larkdrive.TypeSheet || docType == larkdrive.TypeBitable) && format == larkdrive.FileExtensionCsv +} + +func larkExportOptions(docType string) []LarkExportOption { + switch docType { + case larkdrive.TypeDoc, larkdrive.TypeDocx: + return []LarkExportOption{ + {Value: larkdrive.FileExtensionPdf, Label: "PDF"}, + {Value: larkdrive.FileExtensionDocx, Label: "DOCX"}, + } + case larkdrive.TypeSheet, larkdrive.TypeBitable: + return []LarkExportOption{ + {Value: larkdrive.FileExtensionXlsx, Label: "XLSX"}, + {Value: larkdrive.FileExtensionCsv, Label: "CSV", RequiresSubID: true}, + } + default: + return nil + } +} + +func larkExportBaseName(name string) string { + return trimLarkDisplayExt(name) +} + +func larkExportJobStatus(result *larkdrive.ExportTask) string { + if result == nil { + return larkExportStatusProcessing + } + if result.FileToken != nil && *result.FileToken != "" { + return larkExportStatusSuccess + } + if result.JobStatus != nil && *result.JobStatus == 2 { + return larkExportStatusFailed + } + if result.JobErrorMsg != nil && *result.JobErrorMsg != "" && !strings.EqualFold(*result.JobErrorMsg, "success") { + return larkExportStatusFailed + } + return larkExportStatusProcessing +} + +func larkAPIError(resp larkAPIErrorResp) error { + msg := strings.TrimSpace(resp.ErrorResp()) + if msg == "" || msg == "{}" || msg == "null" { + msg = strings.TrimSpace(resp.Error()) + } + return errors.New(msg) +} + +func larkExportTaskErrorMessage(result *larkdrive.ExportTask) string { + if result == nil { + return "" + } + if result.JobErrorMsg != nil { + msg := strings.TrimSpace(*result.JobErrorMsg) + if msg != "" && !strings.EqualFold(msg, "success") { + return msg + } + } + if result.JobStatus != nil { + return fmt.Sprintf("job_status=%d", *result.JobStatus) + } + return "" +} + +func larkExportTaskErrorDetail(result *larkdrive.ExportTask) string { + if result == nil { + return "" + } + detail := map[string]interface{}{} + if result.JobStatus != nil { + detail["job_status"] = *result.JobStatus + } + if result.JobErrorMsg != nil { + detail["job_error_msg"] = strings.TrimSpace(*result.JobErrorMsg) + } + if len(detail) == 0 { + return "" + } + b, err := json.Marshal(detail) + if err != nil { + return "" + } + return string(b) +} + +func (c *Lark) listSheetSubResources(ctx context.Context, token string) ([]LarkExportSubResource, error) { + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larksheets.QuerySpreadsheetSheetResp, error) { + return c.client.Sheets.SpreadsheetSheet.Query(ctx, + larksheets.NewQuerySpreadsheetSheetReqBuilder().SpreadsheetToken(token).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil { + return nil, nil + } + var res []LarkExportSubResource + for _, sheet := range resp.Data.Sheets { + if sheet == nil || sheet.SheetId == nil || strings.TrimSpace(*sheet.SheetId) == "" { + continue + } + name := strings.TrimSpace(larkString(sheet.Title)) + if name == "" { + name = *sheet.SheetId + } + res = append(res, LarkExportSubResource{ + ID: *sheet.SheetId, + Name: name, + Type: strings.TrimSpace(larkString(sheet.ResourceType)), + }) + } + return res, nil +} + +func (c *Lark) listBitableSubResources(ctx context.Context, token string) ([]LarkExportSubResource, error) { + var res []LarkExportSubResource + pageToken := "" + for { + builder := larkbitable.NewListAppTableReqBuilder().AppToken(token).PageSize(100) + if pageToken != "" { + builder.PageToken(pageToken) + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkbitable.ListAppTableResp, error) { + return c.client.Bitable.AppTable.List(ctx, builder.Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil { + return res, nil + } + for _, table := range resp.Data.Items { + if table == nil || table.TableId == nil || strings.TrimSpace(*table.TableId) == "" { + continue + } + name := strings.TrimSpace(larkString(table.Name)) + if name == "" { + name = *table.TableId + } + res = append(res, LarkExportSubResource{ + ID: *table.TableId, + Name: name, + Type: "table", + }) + } + if resp.Data.HasMore == nil || !*resp.Data.HasMore || resp.Data.PageToken == nil || *resp.Data.PageToken == "" { + return res, nil + } + pageToken = *resp.Data.PageToken + } +} diff --git a/drivers/lark/other_test.go b/drivers/lark/other_test.go new file mode 100644 index 00000000000..6887c6bcd65 --- /dev/null +++ b/drivers/lark/other_test.go @@ -0,0 +1,176 @@ +package lark + +import ( + "testing" + + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" +) + +func TestLarkExportType(t *testing.T) { + tests := []struct { + name string + want string + wantErr bool + }{ + {name: "doc.lark-doc", want: larkdrive.TypeDoc}, + {name: "doc.lark-docx", want: larkdrive.TypeDocx}, + {name: "sheet.lark-sheet", want: larkdrive.TypeSheet}, + {name: "base.lark-bitable", want: larkdrive.TypeBitable}, + {name: "file.pdf", wantErr: true}, + } + for _, tt := range tests { + got, err := larkExportType(tt.name) + if tt.wantErr { + if err == nil { + t.Fatalf("larkExportType(%q) expected error", tt.name) + } + continue + } + if err != nil { + t.Fatalf("larkExportType(%q) unexpected error: %v", tt.name, err) + } + if got != tt.want { + t.Fatalf("larkExportType(%q) = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportFormatAllowed(t *testing.T) { + tests := []struct { + docType string + format string + want bool + }{ + {docType: larkdrive.TypeDoc, format: larkdrive.FileExtensionPdf, want: true}, + {docType: larkdrive.TypeDocx, format: larkdrive.FileExtensionDocx, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionXlsx, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionXlsx, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionPdf, want: false}, + } + for _, tt := range tests { + if got := larkExportFormatAllowed(tt.docType, tt.format); got != tt.want { + t.Fatalf("larkExportFormatAllowed(%q, %q) = %v, want %v", tt.docType, tt.format, got, tt.want) + } + } +} + +func TestLarkExportFormatRequiresSubID(t *testing.T) { + tests := []struct { + docType string + format string + want bool + }{ + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionXlsx, want: false}, + {docType: larkdrive.TypeDocx, format: larkdrive.FileExtensionDocx, want: false}, + } + for _, tt := range tests { + if got := larkExportFormatRequiresSubID(tt.docType, tt.format); got != tt.want { + t.Fatalf("larkExportFormatRequiresSubID(%q, %q) = %v, want %v", tt.docType, tt.format, got, tt.want) + } + } +} + +func TestLarkExportOptions(t *testing.T) { + tests := []struct { + docType string + want []LarkExportOption + }{ + {docType: larkdrive.TypeDocx, want: []LarkExportOption{ + {Value: larkdrive.FileExtensionPdf, Label: "PDF"}, + {Value: larkdrive.FileExtensionDocx, Label: "DOCX"}, + }}, + {docType: larkdrive.TypeSheet, want: []LarkExportOption{ + {Value: larkdrive.FileExtensionXlsx, Label: "XLSX"}, + {Value: larkdrive.FileExtensionCsv, Label: "CSV", RequiresSubID: true}, + }}, + } + for _, tt := range tests { + got := larkExportOptions(tt.docType) + if len(got) != len(tt.want) { + t.Fatalf("larkExportOptions(%q) len = %d, want %d", tt.docType, len(got), len(tt.want)) + } + for i := range got { + if got[i] != tt.want[i] { + t.Fatalf("larkExportOptions(%q)[%d] = %+v, want %+v", tt.docType, i, got[i], tt.want[i]) + } + } + } +} + +func TestLarkExportBaseName(t *testing.T) { + tests := map[string]string{ + "weekly.lark-docx": "weekly", + "weekly.report.lark-doc": "weekly.report", + "table.lark-sheet": "table", + "plain.pdf": "plain.pdf", + } + for name, want := range tests { + if got := larkExportBaseName(name); got != want { + t.Fatalf("larkExportBaseName(%q) = %q, want %q", name, got, want) + } + } +} + +func TestLarkExportJobStatus(t *testing.T) { + successToken := "box_token" + successCode := 0 + failCode := 2 + failMsg := "failed" + successMsg := "success" + + tests := []struct { + name string + result *larkdrive.ExportTask + want string + }{ + {name: "nil", result: nil, want: larkExportStatusProcessing}, + {name: "file token wins", result: &larkdrive.ExportTask{FileToken: &successToken, JobStatus: &failCode}, want: larkExportStatusSuccess}, + {name: "status failure", result: &larkdrive.ExportTask{JobStatus: &failCode}, want: larkExportStatusFailed}, + {name: "error message failure", result: &larkdrive.ExportTask{JobStatus: &successCode, JobErrorMsg: &failMsg}, want: larkExportStatusFailed}, + {name: "success message without token still processing", result: &larkdrive.ExportTask{JobStatus: &successCode, JobErrorMsg: &successMsg}, want: larkExportStatusProcessing}, + } + for _, tt := range tests { + if got := larkExportJobStatus(tt.result); got != tt.want { + t.Fatalf("%s: larkExportJobStatus() = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportTaskErrorMessage(t *testing.T) { + failCode := 2 + failMsg := "source document cannot be exported" + successMsg := "success" + + tests := []struct { + name string + result *larkdrive.ExportTask + want string + }{ + {name: "nil", result: nil, want: ""}, + {name: "job error message", result: &larkdrive.ExportTask{JobStatus: &failCode, JobErrorMsg: &failMsg}, want: failMsg}, + {name: "status fallback", result: &larkdrive.ExportTask{JobStatus: &failCode, JobErrorMsg: &successMsg}, want: "job_status=2"}, + } + for _, tt := range tests { + if got := larkExportTaskErrorMessage(tt.result); got != tt.want { + t.Fatalf("%s: larkExportTaskErrorMessage() = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportTaskErrorDetail(t *testing.T) { + failCode := 2 + failMsg := "source document cannot be exported" + + got := larkExportTaskErrorDetail(&larkdrive.ExportTask{ + JobStatus: &failCode, + JobErrorMsg: &failMsg, + }) + want := `{"job_error_msg":"source document cannot be exported","job_status":2}` + if got != want { + t.Fatalf("larkExportTaskErrorDetail() = %q, want %q", got, want) + } +} diff --git a/drivers/lark/util.go b/drivers/lark/util.go index 8c6828bd176..ebef4300c49 100644 --- a/drivers/lark/util.go +++ b/drivers/lark/util.go @@ -3,9 +3,9 @@ package lark import ( "context" "github.com/Xhofe/go-cache" - larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" log "github.com/sirupsen/logrus" "path" + "strings" "time" ) @@ -36,31 +36,47 @@ func (c *Lark) getObjToken(ctx context.Context, folderPath string) (string, bool return emptyFolderToken, false } - req := larkdrive.NewListFileReqBuilder().FolderToken(parentToken).Build() - resp, err := c.client.Drive.File.ListByIterator(ctx, req) - + files, err := c.listFiles(ctx, parentToken) if err != nil { log.WithError(err).Error("failed to list files") return emptyFolderToken, false } - var file *larkdrive.File - for { - found, file, err = resp.Next() - if !found { - break + for _, file := range files { + if *file.Name == name || *file.Name == trimLarkDisplayExt(name) { + objTokenCache.Set(folderPath, *file.Token, exOpts) + return *file.Token, true } + } + + return emptyFolderToken, false +} - if err != nil { - log.WithError(err).Error("failed to get next file") - break +func trimLarkDisplayExt(name string) string { + for _, suffix := range larkCloudDocSuffixes() { + if strings.HasSuffix(name, suffix) { + return strings.TrimSuffix(name, suffix) } + } + return name +} - if *file.Name == name { - objTokenCache.Set(folderPath, *file.Token, exOpts) - return *file.Token, true +func isLarkCloudDocName(name string) bool { + for _, suffix := range larkCloudDocSuffixes() { + if strings.HasSuffix(name, suffix) { + return true } } + return false +} - return emptyFolderToken, false +func larkCloudDocSuffixes() []string { + return []string{ + ".lark-doc", + ".lark-docx", + ".lark-sheet", + ".lark-bitable", + ".lark-mindnote", + ".lark-slides", + } } diff --git a/drivers/local/driver.go b/drivers/local/driver.go index faa2b3bd157..ef7ce6b4e2a 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -18,6 +18,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" @@ -31,6 +32,7 @@ type Local struct { model.Storage Addition mkdirPerm int32 + thumbSize int // zero means no limit thumbConcurrency int @@ -39,6 +41,10 @@ type Local struct { // video thumb position videoThumbPos float64 videoThumbPosIsPercentage bool + thumbPixel int + + // use ffmpeg + useFFmpeg bool } func (d *Local) Config() driver.Config { @@ -65,12 +71,26 @@ func (d *Local) Init(ctx context.Context) error { } d.Addition.RootFolderPath = abs } + + d.useFFmpeg = d.UseFFmpeg + if d.ThumbCacheFolder != "" && !utils.Exists(d.ThumbCacheFolder) { err := os.MkdirAll(d.ThumbCacheFolder, os.FileMode(d.mkdirPerm)) if err != nil { return err } } + d.thumbSize = 144 + if item, err := op.GetSettingItemByKey(conf.ThumbnailSize); err == nil && item != nil && strings.TrimSpace(item.Value) != "" { + v, err := strconv.ParseUint(item.Value, 10, 32) + if err != nil { + return fmt.Errorf("invalid setting %s value: %s, err: %s", conf.ThumbnailSize, item.Value, err) + } + if v == 0 { + return fmt.Errorf("invalid setting %s value: %s, the value must be a positive integer", conf.ThumbnailSize, item.Value) + } + d.thumbSize = int(v) + } if d.ThumbConcurrency != "" { v, err := strconv.ParseUint(d.ThumbConcurrency, 10, 32) if err != nil { @@ -78,6 +98,14 @@ func (d *Local) Init(ctx context.Context) error { } d.thumbConcurrency = int(v) } + if d.ThumbPixel != "" { + v, err := strconv.ParseUint(d.ThumbPixel, 10, 32) + if err != nil { + return err + } + d.thumbPixel = int(v) + } + if d.thumbConcurrency == 0 { d.thumbTokenBucket = NewNopTokenBucket() } else { @@ -146,13 +174,14 @@ func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(reqPath, f.Name())) } } - isFolder := f.IsDir() || isSymlinkDir(f, fullPath) + filePath := filepath.Join(fullPath, f.Name()) + isFolder := f.IsDir() || isLinkedDir(f, filePath) var size int64 if !isFolder { size = f.Size() } var ctime time.Time - t, err := times.Stat(stdpath.Join(fullPath, f.Name())) + t, err := times.Stat(filePath) if err == nil { if t.HasBirthTime() { ctime = t.BirthTime() @@ -161,7 +190,7 @@ func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string file := model.ObjThumb{ Object: model.Object{ - Path: filepath.Join(fullPath, f.Name()), + Path: filePath, Name: f.Name(), Modified: f.ModTime(), Size: size, @@ -197,7 +226,7 @@ func (d *Local) Get(ctx context.Context, path string) (model.Obj, error) { } return nil, err } - isFolder := f.IsDir() || isSymlinkDir(f, path) + isFolder := f.IsDir() || isLinkedDir(f, path) size := f.Size() if isFolder { size = 0 diff --git a/drivers/local/meta.go b/drivers/local/meta.go index 14b0404f784..70ce090db5f 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -8,8 +8,10 @@ import ( type Addition struct { driver.RootPath Thumbnail bool `json:"thumbnail" required:"true" help:"enable thumbnail"` + UseFFmpeg bool `json:"use_ffmpeg" required:"true" help:"use ffmpeg to generate thumbnail"` ThumbCacheFolder string `json:"thumb_cache_folder"` ThumbConcurrency string `json:"thumb_concurrency" default:"16" required:"false" help:"Number of concurrent thumbnail generation goroutines. This controls how many thumbnails can be generated in parallel."` + ThumbPixel string `json:"thumb_pixel" default:"320" required:"false" help:"Specifies the target width for image thumbnails in pixels. The height of the thumbnail will be calculated automatically to maintain the original aspect ratio of the image."` VideoThumbPos string `json:"video_thumb_pos" default:"20%" required:"false" help:"The position of the video thumbnail. If the value is a number (integer ot floating point), it represents the time in seconds. If the value ends with '%', it represents the percentage of the video duration."` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` MkdirPerm string `json:"mkdir_perm" default:"777"` diff --git a/drivers/local/util.go b/drivers/local/util.go index 802f60cf627..fa95ddd24a0 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -7,6 +7,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -18,14 +19,18 @@ import ( ffmpeg "github.com/u2takey/ffmpeg-go" ) -func isSymlinkDir(f fs.FileInfo, path string) bool { - if f.Mode()&os.ModeSymlink == os.ModeSymlink { - dst, err := os.Readlink(filepath.Join(path, f.Name())) +func isLinkedDir(f fs.FileInfo, path string) bool { + if f.Mode()&os.ModeSymlink == os.ModeSymlink || (runtime.GOOS == "windows" && f.Mode()&os.ModeIrregular != 0) { + dst, err := os.Readlink(path) if err != nil { return false } if !filepath.IsAbs(dst) { - dst = filepath.Join(path, dst) + dst = filepath.Join(filepath.Dir(path), dst) + } + dst, err = filepath.Abs(dst) + if err != nil { + return false } stat, err := os.Stat(dst) if err != nil { @@ -36,6 +41,87 @@ func isSymlinkDir(f fs.FileInfo, path string) bool { return false } +// resizeImageToBufferWithFFmpegGo 使用 ffmpeg-go 调整图片大小并输出到内存缓冲区 +func resizeImageToBufferWithFFmpegGo(inputFile string, width int, outputFormat string /* e.g., "image2pipe", "png_pipe", "mjpeg" */) (*bytes.Buffer, error) { + outBuffer := bytes.NewBuffer(nil) + + // Determine codec based on desired output format for piping + // For generic image piping, 'image2' is often used with -f image2pipe + // For specific formats to buffer, you might specify the codec directly + var vcodec string + switch outputFormat { + case "png_pipe": // if you want to ensure PNG format in buffer + vcodec = "png" + case "mjpeg": // if you want to ensure JPEG format in buffer + vcodec = "mjpeg" + // default or "image2pipe" could leave codec choice more to ffmpeg or require -c:v later + } + + outputArgs := ffmpeg.KwArgs{ + "vf": fmt.Sprintf("scale=%d:-1:flags=lanczos,format=yuv444p", width), + "vframes": "1", + "f": outputFormat, // Format for piping (e.g., image2pipe, png_pipe) + } + if vcodec != "" { + outputArgs["vcodec"] = vcodec + } + if outputFormat == "mjpeg" { + outputArgs["q:v"] = "3" + } + + err := ffmpeg.Input(inputFile). + Output("pipe:", outputArgs). // Output to pipe (stdout) + GlobalArgs("-loglevel", "error"). + Silent(true). // Suppress ffmpeg's own console output + WithOutput(outBuffer, os.Stderr). // Capture stdout to outBuffer, stderr to os.Stderr + // ErrorToStdOut(). // Alternative: send ffmpeg's stderr to Go's stdout + Run() + + if err != nil { + return nil, fmt.Errorf("ffmpeg-go failed to resize image %s to buffer: %w", inputFile, err) + } + if outBuffer == nil || outBuffer.Len() == 0 { + return nil, fmt.Errorf("ffmpeg-go produced empty buffer for %s", inputFile) + } + + return outBuffer, nil +} + +func generateThumbnailWithImagingOptimized(imagePath string, targetWidth int, quality int) (*bytes.Buffer, error) { + + file, err := os.Open(imagePath) + if err != nil { + return nil, fmt.Errorf("failed to open image: %w", err) + } + defer file.Close() + + img, err := imaging.Decode(file, imaging.AutoOrientation(true)) + if err != nil { + return nil, fmt.Errorf("failed to decode image: %w", err) + } + + thumbImg := imaging.Resize(img, targetWidth, 0, imaging.Lanczos) + img = nil + + var buf bytes.Buffer + // imaging.Encode + // imaging.PNG, imaging.JPEG, imaging.GIF, imaging.BMP, imaging.TIFF + outputFormat := imaging.JPEG + encodeOptions := []imaging.EncodeOption{imaging.JPEGQuality(quality)} + + // outputFormat := imaging.PNG + // encodeOptions := []imaging.EncodeOption{} + + err = imaging.Encode(&buf, thumbImg, outputFormat, encodeOptions...) + if err != nil { + return nil, fmt.Errorf("failed to encode thumbnail: %w", err) + } + + thumbImg = nil + + return &buf, nil +} + // Get the snapshot of the video func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) { // Run ffprobe to get the video duration @@ -80,7 +166,7 @@ func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) // The "noaccurate_seek" option prevents this error and would also speed up // the seek process. stream := ffmpeg.Input(videoPath, ffmpeg.KwArgs{"ss": ss, "noaccurate_seek": ""}). - Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg"}). + Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg", "vf": fmt.Sprintf("scale=%d:-1:flags=lanczos", d.thumbPixel)}). GlobalArgs("-loglevel", "error").Silent(true). WithOutput(srcBuf, os.Stdout) if err = stream.Run(); err != nil { @@ -106,7 +192,7 @@ func readDir(dirname string) ([]fs.FileInfo, error) { func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { fullPath := file.GetPath() thumbPrefix := "alist_thumb_" - thumbName := thumbPrefix + utils.GetMD5EncodeStr(fullPath) + ".png" + thumbName := thumbPrefix + utils.GetMD5EncodeStr(fmt.Sprintf("%s:%d", fullPath, d.thumbSize)) + ".png" if d.ThumbCacheFolder != "" { // skip if the file is a thumbnail if strings.HasPrefix(file.GetName(), thumbPrefix) { @@ -125,29 +211,26 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { } srcBuf = videoBuf } else { - imgData, err := os.ReadFile(fullPath) - if err != nil { - return nil, nil, err + if d.useFFmpeg { + imgData, err := resizeImageToBufferWithFFmpegGo(fullPath, d.thumbPixel, "image2pipe") + srcBuf = imgData + if err != nil { + return nil, nil, err + } + } else { + imgData, err := generateThumbnailWithImagingOptimized(fullPath, d.thumbPixel, 70) + srcBuf = imgData + if err != nil { + return nil, nil, err + } } - imgBuf := bytes.NewBuffer(imgData) - srcBuf = imgBuf } - image, err := imaging.Decode(srcBuf, imaging.AutoOrientation(true)) - if err != nil { - return nil, nil, err - } - thumbImg := imaging.Resize(image, 144, 0, imaging.Lanczos) - var buf bytes.Buffer - err = imaging.Encode(&buf, thumbImg, imaging.PNG) - if err != nil { - return nil, nil, err - } if d.ThumbCacheFolder != "" { - err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0666) + err := os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), srcBuf.Bytes(), 0666) if err != nil { return nil, nil, err } } - return &buf, nil, nil + return srcBuf, nil, nil } diff --git a/drivers/mediafire/driver.go b/drivers/mediafire/driver.go new file mode 100644 index 00000000000..e77510eabc0 --- /dev/null +++ b/drivers/mediafire/driver.go @@ -0,0 +1,433 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "context" + "fmt" + "math/rand" + "net/http" + "os" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Mediafire struct { + model.Storage + Addition + cron *cron.Cron + + actionToken string + + appBase string + apiBase string + hostBase string + maxRetries int + + secChUa string + secChUaPlatform string + userAgent string +} + +func (d *Mediafire) Config() driver.Config { + return config +} + +func (d *Mediafire) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Mediafire) Init(ctx context.Context) error { + if d.SessionToken == "" { + return fmt.Errorf("Init :: [MediaFire] {critical} missing sessionToken") + } + + if d.Cookie == "" { + return fmt.Errorf("Init :: [MediaFire] {critical} missing Cookie") + } + + if _, err := d.getSessionToken(ctx); err != nil { + + d.renewToken(ctx) + + num := rand.Intn(4) + 6 + + d.cron = cron.NewCron(time.Minute * time.Duration(num)) + d.cron.Do(func() { + d.renewToken(ctx) + }) + + } + + return nil +} + +func (d *Mediafire) Drop(ctx context.Context) error { + return nil +} + +func (d *Mediafire) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(ctx, dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return d.fileToObj(src), nil + }) +} + +func (d *Mediafire) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + + downloadUrl, err := d.getDirectDownloadLink(ctx, file.GetID()) + if err != nil { + return nil, err + } + + res, err := base.NoRedirectClient.R().SetDoNotParseResponse(true).SetContext(ctx).Get(downloadUrl) + if err != nil { + return nil, err + } + defer func() { + _ = res.RawBody().Close() + }() + + if res.StatusCode() == 302 { + downloadUrl = res.Header().Get("location") + } + + return &model.Link{ + URL: downloadUrl, + Header: http.Header{ + "Origin": []string{d.appBase}, + "Referer": []string{d.appBase + "/"}, + "sec-ch-ua": []string{d.secChUa}, + "sec-ch-ua-platform": []string{d.secChUaPlatform}, + "User-Agent": []string{d.userAgent}, + //"User-Agent": []string{base.UserAgent}, + }, + }, nil +} + +func (d *Mediafire) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "parent_key": parentDir.GetID(), + "foldername": dirName, + } + + var resp MediafireFolderCreateResponse + _, err := d.postForm("/folder/create.php", data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + created, _ := time.Parse("2006-01-02T15:04:05Z", resp.Response.CreatedUTC) + + return &model.ObjThumb{ + Object: model.Object{ + ID: resp.Response.FolderKey, + Name: resp.Response.Name, + Size: 0, + Modified: created, + Ctime: created, + IsFolder: true, + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/move.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key_src": srcObj.GetID(), + "folder_key_dst": dstDir.GetID(), + } + } else { + + endpoint = "/file/move.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "folder_key": dstDir.GetID(), + } + } + + var resp MediafireMoveResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return srcObj, nil +} + +func (d *Mediafire) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/update.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": srcObj.GetID(), + "foldername": newName, + } + } else { + + endpoint = "/file/update.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "filename": newName, + } + } + + var resp MediafireRenameResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/copy.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key_src": srcObj.GetID(), + "folder_key_dst": dstDir.GetID(), + } + } else { + + endpoint = "/file/copy.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "folder_key": dstDir.GetID(), + } + } + + var resp MediafireCopyResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + var newID string + if srcObj.IsDir() { + if len(resp.Response.NewFolderKeys) > 0 { + newID = resp.Response.NewFolderKeys[0] + } + } else { + if len(resp.Response.NewQuickKeys) > 0 { + newID = resp.Response.NewQuickKeys[0] + } + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: newID, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Remove(ctx context.Context, obj model.Obj) error { + var data map[string]string + var endpoint string + + if obj.IsDir() { + + endpoint = "/folder/delete.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": obj.GetID(), + } + } else { + + endpoint = "/file/delete.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": obj.GetID(), + } + } + + var resp MediafireRemoveResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return err + } + + if resp.Response.Result != "Success" { + return fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return nil +} + +func (d *Mediafire) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + _, err := d.PutResult(ctx, dstDir, file, up) + return err +} + +func (d *Mediafire) PutResult(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return nil, err + } + defer tempFile.Close() + + osFile, ok := tempFile.(*os.File) + if !ok { + return nil, fmt.Errorf("expected *os.File, got %T", tempFile) + } + + fileHash, err := d.calculateSHA256(osFile) + if err != nil { + return nil, err + } + + checkResp, err := d.uploadCheck(ctx, file.GetName(), file.GetSize(), fileHash, dstDir.GetID()) + if err != nil { + return nil, err + } + + if checkResp.Response.ResumableUpload.AllUnitsReady == "yes" { + up(100.0) + } + + if checkResp.Response.HashExists == "yes" && checkResp.Response.InAccount == "yes" { + up(100.0) + existingFile, err := d.getExistingFileInfo(ctx, fileHash, file.GetName(), dstDir.GetID()) + if err == nil { + return existingFile, nil + } + } + + var pollKey string + + if checkResp.Response.ResumableUpload.AllUnitsReady != "yes" { + + var err error + + pollKey, err = d.uploadUnits(ctx, osFile, checkResp, file.GetName(), fileHash, dstDir.GetID(), up) + if err != nil { + return nil, err + } + } else { + + pollKey = checkResp.Response.ResumableUpload.UploadKey + } + + //fmt.Printf("pollKey: %+v\n", pollKey) + + pollResp, err := d.pollUpload(ctx, pollKey) + if err != nil { + return nil, err + } + + quickKey := pollResp.Response.Doupload.QuickKey + + return &model.ObjThumb{ + Object: model.Object{ + ID: quickKey, + Name: file.GetName(), + Size: file.GetSize(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *Mediafire) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Mediafire)(nil) diff --git a/drivers/mediafire/meta.go b/drivers/mediafire/meta.go new file mode 100644 index 00000000000..243d55570af --- /dev/null +++ b/drivers/mediafire/meta.go @@ -0,0 +1,54 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + //driver.RootID + + SessionToken string `json:"session_token" required:"true" type:"string" help:"Required for MediaFire API"` + Cookie string `json:"cookie" required:"true" type:"string" help:"Required for navigation"` + + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"100"` +} + +var config = driver.Config{ + Name: "MediaFire", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Mediafire{ + appBase: "https://app.mediafire.com", + apiBase: "https://www.mediafire.com/api/1.5", + hostBase: "https://www.mediafire.com", + maxRetries: 3, + secChUa: "\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"139\", \"Google Chrome\";v=\"139\"", + secChUaPlatform: "Windows", + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36", + } + }) +} diff --git a/drivers/mediafire/types.go b/drivers/mediafire/types.go new file mode 100644 index 00000000000..0073b58179c --- /dev/null +++ b/drivers/mediafire/types.go @@ -0,0 +1,232 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +type MediafireRenewTokenResponse struct { + Response struct { + Action string `json:"action"` + SessionToken string `json:"session_token"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireResponse struct { + Response struct { + Action string `json:"action"` + FolderContent struct { + ChunkSize string `json:"chunk_size"` + ContentType string `json:"content_type"` + ChunkNumber string `json:"chunk_number"` + FolderKey string `json:"folderkey"` + Folders []MediafireFolder `json:"folders,omitempty"` + Files []MediafireFile `json:"files,omitempty"` + MoreChunks string `json:"more_chunks"` + } `json:"folder_content"` + Result string `json:"result"` + } `json:"response"` +} + +type MediafireFolder struct { + FolderKey string `json:"folderkey"` + Name string `json:"name"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` +} + +type MediafireFile struct { + QuickKey string `json:"quickkey"` + Filename string `json:"filename"` + Size string `json:"size"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + MimeType string `json:"mimetype"` +} + +type File struct { + ID string + Name string + Size int64 + CreatedUTC string + IsFolder bool +} + +type FolderContentResponse struct { + Folders []MediafireFolder + Files []MediafireFile + MoreChunks bool +} + +type MediafireLinksResponse struct { + Response struct { + Action string `json:"action"` + Links []struct { + QuickKey string `json:"quickkey"` + View string `json:"view"` + NormalDownload string `json:"normal_download"` + OneTime struct { + Download string `json:"download"` + View string `json:"view"` + } `json:"one_time"` + } `json:"links"` + OneTimeKeyRequestCount string `json:"one_time_key_request_count"` + OneTimeKeyRequestMaxCount string `json:"one_time_key_request_max_count"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireDirectDownloadResponse struct { + Response struct { + Action string `json:"action"` + Links []struct { + QuickKey string `json:"quickkey"` + DirectDownload string `json:"direct_download"` + } `json:"links"` + DirectDownloadFreeBandwidth string `json:"direct_download_free_bandwidth"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireFolderCreateResponse struct { + Response struct { + Action string `json:"action"` + FolderKey string `json:"folder_key"` + UploadKey string `json:"upload_key"` + ParentFolderKey string `json:"parent_folderkey"` + Name string `json:"name"` + Description string `json:"description"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + Privacy string `json:"privacy"` + FileCount string `json:"file_count"` + FolderCount string `json:"folder_count"` + Revision string `json:"revision"` + DropboxEnabled string `json:"dropbox_enabled"` + Flag string `json:"flag"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireMoveResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + NewNames []string `json:"new_names"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireRenameResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireCopyResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + NewQuickKeys []string `json:"new_quickkeys,omitempty"` + NewFolderKeys []string `json:"new_folderkeys,omitempty"` + SkippedCount string `json:"skipped_count,omitempty"` + OtherCount string `json:"other_count,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireRemoveResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireCheckResponse struct { + Response struct { + Action string `json:"action"` + HashExists string `json:"hash_exists"` + InAccount string `json:"in_account"` + InFolder string `json:"in_folder"` + FileExists string `json:"file_exists"` + ResumableUpload struct { + AllUnitsReady string `json:"all_units_ready"` + NumberOfUnits string `json:"number_of_units"` + UnitSize string `json:"unit_size"` + Bitmap struct { + Count string `json:"count"` + Words []string `json:"words"` + } `json:"bitmap"` + UploadKey string `json:"upload_key"` + } `json:"resumable_upload"` + AvailableSpace string `json:"available_space"` + UsedStorageSize string `json:"used_storage_size"` + StorageLimit string `json:"storage_limit"` + StorageLimitExceeded string `json:"storage_limit_exceeded"` + UploadURL struct { + Simple string `json:"simple"` + SimpleFallback string `json:"simple_fallback"` + Resumable string `json:"resumable"` + ResumableFallback string `json:"resumable_fallback"` + } `json:"upload_url"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} +type MediafireActionTokenResponse struct { + Response struct { + Action string `json:"action"` + ActionToken string `json:"action_token"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafirePollResponse struct { + Response struct { + Action string `json:"action"` + Doupload struct { + Result string `json:"result"` + Status string `json:"status"` + Description string `json:"description"` + QuickKey string `json:"quickkey"` + Hash string `json:"hash"` + Filename string `json:"filename"` + Size string `json:"size"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + Revision string `json:"revision"` + } `json:"doupload"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireFileSearchResponse struct { + Response struct { + Action string `json:"action"` + FileInfo []File `json:"file_info"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} diff --git a/drivers/mediafire/util.go b/drivers/mediafire/util.go new file mode 100644 index 00000000000..091abd0cd64 --- /dev/null +++ b/drivers/mediafire/util.go @@ -0,0 +1,626 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { + tokenURL := d.hostBase + "/application/get_session_token.php" + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, tokenURL, nil) + if err != nil { + return "", err + } + + req.Header.Set("Accept", "*/*") + req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Content-Length", "0") + req.Header.Set("Cookie", d.Cookie) + req.Header.Set("DNT", "1") + req.Header.Set("Origin", d.hostBase) + req.Header.Set("Priority", "u=1, i") + req.Header.Set("Referer", (d.hostBase + "/")) + req.Header.Set("Sec-Ch-Ua", d.secChUa) + req.Header.Set("Sec-Ch-Ua-Mobile", "?0") + req.Header.Set("Sec-Ch-Ua-Platform", d.secChUaPlatform) + req.Header.Set("Sec-Fetch-Dest", "empty") + req.Header.Set("Sec-Fetch-Mode", "cors") + req.Header.Set("Sec-Fetch-Site", "same-site") + req.Header.Set("User-Agent", d.userAgent) + //req.Header.Set("Connection", "keep-alive") + + resp, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + //fmt.Printf("getSessionToken :: Raw response: %s\n", string(body)) + //fmt.Printf("getSessionToken :: Parsed response: %+v\n", resp) + + var tokenResp struct { + Response struct { + SessionToken string `json:"session_token"` + } `json:"response"` + } + + if resp.StatusCode == 200 { + if err := json.Unmarshal(body, &tokenResp); err != nil { + return "", err + } + + if tokenResp.Response.SessionToken == "" { + return "", fmt.Errorf("empty session token received") + } + + cookieMap := make(map[string]string) + for _, cookie := range resp.Cookies() { + cookieMap[cookie.Name] = cookie.Value + } + + if len(cookieMap) > 0 { + + var cookies []string + for name, value := range cookieMap { + cookies = append(cookies, fmt.Sprintf("%s=%s", name, value)) + } + d.Cookie = strings.Join(cookies, "; ") + op.MustSaveDriverStorage(d) + + //fmt.Printf("getSessionToken :: Captured cookies: %s\n", d.Cookie) + } + + } else { + return "", fmt.Errorf("getSessionToken :: failed to get session token, status code: %d", resp.StatusCode) + } + + d.SessionToken = tokenResp.Response.SessionToken + + //fmt.Printf("Init :: Obtain Session Token %v", d.SessionToken) + + op.MustSaveDriverStorage(d) + + return d.SessionToken, nil +} + +func (d *Mediafire) renewToken(_ context.Context) error { + query := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + } + + var resp MediafireRenewTokenResponse + _, err := d.postForm("/user/renew_session_token.php", query, &resp) + if err != nil { + return fmt.Errorf("failed to renew token: %w", err) + } + + //fmt.Printf("getInfo :: Raw response: %s\n", string(body)) + //fmt.Printf("getInfo :: Parsed response: %+v\n", resp) + + if resp.Response.Result != "Success" { + return fmt.Errorf("MediaFire token renewal failed: %s", resp.Response.Result) + } + + d.SessionToken = resp.Response.SessionToken + + //fmt.Printf("Init :: Renew Session Token: %s", resp.Response.Result) + + op.MustSaveDriverStorage(d) + + return nil +} + +func (d *Mediafire) getFiles(ctx context.Context, folderKey string) ([]File, error) { + files := make([]File, 0) + hasMore := true + chunkNumber := 1 + + for hasMore { + resp, err := d.getFolderContent(ctx, folderKey, chunkNumber) + if err != nil { + return nil, err + } + + for _, folder := range resp.Folders { + files = append(files, File{ + ID: folder.FolderKey, + Name: folder.Name, + Size: 0, + CreatedUTC: folder.CreatedUTC, + IsFolder: true, + }) + } + + for _, file := range resp.Files { + size, _ := strconv.ParseInt(file.Size, 10, 64) + files = append(files, File{ + ID: file.QuickKey, + Name: file.Filename, + Size: size, + CreatedUTC: file.CreatedUTC, + IsFolder: false, + }) + } + + hasMore = resp.MoreChunks + chunkNumber++ + } + + return files, nil +} + +func (d *Mediafire) getFolderContent(ctx context.Context, folderKey string, chunkNumber int) (*FolderContentResponse, error) { + + foldersResp, err := d.getFolderContentByType(ctx, folderKey, "folders", chunkNumber) + if err != nil { + return nil, err + } + + filesResp, err := d.getFolderContentByType(ctx, folderKey, "files", chunkNumber) + if err != nil { + return nil, err + } + + return &FolderContentResponse{ + Folders: foldersResp.Response.FolderContent.Folders, + Files: filesResp.Response.FolderContent.Files, + MoreChunks: foldersResp.Response.FolderContent.MoreChunks == "yes" || filesResp.Response.FolderContent.MoreChunks == "yes", + }, nil +} + +func (d *Mediafire) getFolderContentByType(_ context.Context, folderKey, contentType string, chunkNumber int) (*MediafireResponse, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": folderKey, + "content_type": contentType, + "chunk": strconv.Itoa(chunkNumber), + "chunk_size": strconv.FormatInt(d.ChunkSize, 10), + "details": "yes", + "order_direction": d.OrderDirection, + "order_by": d.OrderBy, + "filter": "", + } + + var resp MediafireResponse + _, err := d.postForm("/folder/get_content.php", data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) fileToObj(f File) *model.ObjThumb { + created, _ := time.Parse("2006-01-02T15:04:05Z", f.CreatedUTC) + + var thumbnailURL string + if !f.IsFolder && f.ID != "" { + thumbnailURL = d.hostBase + "/convkey/acaa/" + f.ID + "3g.jpg" + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: f.ID, + //Path: "", + Name: f.Name, + Size: f.Size, + Modified: created, + Ctime: created, + IsFolder: f.IsFolder, + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumbnailURL, + }, + } +} + +func (d *Mediafire) getForm(endpoint string, query map[string]string, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + + req.SetQueryParams(query) + + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + //"User-Agent": base.UserAgent, + "User-Agent": d.userAgent, + "Origin": d.appBase, + "Referer": d.appBase + "/", + }) + + // If response OK + if resp != nil { + req.SetResult(resp) + } + + // Targets MediaFire API + res, err := req.Get(d.apiBase + endpoint) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (d *Mediafire) postForm(endpoint string, data map[string]string, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + + req.SetFormData(data) + + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + "Content-Type": "application/x-www-form-urlencoded", + //"User-Agent": base.UserAgent, + "User-Agent": d.userAgent, + "Origin": d.appBase, + "Referer": d.appBase + "/", + }) + + // If response OK + if resp != nil { + req.SetResult(resp) + } + + // Targets MediaFire API + res, err := req.Post(d.apiBase + endpoint) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (d *Mediafire) getDirectDownloadLink(_ context.Context, fileID string) (string, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "quick_key": fileID, + "link_type": "direct_download", + "response_format": "json", + } + + var resp MediafireDirectDownloadResponse + _, err := d.getForm("/file/get_links.php", data, &resp) + if err != nil { + return "", err + } + + if resp.Response.Result != "Success" { + return "", fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + if len(resp.Response.Links) == 0 { + return "", fmt.Errorf("no download links found") + } + + return resp.Response.Links[0].DirectDownload, nil +} + +func (d *Mediafire) calculateSHA256(file *os.File) (string, error) { + hasher := sha256.New() + if _, err := file.Seek(0, 0); err != nil { + return "", err + } + if _, err := io.Copy(hasher, file); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func (d *Mediafire) uploadCheck(ctx context.Context, filename string, filesize int64, filehash, folderKey string) (*MediafireCheckResponse, error) { + + actionToken, err := d.getActionToken(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get action token: %w", err) + } + + query := map[string]string{ + "session_token": actionToken, /* d.SessionToken */ + "filename": filename, + "size": strconv.FormatInt(filesize, 10), + "hash": filehash, + "folder_key": folderKey, + "resumable": "yes", + "response_format": "json", + } + + var resp MediafireCheckResponse + _, err = d.postForm("/upload/check.php", query, &resp) + if err != nil { + return nil, err + } + + //fmt.Printf("uploadCheck :: Raw response: %s\n", string(body)) + //fmt.Printf("uploadCheck :: Parsed response: %+v\n", resp) + + //fmt.Printf("uploadCheck :: ResumableUpload section: %+v\n", resp.Response.ResumableUpload) + //fmt.Printf("uploadCheck :: Upload key specifically: '%s'\n", resp.Response.ResumableUpload.UploadKey) + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire upload check failed: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) resumableUpload(ctx context.Context, folderKey, uploadKey string, unitData []byte, unitID int, fileHash, filename string, totalFileSize int64) (string, error) { + actionToken, err := d.getActionToken(ctx) + if err != nil { + return "", err + } + + url := d.apiBase + "/upload/resumable.php" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(unitData)) + if err != nil { + return "", err + } + + q := req.URL.Query() + q.Add("folder_key", folderKey) + q.Add("response_format", "json") + q.Add("session_token", actionToken) + q.Add("key", uploadKey) + req.URL.RawQuery = q.Encode() + + req.Header.Set("x-filehash", fileHash) + req.Header.Set("x-filesize", strconv.FormatInt(totalFileSize, 10)) + req.Header.Set("x-unit-id", strconv.Itoa(unitID)) + req.Header.Set("x-unit-size", strconv.FormatInt(int64(len(unitData)), 10)) + req.Header.Set("x-unit-hash", d.sha256Hex(bytes.NewReader(unitData))) + req.Header.Set("x-filename", filename) + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = int64(len(unitData)) + + /* fmt.Printf("Debug resumable upload request:\n") + fmt.Printf(" URL: %s\n", req.URL.String()) + fmt.Printf(" Headers: %+v\n", req.Header) + fmt.Printf(" Unit ID: %d\n", unitID) + fmt.Printf(" Unit Size: %d\n", len(unitData)) + fmt.Printf(" Upload Key: %s\n", uploadKey) + fmt.Printf(" Action Token: %s\n", actionToken) */ + + res, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %v", err) + } + + //fmt.Printf("MediaFire resumable upload response (status %d): %s\n", res.StatusCode, string(body)) + + var uploadResp struct { + Response struct { + Doupload struct { + Key string `json:"key"` + } `json:"doupload"` + Result string `json:"result"` + } `json:"response"` + } + + if err := json.Unmarshal(body, &uploadResp); err != nil { + return "", fmt.Errorf("failed to parse response: %v", err) + } + + if res.StatusCode != 200 { + return "", fmt.Errorf("resumable upload failed with status %d", res.StatusCode) + } + + return uploadResp.Response.Doupload.Key, nil +} + +func (d *Mediafire) uploadUnits(ctx context.Context, file *os.File, checkResp *MediafireCheckResponse, filename, fileHash, folderKey string, up driver.UpdateProgress) (string, error) { + unitSize, _ := strconv.ParseInt(checkResp.Response.ResumableUpload.UnitSize, 10, 64) + numUnits, _ := strconv.Atoi(checkResp.Response.ResumableUpload.NumberOfUnits) + uploadKey := checkResp.Response.ResumableUpload.UploadKey + + stringWords := checkResp.Response.ResumableUpload.Bitmap.Words + intWords := make([]int, len(stringWords)) + for i, word := range stringWords { + intWords[i], _ = strconv.Atoi(word) + } + + var finalUploadKey string + + for unitID := 0; unitID < numUnits; unitID++ { + + if utils.IsCanceled(ctx) { + return "", ctx.Err() + } + + if d.isUnitUploaded(intWords, unitID) { + up(float64(unitID+1) * 100 / float64(numUnits)) + continue + } + + uploadKey, err := d.uploadSingleUnit(ctx, file, unitID, unitSize, fileHash, filename, uploadKey, folderKey) + if err != nil { + return "", err + } + + finalUploadKey = uploadKey + + up(float64(unitID+1) * 100 / float64(numUnits)) + } + + return finalUploadKey, nil +} + +func (d *Mediafire) uploadSingleUnit(ctx context.Context, file *os.File, unitID int, unitSize int64, fileHash, filename, uploadKey, folderKey string) (string, error) { + start := int64(unitID) * unitSize + size := unitSize + + stat, err := file.Stat() + if err != nil { + return "", err + } + fileSize := stat.Size() + + if start+size > fileSize { + size = fileSize - start + } + + unitData := make([]byte, size) + if _, err := file.ReadAt(unitData, start); err != nil { + return "", err + } + + return d.resumableUpload(ctx, folderKey, uploadKey, unitData, unitID, fileHash, filename, fileSize) +} + +func (d *Mediafire) getActionToken(_ context.Context) (string, error) { + + if d.actionToken != "" { + return d.actionToken, nil + } + + data := map[string]string{ + "type": "upload", + "lifespan": "1440", + "response_format": "json", + "session_token": d.SessionToken, + } + + var resp MediafireActionTokenResponse + _, err := d.postForm("/user/get_action_token.php", data, &resp) + if err != nil { + return "", err + } + + if resp.Response.Result != "Success" { + return "", fmt.Errorf("MediaFire action token failed: %s", resp.Response.Result) + } + + return resp.Response.ActionToken, nil +} + +func (d *Mediafire) pollUpload(ctx context.Context, key string) (*MediafirePollResponse, error) { + + actionToken, err := d.getActionToken(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get action token: %w", err) + } + + //fmt.Printf("Debug Key: %+v\n", key) + + query := map[string]string{ + "key": key, + "response_format": "json", + "session_token": actionToken, /* d.SessionToken */ + } + + var resp MediafirePollResponse + _, err = d.postForm("/upload/poll_upload.php", query, &resp) + if err != nil { + return nil, err + } + + //fmt.Printf("pollUpload :: Raw response: %s\n", string(body)) + //fmt.Printf("pollUpload :: Parsed response: %+v\n", resp) + + //fmt.Printf("pollUpload :: Debug Result: %+v\n", resp.Response.Result) + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire poll upload failed: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) sha256Hex(r io.Reader) string { + h := sha256.New() + io.Copy(h, r) + return hex.EncodeToString(h.Sum(nil)) +} + +func (d *Mediafire) isUnitUploaded(words []int, unitID int) bool { + wordIndex := unitID / 16 + bitIndex := unitID % 16 + if wordIndex >= len(words) { + return false + } + return (words[wordIndex]>>bitIndex)&1 == 1 +} + +func (d *Mediafire) getExistingFileInfo(ctx context.Context, fileHash, filename, folderKey string) (*model.ObjThumb, error) { + + if fileInfo, err := d.getFileByHash(ctx, fileHash); err == nil && fileInfo != nil { + return fileInfo, nil + } + + files, err := d.getFiles(ctx, folderKey) + if err != nil { + return nil, err + } + + for _, file := range files { + if file.Name == filename && !file.IsFolder { + return d.fileToObj(file), nil + } + } + + return nil, fmt.Errorf("existing file not found") +} + +func (d *Mediafire) getFileByHash(_ context.Context, hash string) (*model.ObjThumb, error) { + query := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "hash": hash, + } + + var resp MediafireFileSearchResponse + _, err := d.postForm("/file/get_info.php", query, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire file search failed: %s", resp.Response.Result) + } + + if len(resp.Response.FileInfo) == 0 { + return nil, fmt.Errorf("file not found by hash") + } + + file := resp.Response.FileInfo[0] + return d.fileToObj(file), nil +} diff --git a/drivers/mediatrack/meta.go b/drivers/mediatrack/meta.go index 47f112c3573..ade8ae1c8fe 100644 --- a/drivers/mediatrack/meta.go +++ b/drivers/mediatrack/meta.go @@ -9,8 +9,9 @@ type Addition struct { AccessToken string `json:"access_token" required:"true"` ProjectID string `json:"project_id"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"updated_at,title,size" default:"title"` - OrderDesc bool `json:"order_desc"` + OrderBy string `json:"order_by" type:"select" options:"updated_at,title,size" default:"title"` + OrderDesc bool `json:"order_desc"` + DeviceFingerprint string `json:"device_fingerprint" required:"true"` } var config = driver.Config{ diff --git a/drivers/mediatrack/util.go b/drivers/mediatrack/util.go index 37ca0b3d09c..f5b751111a1 100644 --- a/drivers/mediatrack/util.go +++ b/drivers/mediatrack/util.go @@ -17,6 +17,9 @@ import ( func (d *MediaTrack) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) + if d.DeviceFingerprint != "" { + req.SetHeader("X-Device-Fingerprint", d.DeviceFingerprint) + } if callback != nil { callback(req) } diff --git a/drivers/misskey/util.go b/drivers/misskey/util.go index f8baeafa6cb..d301955ec42 100644 --- a/drivers/misskey/util.go +++ b/drivers/misskey/util.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io" + "net/http" "time" "github.com/go-resty/resty/v2" @@ -56,23 +57,27 @@ func setBody(body interface{}) base.ReqCallback { } func handleFolderId(dir model.Obj) interface{} { - if dir.GetID() == "" { - return nil + if isRootFolder(dir) { + return nil // Root folder doesn't need folderId } return dir.GetID() } +func isRootFolder(dir model.Obj) bool { + return dir.GetID() == "" +} + // API layer methods func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { var files []MFile var body map[string]string - if dir.GetPath() != "/" { + if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} } - err := d.request("/files", "POST", setBody(body), &files) + err := d.request("/files", http.MethodPost, setBody(body), &files) if err != nil { return []model.Obj{}, err } @@ -84,12 +89,12 @@ func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) { var folders []MFolder var body map[string]string - if dir.GetPath() != "/" { + if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} } - err := d.request("/folders", "POST", setBody(body), &folders) + err := d.request("/folders", http.MethodPost, setBody(body), &folders) if err != nil { return []model.Obj{}, err } @@ -106,7 +111,7 @@ func (d *Misskey) list(dir model.Obj) ([]model.Obj, error) { func (d *Misskey) link(file model.Obj) (*model.Link, error) { var mFile MFile - err := d.request("/files/show", "POST", setBody(map[string]string{"fileId": file.GetID()}), &mFile) + err := d.request("/files/show", http.MethodPost, setBody(map[string]string{"fileId": file.GetID()}), &mFile) if err != nil { return nil, err } @@ -117,7 +122,7 @@ func (d *Misskey) link(file model.Obj) (*model.Link, error) { func (d *Misskey) makeDir(parentDir model.Obj, dirName string) (model.Obj, error) { var folder MFolder - err := d.request("/folders/create", "POST", setBody(map[string]interface{}{"parentId": handleFolderId(parentDir), "name": dirName}), &folder) + err := d.request("/folders/create", http.MethodPost, setBody(map[string]interface{}{"parentId": handleFolderId(parentDir), "name": dirName}), &folder) if err != nil { return nil, err } @@ -127,11 +132,11 @@ func (d *Misskey) makeDir(parentDir model.Obj, dirName string) (model.Obj, error func (d *Misskey) move(srcObj, dstDir model.Obj) (model.Obj, error) { if srcObj.IsDir() { var folder MFolder - err := d.request("/folders/update", "POST", setBody(map[string]interface{}{"folderId": srcObj.GetID(), "parentId": handleFolderId(dstDir)}), &folder) + err := d.request("/folders/update", http.MethodPost, setBody(map[string]interface{}{"folderId": srcObj.GetID(), "parentId": handleFolderId(dstDir)}), &folder) return mFolder2Object(folder), err } else { var file MFile - err := d.request("/files/update", "POST", setBody(map[string]interface{}{"fileId": srcObj.GetID(), "folderId": handleFolderId(dstDir)}), &file) + err := d.request("/files/update", http.MethodPost, setBody(map[string]interface{}{"fileId": srcObj.GetID(), "folderId": handleFolderId(dstDir)}), &file) return mFile2Object(file), err } } @@ -139,11 +144,11 @@ func (d *Misskey) move(srcObj, dstDir model.Obj) (model.Obj, error) { func (d *Misskey) rename(srcObj model.Obj, newName string) (model.Obj, error) { if srcObj.IsDir() { var folder MFolder - err := d.request("/folders/update", "POST", setBody(map[string]string{"folderId": srcObj.GetID(), "name": newName}), &folder) + err := d.request("/folders/update", http.MethodPost, setBody(map[string]string{"folderId": srcObj.GetID(), "name": newName}), &folder) return mFolder2Object(folder), err } else { var file MFile - err := d.request("/files/update", "POST", setBody(map[string]string{"fileId": srcObj.GetID(), "name": newName}), &file) + err := d.request("/files/update", http.MethodPost, setBody(map[string]string{"fileId": srcObj.GetID(), "name": newName}), &file) return mFile2Object(file), err } } @@ -171,7 +176,7 @@ func (d *Misskey) copy(srcObj, dstDir model.Obj) (model.Obj, error) { if err != nil { return nil, err } - err = d.request("/files/upload-from-url", "POST", setBody(map[string]interface{}{"url": url.URL, "folderId": handleFolderId(dstDir)}), &file) + err = d.request("/files/upload-from-url", http.MethodPost, setBody(map[string]interface{}{"url": url.URL, "folderId": handleFolderId(dstDir)}), &file) if err != nil { return nil, err } @@ -181,10 +186,10 @@ func (d *Misskey) copy(srcObj, dstDir model.Obj) (model.Obj, error) { func (d *Misskey) remove(obj model.Obj) error { if obj.IsDir() { - err := d.request("/folders/delete", "POST", setBody(map[string]string{"folderId": obj.GetID()}), nil) + err := d.request("/folders/delete", http.MethodPost, setBody(map[string]string{"folderId": obj.GetID()}), nil) return err } else { - err := d.request("/files/delete", "POST", setBody(map[string]string{"fileId": obj.GetID()}), nil) + err := d.request("/files/delete", http.MethodPost, setBody(map[string]string{"fileId": obj.GetID()}), nil) return err } } @@ -196,16 +201,24 @@ func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileSt Reader: stream, UpdateProgress: up, }) + + // Build form data, only add folderId if not root folder + formData := map[string]string{ + "name": stream.GetName(), + "comment": "", + "isSensitive": "false", + "force": "false", + } + + folderId := handleFolderId(dstDir) + if folderId != nil { + formData["folderId"] = folderId.(string) + } + req := base.RestyClient.R(). SetContext(ctx). SetFileReader("file", stream.GetName(), reader). - SetFormData(map[string]string{ - "folderId": handleFolderId(dstDir).(string), - "name": stream.GetName(), - "comment": "", - "isSensitive": "false", - "force": "false", - }). + SetFormData(formData). SetResult(&file). SetAuthToken(d.AccessToken) diff --git a/drivers/onedrive/meta.go b/drivers/onedrive/meta.go index a60e5f33a93..54a7340a942 100644 --- a/drivers/onedrive/meta.go +++ b/drivers/onedrive/meta.go @@ -11,7 +11,7 @@ type Addition struct { IsSharepoint bool `json:"is_sharepoint"` ClientID string `json:"client_id" required:"true"` ClientSecret string `json:"client_secret" required:"true"` - RedirectUri string `json:"redirect_uri" required:"true" default:"https://alist.nn.ci/tool/onedrive/callback"` + RedirectUri string `json:"redirect_uri" required:"true" default:"https://alistgo.com/tool/onedrive/callback"` RefreshToken string `json:"refresh_token" required:"true"` SiteId string `json:"site_id"` ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index e256b7ae262..28ed5ccc3cc 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -8,6 +8,7 @@ import ( "io" "net/http" stdpath "path" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -17,7 +18,6 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" - log "github.com/sirupsen/logrus" ) var onedriveHostMap = map[string]Host{ @@ -204,19 +204,18 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() var finish int64 = 0 DEFAULT := d.ChunkSize * 1024 * 1024 + retryCount := 0 + maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } - log.Debugf("upload: %d", finish) - var byteSize int64 = DEFAULT left := stream.GetSize() - finish - if left < DEFAULT { - byteSize = left - } + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Onedrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) - log.Debug(err, n) + utils.Log.Debug(err, n) if err != nil { return err } @@ -228,19 +227,31 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil req.ContentLength = byteSize // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) - finish += byteSize res, err := base.HttpClient.Do(req) if err != nil { return err } // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - if res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200 { + switch { + case res.StatusCode >= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< 1 && (id[0] == 'd' || id[0] == 'f') { + return id[1:] + } + return id +} + +// Get folder ID from path, return "0" for root +func getFolderID(path string) string { + if path == "/" || path == "" { + return "0" + } + return extractID(path) +} \ No newline at end of file diff --git a/drivers/pcloud/util.go b/drivers/pcloud/util.go new file mode 100644 index 00000000000..f2c1875e133 --- /dev/null +++ b/drivers/pcloud/util.go @@ -0,0 +1,297 @@ +package pcloud + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + defaultClientID = "DnONSzyJXpm" + defaultClientSecret = "VKEnd3ze4jsKFGg8TJiznwFG8" +) + +// Get API base URL +func (d *PCloud) getAPIURL() string { + return "https://" + d.Hostname +} + +// Get OAuth client credentials +func (d *PCloud) getClientCredentials() (string, string) { + clientID := d.ClientID + clientSecret := d.ClientSecret + + if clientID == "" { + clientID = defaultClientID + } + if clientSecret == "" { + clientSecret = defaultClientSecret + } + + return clientID, clientSecret +} + +// Refresh OAuth access token +func (d *PCloud) refreshToken() error { + clientID, clientSecret := d.getClientCredentials() + + var resp TokenResponse + _, err := base.RestyClient.R(). + SetFormData(map[string]string{ + "client_id": clientID, + "client_secret": clientSecret, + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + }). + SetResult(&resp). + Post(d.getAPIURL() + "/oauth2_token") + + if err != nil { + return err + } + + d.AccessToken = resp.AccessToken + return nil +} + +// shouldRetry determines if an error should be retried based on pCloud-specific logic +func (d *PCloud) shouldRetry(statusCode int, apiError *ErrorResult) bool { + // HTTP-level retry conditions + if statusCode == 429 || statusCode >= 500 { + return true + } + + // pCloud API-specific retry conditions (like rclone) + if apiError != nil && apiError.Result != 0 { + // 4xxx: rate limiting + if apiError.Result/1000 == 4 { + return true + } + // 5xxx: internal errors + if apiError.Result/1000 == 5 { + return true + } + } + + return false +} + +// requestWithRetry makes authenticated API request with retry logic +func (d *PCloud) requestWithRetry(endpoint string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + maxRetries := 3 + baseDelay := 500 * time.Millisecond + + for attempt := 0; attempt <= maxRetries; attempt++ { + body, err := d.request(endpoint, method, callback, resp) + if err == nil { + return body, nil + } + + // If this is the last attempt, return the error + if attempt == maxRetries { + return nil, err + } + + // Check if we should retry based on error type + if !d.shouldRetryError(err) { + return nil, err + } + + // Exponential backoff + delay := baseDelay * time.Duration(1< +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "sync" + "time" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + proton_api_bridge "github.com/henrybear327/Proton-API-Bridge" + "github.com/henrybear327/Proton-API-Bridge/common" + "github.com/henrybear327/go-proton-api" +) + +type ProtonDrive struct { + model.Storage + Addition + + protonDrive *proton_api_bridge.ProtonDrive + credentials *common.ProtonDriveCredential + + apiBase string + appVersion string + protonJson string + userAgent string + sdkVersion string + webDriveAV string + + tempServer *http.Server + tempServerPort int + downloadTokens map[string]*downloadInfo + tokenMutex sync.RWMutex + + c *proton.Client + //m *proton.Manager + + credentialCacheFile string + + //userKR *crypto.KeyRing + addrKRs map[string]*crypto.KeyRing + addrData map[string]proton.Address + + MainShare *proton.Share + RootLink *proton.Link + + DefaultAddrKR *crypto.KeyRing + MainShareKR *crypto.KeyRing +} + +func (d *ProtonDrive) Config() driver.Config { + return config +} + +func (d *ProtonDrive) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ProtonDrive) Init(ctx context.Context) error { + + defer func() { + if r := recover(); r != nil { + fmt.Printf("ProtonDrive initialization panic: %v", r) + } + }() + + if d.Username == "" { + return fmt.Errorf("username is required") + } + if d.Password == "" { + return fmt.Errorf("password is required") + } + + //fmt.Printf("ProtonDrive Init: Username=%s, TwoFACode=%s", d.Username, d.TwoFACode) + + if ctx == nil { + return fmt.Errorf("context cannot be nil") + } + + cachedCredentials, err := d.loadCachedCredentials() + useReusableLogin := false + var reusableCredential *common.ReusableCredentialData + + if err == nil && cachedCredentials != nil && + cachedCredentials.UID != "" && cachedCredentials.AccessToken != "" && + cachedCredentials.RefreshToken != "" && cachedCredentials.SaltedKeyPass != "" { + useReusableLogin = true + reusableCredential = cachedCredentials + } else { + useReusableLogin = false + reusableCredential = &common.ReusableCredentialData{} + } + + config := &common.Config{ + AppVersion: d.appVersion, + UserAgent: d.userAgent, + FirstLoginCredential: &common.FirstLoginCredentialData{ + Username: d.Username, + Password: d.Password, + TwoFA: d.TwoFACode, + }, + EnableCaching: true, + ConcurrentBlockUploadCount: 5, + ConcurrentFileCryptoCount: 2, + UseReusableLogin: false, + ReplaceExistingDraft: true, + ReusableCredential: reusableCredential, + CredentialCacheFile: d.credentialCacheFile, + } + + if config.FirstLoginCredential == nil { + return fmt.Errorf("failed to create login credentials, FirstLoginCredential cannot be nil") + } + + //fmt.Printf("Calling NewProtonDrive...") + + protonDrive, credentials, err := proton_api_bridge.NewProtonDrive( + ctx, + config, + func(auth proton.Auth) {}, + func() {}, + ) + + if credentials == nil && !useReusableLogin { + return fmt.Errorf("failed to get credentials from NewProtonDrive") + } + + if err != nil { + return fmt.Errorf("failed to initialize ProtonDrive: %w", err) + } + + d.protonDrive = protonDrive + + var finalCredentials *common.ProtonDriveCredential + + if useReusableLogin { + + // For reusable login, create credentials from cached data + finalCredentials = &common.ProtonDriveCredential{ + UID: reusableCredential.UID, + AccessToken: reusableCredential.AccessToken, + RefreshToken: reusableCredential.RefreshToken, + SaltedKeyPass: reusableCredential.SaltedKeyPass, + } + + d.credentials = finalCredentials + } else { + d.credentials = credentials + } + + clientOptions := []proton.Option{ + proton.WithAppVersion(d.appVersion), + proton.WithUserAgent(d.userAgent), + } + manager := proton.New(clientOptions...) + d.c = manager.NewClient(d.credentials.UID, d.credentials.AccessToken, d.credentials.RefreshToken) + + saltedKeyPassBytes, err := base64.StdEncoding.DecodeString(d.credentials.SaltedKeyPass) + if err != nil { + return fmt.Errorf("failed to decode salted key pass: %w", err) + } + + _, addrKRs, addrs, _, err := getAccountKRs(ctx, d.c, nil, saltedKeyPassBytes) + if err != nil { + return fmt.Errorf("failed to get account keyrings: %w", err) + } + + d.MainShare = protonDrive.MainShare + d.RootLink = protonDrive.RootLink + d.MainShareKR = protonDrive.MainShareKR + d.DefaultAddrKR = protonDrive.DefaultAddrKR + d.addrKRs = addrKRs + d.addrData = addrs + + return nil +} + +func (d *ProtonDrive) Drop(ctx context.Context) error { + if d.tempServer != nil { + d.tempServer.Shutdown(ctx) + } + return nil +} + +func (d *ProtonDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var linkID string + + if dir.GetPath() == "/" { + linkID = d.protonDrive.RootLink.LinkID + } else { + + link, err := d.searchByPath(ctx, dir.GetPath(), true) + if err != nil { + return nil, err + } + linkID = link.LinkID + } + + entries, err := d.protonDrive.ListDirectory(ctx, linkID) + if err != nil { + return nil, fmt.Errorf("failed to list directory: %w", err) + } + + //fmt.Printf("Found %d entries for path %s\n", len(entries), dir.GetPath()) + //fmt.Printf("Found %d entries\n", len(entries)) + + if len(entries) == 0 { + emptySlice := []model.Obj{} + + //fmt.Printf("Returning empty slice (entries): %+v\n", emptySlice) + + return emptySlice, nil + } + + var objects []model.Obj + for _, entry := range entries { + obj := &model.Object{ + Name: entry.Name, + Size: entry.Link.Size, + Modified: time.Unix(entry.Link.ModifyTime, 0), + IsFolder: entry.IsFolder, + } + objects = append(objects, obj) + } + + return objects, nil +} + +func (d *ProtonDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + link, err := d.searchByPath(ctx, file.GetPath(), false) + if err != nil { + return nil, err + } + + if err := d.ensureTempServer(); err != nil { + return nil, fmt.Errorf("failed to start temp server: %w", err) + } + + token := d.generateDownloadToken(link.LinkID, file.GetName()) + + /* return &model.Link{ + URL: fmt.Sprintf("protondrive://download/%s", link.LinkID), + }, nil */ + + return &model.Link{ + URL: fmt.Sprintf("http://localhost:%d/temp/%s", d.tempServerPort, token), + }, nil +} + +func (d *ProtonDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var parentLinkID string + + if parentDir.GetPath() == "/" { + parentLinkID = d.protonDrive.RootLink.LinkID + } else { + link, err := d.searchByPath(ctx, parentDir.GetPath(), true) + if err != nil { + return nil, err + } + parentLinkID = link.LinkID + } + + _, err := d.protonDrive.CreateNewFolderByID(ctx, parentLinkID, dirName) + if err != nil { + return nil, fmt.Errorf("failed to create directory: %w", err) + } + + newDir := &model.Object{ + Name: dirName, + IsFolder: true, + Modified: time.Now(), + } + return newDir, nil +} + +func (d *ProtonDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.DirectMove(ctx, srcObj, dstDir) +} + +func (d *ProtonDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + return d.DirectRename(ctx, srcObj, newName) +} + +func (d *ProtonDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, fmt.Errorf("directory copy not supported") + } + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), false) + if err != nil { + return nil, err + } + + reader, linkSize, fileSystemAttrs, err := d.protonDrive.DownloadFile(ctx, srcLink, 0) + if err != nil { + return nil, fmt.Errorf("failed to download source file: %w", err) + } + defer reader.Close() + + actualSize := linkSize + if fileSystemAttrs != nil && fileSystemAttrs.Size > 0 { + actualSize = fileSystemAttrs.Size + } + + tempFile, err := utils.CreateTempFile(reader, actualSize) + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + defer tempFile.Close() + + updatedObj := &model.Object{ + Name: srcObj.GetName(), + // Use the accurate and real size + Size: actualSize, + Modified: srcObj.ModTime(), + IsFolder: false, + } + + return d.Put(ctx, dstDir, &fileStreamer{ + ReadCloser: tempFile, + obj: updatedObj, + }, nil) +} + +func (d *ProtonDrive) Remove(ctx context.Context, obj model.Obj) error { + link, err := d.searchByPath(ctx, obj.GetPath(), obj.IsDir()) + if err != nil { + return err + } + + if obj.IsDir() { + return d.protonDrive.MoveFolderToTrashByID(ctx, link.LinkID, false) + } else { + return d.protonDrive.MoveFileToTrashByID(ctx, link.LinkID) + } +} + +func (d *ProtonDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var parentLinkID string + + if dstDir.GetPath() == "/" { + parentLinkID = d.protonDrive.RootLink.LinkID + } else { + link, err := d.searchByPath(ctx, dstDir.GetPath(), true) + if err != nil { + return nil, err + } + parentLinkID = link.LinkID + } + + tempFile, err := utils.CreateTempFile(file, file.GetSize()) + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + defer tempFile.Close() + + err = d.uploadFile(ctx, parentLinkID, file.GetName(), tempFile, file.GetSize(), up) + if err != nil { + return nil, err + } + + uploadedObj := &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + Modified: file.ModTime(), + IsFolder: false, + } + return uploadedObj, nil +} + +func (d *ProtonDrive) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +var _ driver.Driver = (*ProtonDrive)(nil) diff --git a/drivers/proton_drive/meta.go b/drivers/proton_drive/meta.go new file mode 100644 index 00000000000..ed33a41a2f8 --- /dev/null +++ b/drivers/proton_drive/meta.go @@ -0,0 +1,69 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + //driver.RootID + + Username string `json:"username" required:"true" type:"string"` + Password string `json:"password" required:"true" type:"string"` + TwoFACode string `json:"two_fa_code,omitempty" type:"string"` +} + +type Config struct { + Name string `json:"name"` + LocalSort bool `json:"local_sort"` + OnlyLocal bool `json:"only_local"` + OnlyProxy bool `json:"only_proxy"` + NoCache bool `json:"no_cache"` + NoUpload bool `json:"no_upload"` + NeedMs bool `json:"need_ms"` + DefaultRoot string `json:"default_root"` +} + +var config = driver.Config{ + Name: "ProtonDrive", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ProtonDrive{ + apiBase: "https://drive.proton.me/api", + appVersion: "windows-drive@1.11.3+rclone+proton", + credentialCacheFile: ".prtcrd", + protonJson: "application/vnd.protonmail.v1+json", + sdkVersion: "js@0.3.0", + userAgent: "ProtonDrive/v1.70.0 (Windows NT 10.0.22000; Win64; x64)", + webDriveAV: "web-drive@5.2.0+0f69f7a8", + } + }) +} diff --git a/drivers/proton_drive/types.go b/drivers/proton_drive/types.go new file mode 100644 index 00000000000..37a9edc6c46 --- /dev/null +++ b/drivers/proton_drive/types.go @@ -0,0 +1,124 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "errors" + "io" + "os" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/henrybear327/go-proton-api" +) + +type ProtonFile struct { + *proton.Link + Name string + IsFolder bool +} + +func (p *ProtonFile) GetName() string { + return p.Name +} + +func (p *ProtonFile) GetSize() int64 { + return p.Link.Size +} + +func (p *ProtonFile) GetPath() string { + return p.Name +} + +func (p *ProtonFile) IsDir() bool { + return p.IsFolder +} + +func (p *ProtonFile) ModTime() time.Time { + return time.Unix(p.Link.ModifyTime, 0) +} + +func (p *ProtonFile) CreateTime() time.Time { + return time.Unix(p.Link.CreateTime, 0) +} + +type downloadInfo struct { + LinkID string + FileName string +} + +type fileStreamer struct { + io.ReadCloser + obj model.Obj +} + +func (fs *fileStreamer) GetMimetype() string { return "" } +func (fs *fileStreamer) NeedStore() bool { return false } +func (fs *fileStreamer) IsForceStreamUpload() bool { return false } +func (fs *fileStreamer) GetExist() model.Obj { return nil } +func (fs *fileStreamer) SetExist(model.Obj) {} +func (fs *fileStreamer) RangeRead(http_range.Range) (io.Reader, error) { + return nil, errors.New("not supported") +} +func (fs *fileStreamer) CacheFullInTempFile() (model.File, error) { + return nil, errors.New("not supported") +} +func (fs *fileStreamer) SetTmpFile(r *os.File) {} +func (fs *fileStreamer) GetFile() model.File { return nil } +func (fs *fileStreamer) GetName() string { return fs.obj.GetName() } +func (fs *fileStreamer) GetSize() int64 { return fs.obj.GetSize() } +func (fs *fileStreamer) GetPath() string { return fs.obj.GetPath() } +func (fs *fileStreamer) IsDir() bool { return fs.obj.IsDir() } +func (fs *fileStreamer) ModTime() time.Time { return fs.obj.ModTime() } +func (fs *fileStreamer) CreateTime() time.Time { return fs.obj.ModTime() } +func (fs *fileStreamer) GetHash() utils.HashInfo { return fs.obj.GetHash() } +func (fs *fileStreamer) GetID() string { return fs.obj.GetID() } + +type httpRange struct { + start, end int64 +} + +type MoveRequest struct { + ParentLinkID string `json:"ParentLinkID"` + NodePassphrase string `json:"NodePassphrase"` + NodePassphraseSignature *string `json:"NodePassphraseSignature"` + Name string `json:"Name"` + NameSignatureEmail string `json:"NameSignatureEmail"` + Hash string `json:"Hash"` + OriginalHash string `json:"OriginalHash"` + ContentHash *string `json:"ContentHash"` // Maybe null +} + +type progressReader struct { + reader io.Reader + total int64 + current int64 + callback driver.UpdateProgress +} + +type RenameRequest struct { + Name string `json:"Name"` // PGP encrypted name + NameSignatureEmail string `json:"NameSignatureEmail"` // User's signature email + Hash string `json:"Hash"` // New name hash + OriginalHash string `json:"OriginalHash"` // Current name hash +} + +type RenameResponse struct { + Code int `json:"Code"` +} diff --git a/drivers/proton_drive/util.go b/drivers/proton_drive/util.go new file mode 100644 index 00000000000..7bd52fcaa33 --- /dev/null +++ b/drivers/proton_drive/util.go @@ -0,0 +1,918 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/henrybear327/Proton-API-Bridge/common" + "github.com/henrybear327/go-proton-api" +) + +func (d *ProtonDrive) loadCachedCredentials() (*common.ReusableCredentialData, error) { + if d.credentialCacheFile == "" { + return nil, nil + } + + if _, err := os.Stat(d.credentialCacheFile); os.IsNotExist(err) { + return nil, nil + } + + data, err := os.ReadFile(d.credentialCacheFile) + if err != nil { + return nil, fmt.Errorf("failed to read credential cache file: %w", err) + } + + var credentials common.ReusableCredentialData + if err := json.Unmarshal(data, &credentials); err != nil { + return nil, fmt.Errorf("failed to parse cached credentials: %w", err) + } + + if credentials.UID == "" || credentials.AccessToken == "" || + credentials.RefreshToken == "" || credentials.SaltedKeyPass == "" { + return nil, fmt.Errorf("cached credentials are incomplete") + } + + return &credentials, nil +} + +func (d *ProtonDrive) searchByPath(ctx context.Context, fullPath string, isFolder bool) (*proton.Link, error) { + if fullPath == "/" { + return d.protonDrive.RootLink, nil + } + + cleanPath := strings.Trim(fullPath, "/") + pathParts := strings.Split(cleanPath, "/") + + currentLink := d.protonDrive.RootLink + + for i, part := range pathParts { + isLastPart := i == len(pathParts)-1 + searchForFolder := !isLastPart || isFolder + + entries, err := d.protonDrive.ListDirectory(ctx, currentLink.LinkID) + if err != nil { + return nil, fmt.Errorf("failed to list directory: %w", err) + + } + + found := false + for _, entry := range entries { + // entry.Name is already decrypted! + if entry.Name == part && entry.IsFolder == searchForFolder { + currentLink = entry.Link + found = true + break + } + } + + if !found { + return nil, fmt.Errorf("path not found: %s (looking for part: %s)", fullPath, part) + } + } + + return currentLink, nil +} + +func (pr *progressReader) Read(p []byte) (int, error) { + n, err := pr.reader.Read(p) + pr.current += int64(n) + + if pr.callback != nil { + percentage := float64(pr.current) / float64(pr.total) * 100 + pr.callback(percentage) + } + + return n, err +} + +func (d *ProtonDrive) uploadFile(ctx context.Context, parentLinkID, fileName string, file *os.File, size int64, up driver.UpdateProgress) error { + + fileInfo, err := file.Stat() + if err != nil { + return fmt.Errorf("failed to get file info: %w", err) + } + + _, err = d.protonDrive.GetLink(ctx, parentLinkID) + if err != nil { + return fmt.Errorf("failed to get parent link: %w", err) + } + + reader := &progressReader{ + reader: bufio.NewReader(file), + total: size, + current: 0, + callback: up, + } + + _, _, err = d.protonDrive.UploadFileByReader(ctx, parentLinkID, fileName, fileInfo.ModTime(), reader, 0) + if err != nil { + return fmt.Errorf("failed to upload file: %w", err) + } + + return nil +} + +func (d *ProtonDrive) ensureTempServer() error { + if d.tempServer != nil { + + // Already running + return nil + } + + listener, err := net.Listen("tcp", ":0") + if err != nil { + return err + } + d.tempServerPort = listener.Addr().(*net.TCPAddr).Port + + mux := http.NewServeMux() + mux.HandleFunc("/temp/", d.handleTempDownload) + + d.tempServer = &http.Server{ + Handler: mux, + } + + go func() { + d.tempServer.Serve(listener) + }() + + return nil +} + +func (d *ProtonDrive) handleTempDownload(w http.ResponseWriter, r *http.Request) { + token := strings.TrimPrefix(r.URL.Path, "/temp/") + + d.tokenMutex.RLock() + info, exists := d.downloadTokens[token] + d.tokenMutex.RUnlock() + + if !exists { + http.Error(w, "Invalid or expired token", http.StatusNotFound) + return + } + + link, err := d.protonDrive.GetLink(r.Context(), info.LinkID) + if err != nil { + http.Error(w, "Failed to get file link", http.StatusInternalServerError) + return + } + + // Get file size for range calculations + _, _, attrs, err := d.protonDrive.DownloadFile(r.Context(), link, 0) + if err != nil { + http.Error(w, "Failed to get file info", http.StatusInternalServerError) + return + } + + fileSize := attrs.Size + + rangeHeader := r.Header.Get("Range") + if rangeHeader != "" { + + // Parse range header like "bytes=0-1023" or "bytes=1024-" + ranges, err := parseRange(rangeHeader, fileSize) + if err != nil { + http.Error(w, "Invalid range", http.StatusRequestedRangeNotSatisfiable) + return + } + + if len(ranges) == 1 { + + // Single range request, small + start, end := ranges[0].start, ranges[0].end + contentLength := end - start + 1 + + // Start download from offset + reader, _, _, err := d.protonDrive.DownloadFile(r.Context(), link, start) + if err != nil { + http.Error(w, "Failed to start download", http.StatusInternalServerError) + return + } + defer reader.Close() + + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, fileSize)) + w.Header().Set("Content-Length", fmt.Sprintf("%d", contentLength)) + w.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(link.Name))) + + // Partial content... + // Setting fileName is more cosmetical here + //.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", link.Name)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", info.FileName)) + w.Header().Set("Accept-Ranges", "bytes") + + w.WriteHeader(http.StatusPartialContent) + + io.CopyN(w, reader, contentLength) + return + } + } + + // Full file download (non-range request) + reader, _, _, err := d.protonDrive.DownloadFile(r.Context(), link, 0) + if err != nil { + http.Error(w, "Failed to start download", http.StatusInternalServerError) + return + } + defer reader.Close() + + // Set headers for full content + w.Header().Set("Content-Length", fmt.Sprintf("%d", fileSize)) + w.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(link.Name))) + + // Setting fileName is needed since ProtonDrive fileName is more like a random string + //w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", link.Name)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", info.FileName)) + + w.Header().Set("Accept-Ranges", "bytes") + + // Stream the full file + io.Copy(w, reader) +} + +func (d *ProtonDrive) generateDownloadToken(linkID, fileName string) string { + token := fmt.Sprintf("%d_%s", time.Now().UnixNano(), linkID[:8]) + + d.tokenMutex.Lock() + if d.downloadTokens == nil { + d.downloadTokens = make(map[string]*downloadInfo) + } + + d.downloadTokens[token] = &downloadInfo{ + LinkID: linkID, + FileName: fileName, + } + + d.tokenMutex.Unlock() + + go func() { + + // Token expires in 1 hour + time.Sleep(1 * time.Hour) + d.tokenMutex.Lock() + + delete(d.downloadTokens, token) + d.tokenMutex.Unlock() + }() + + return token +} + +func parseRange(rangeHeader string, size int64) ([]httpRange, error) { + if !strings.HasPrefix(rangeHeader, "bytes=") { + return nil, fmt.Errorf("invalid range header") + } + + rangeSpec := strings.TrimPrefix(rangeHeader, "bytes=") + ranges := strings.Split(rangeSpec, ",") + + var result []httpRange + for _, r := range ranges { + r = strings.TrimSpace(r) + if strings.Contains(r, "-") { + parts := strings.Split(r, "-") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid range format") + } + + var start, end int64 + var err error + + if parts[0] == "" { + + // Suffix range (e.g., "-500") + if parts[1] == "" { + return nil, fmt.Errorf("invalid range format") + } + end = size - 1 + start, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, err + } + start = size - start + if start < 0 { + start = 0 + } + } else if parts[1] == "" { + + // Prefix range (e.g., "500-") + start, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, err + } + end = size - 1 + } else { + // Full range (e.g., "0-1023") + start, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, err + } + end, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, err + } + } + + if start >= size || end >= size || start > end { + return nil, fmt.Errorf("range out of bounds") + } + + result = append(result, httpRange{start: start, end: end}) + } + } + + return result, nil +} + +func (d *ProtonDrive) encryptFileName(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Temporary file (request) + tempReq := proton.CreateFileReq{ + SignatureAddress: d.MainShare.Creator, + } + + // Encrypt the filename + err = tempReq.SetName(name, d.DefaultAddrKR, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to encrypt filename: %w", err) + } + + return tempReq.Name, nil +} + +func (d *ProtonDrive) generateFileNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func (d *ProtonDrive) getOriginalNameHash(link *proton.Link) (string, error) { + if link == nil { + return "", fmt.Errorf("link cannot be nil") + } + + if link.Hash == "" { + return "", fmt.Errorf("link hash is empty") + } + + return link.Hash, nil +} + +func (d *ProtonDrive) getLink(ctx context.Context, linkID string) (*proton.Link, error) { + if linkID == "" { + return nil, fmt.Errorf("linkID cannot be empty") + } + + link, err := d.c.GetLink(ctx, d.MainShare.ShareID, linkID) + if err != nil { + return nil, err + } + + return &link, nil +} + +func (d *ProtonDrive) getLinkKR(ctx context.Context, link *proton.Link) (*crypto.KeyRing, error) { + if link == nil { + return nil, fmt.Errorf("link cannot be nil") + } + + // Root Link or Root Dir + if link.ParentLinkID == "" { + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + return link.GetKeyRing(d.MainShareKR, signatureVerificationKR) + } + + // Get parent keyring recursively + parentLink, err := d.getLink(ctx, link.ParentLinkID) + if err != nil { + return nil, err + } + + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return nil, err + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + + return link.GetKeyRing(parentNodeKR, signatureVerificationKR) +} + +var ( + ErrKeyPassOrSaltedKeyPassMustBeNotNil = errors.New("either keyPass or saltedKeyPass must be not nil") + ErrFailedToUnlockUserKeys = errors.New("failed to unlock user keys") +) + +func getAccountKRs(ctx context.Context, c *proton.Client, keyPass, saltedKeyPass []byte) (*crypto.KeyRing, map[string]*crypto.KeyRing, map[string]proton.Address, []byte, error) { + + user, err := c.GetUser(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("user %#v", user) + + addrsArr, err := c.GetAddresses(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("addr %#v", addr) + + if saltedKeyPass == nil { + if keyPass == nil { + return nil, nil, nil, nil, ErrKeyPassOrSaltedKeyPassMustBeNotNil + } + + // Due to limitations, salts are stored using cacheCredentialToFile + salts, err := c.GetSalts(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("salts %#v", salts) + + saltedKeyPass, err = salts.SaltForKey(keyPass, user.Keys.Primary().ID) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("saltedKeyPass ok") + } + + userKR, addrKRs, err := proton.Unlock(user, addrsArr, saltedKeyPass, nil) + if err != nil { + return nil, nil, nil, nil, err + + } else if userKR.CountDecryptionEntities() == 0 { + return nil, nil, nil, nil, ErrFailedToUnlockUserKeys + } + + addrs := make(map[string]proton.Address) + for _, addr := range addrsArr { + addrs[addr.Email] = addr + } + + return userKR, addrKRs, addrs, saltedKeyPass, nil +} + +func (d *ProtonDrive) getSignatureVerificationKeyring(emailAddresses []string, verificationAddrKRs ...*crypto.KeyRing) (*crypto.KeyRing, error) { + ret, err := crypto.NewKeyRing(nil) + if err != nil { + return nil, err + } + + for _, emailAddress := range emailAddresses { + if addr, ok := d.addrData[emailAddress]; ok { + if addrKR, exists := d.addrKRs[addr.ID]; exists { + err = d.addKeysFromKR(ret, addrKR) + if err != nil { + return nil, err + } + } + } + } + + for _, kr := range verificationAddrKRs { + err = d.addKeysFromKR(ret, kr) + if err != nil { + return nil, err + } + } + + if ret.CountEntities() == 0 { + return nil, fmt.Errorf("no keyring for signature verification") + } + + return ret, nil +} + +func (d *ProtonDrive) addKeysFromKR(kr *crypto.KeyRing, newKRs ...*crypto.KeyRing) error { + for i := range newKRs { + for _, key := range newKRs[i].GetKeys() { + err := kr.AddKey(key) + if err != nil { + return err + } + } + } + return nil +} + +func (d *ProtonDrive) DirectRename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + //fmt.Printf("DEBUG DirectRename: path=%s, newName=%s", srcObj.GetPath(), newName) + + if d.MainShare == nil || d.DefaultAddrKR == nil { + return nil, fmt.Errorf("missing required fields: MainShare=%v, DefaultAddrKR=%v", + d.MainShare != nil, d.DefaultAddrKR != nil) + } + + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), srcObj.IsDir()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + parentLinkID := srcLink.ParentLinkID + if parentLinkID == "" { + return nil, fmt.Errorf("cannot rename root folder") + } + + encryptedName, err := d.encryptFileName(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateFileNameHash(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + renameReq := RenameRequest{ + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + } + + err = d.executeRenameAPI(ctx, srcLink.LinkID, renameReq) + if err != nil { + return nil, fmt.Errorf("rename API call failed: %w", err) + } + + return &model.Object{ + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) executeRenameAPI(ctx context.Context, linkID string, req RenameRequest) error { + + renameURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/rename", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal rename request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", renameURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.credentials.UID) + httpReq.Header.Set("Authorization", "Bearer "+d.credentials.AccessToken) + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute rename request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("rename failed with status %d", resp.StatusCode) + } + + var renameResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&renameResp); err != nil { + return fmt.Errorf("failed to decode rename response: %w", err) + } + + if renameResp.Code != 1000 { + return fmt.Errorf("rename failed with code %d", renameResp.Code) + } + + return nil +} + +func (d *ProtonDrive) executeMoveAPI(ctx context.Context, linkID string, req MoveRequest) error { + //fmt.Printf("DEBUG Move Request - Name: %s\n", req.Name) + //fmt.Printf("DEBUG Move Request - Hash: %s\n", req.Hash) + //fmt.Printf("DEBUG Move Request - OriginalHash: %s\n", req.OriginalHash) + //fmt.Printf("DEBUG Move Request - ParentLinkID: %s\n", req.ParentLinkID) + + //fmt.Printf("DEBUG Move Request - Name length: %d\n", len(req.Name)) + //fmt.Printf("DEBUG Move Request - NameSignatureEmail: %s\n", req.NameSignatureEmail) + //fmt.Printf("DEBUG Move Request - ContentHash: %v\n", req.ContentHash) + //fmt.Printf("DEBUG Move Request - NodePassphrase length: %d\n", len(req.NodePassphrase)) + //fmt.Printf("DEBUG Move Request - NodePassphraseSignature length: %d\n", len(req.NodePassphraseSignature)) + + //fmt.Printf("DEBUG Move Request - SrcLinkID: %s\n", linkID) + //fmt.Printf("DEBUG Move Request - DstParentLinkID: %s\n", req.ParentLinkID) + //fmt.Printf("DEBUG Move Request - ShareID: %s\n", d.MainShare.ShareID) + + srcLink, _ := d.getLink(ctx, linkID) + if srcLink != nil && srcLink.ParentLinkID == req.ParentLinkID { + return fmt.Errorf("cannot move to same parent directory") + } + + moveURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/move", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal move request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", moveURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Authorization", "Bearer "+d.credentials.AccessToken) + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.credentials.UID) + httpReq.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute move request: %w", err) + } + defer resp.Body.Close() + + var moveResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&moveResp); err != nil { + return fmt.Errorf("failed to decode move response: %w", err) + } + + if moveResp.Code != 1000 { + return fmt.Errorf("move operation failed with code: %d", moveResp.Code) + } + + return nil +} + +func (d *ProtonDrive) DirectMove(ctx context.Context, srcObj model.Obj, dstDir model.Obj) (model.Obj, error) { + //fmt.Printf("DEBUG DirectMove: srcPath=%s, dstPath=%s", srcObj.GetPath(), dstDir.GetPath()) + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), srcObj.IsDir()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + var dstParentLinkID string + if dstDir.GetPath() == "/" { + dstParentLinkID = d.RootLink.LinkID + } else { + dstLink, err := d.searchByPath(ctx, dstDir.GetPath(), true) + if err != nil { + return nil, fmt.Errorf("failed to find destination: %w", err) + } + dstParentLinkID = dstLink.LinkID + } + + if srcObj.IsDir() { + + // Check if destination is a descendant of source + if err := d.checkCircularMove(ctx, srcLink.LinkID, dstParentLinkID); err != nil { + return nil, err + } + } + + // Encrypt the filename for the new location + encryptedName, err := d.encryptFileName(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateNameHash(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + // Re-encrypt node passphrase for new parent context + reencryptedPassphrase, err := d.reencryptNodePassphrase(ctx, srcLink, dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to re-encrypt node passphrase: %w", err) + } + + moveReq := MoveRequest{ + ParentLinkID: dstParentLinkID, + NodePassphrase: reencryptedPassphrase, + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + ContentHash: nil, + + // *** Causes rejection *** + /* NodePassphraseSignature: srcLink.NodePassphraseSignature, */ + } + + //fmt.Printf("DEBUG MoveRequest validation:\n") + //fmt.Printf(" Name length: %d\n", len(moveReq.Name)) + //fmt.Printf(" Hash: %s\n", moveReq.Hash) + //fmt.Printf(" OriginalHash: %s\n", moveReq.OriginalHash) + //fmt.Printf(" NodePassphrase length: %d\n", len(moveReq.NodePassphrase)) + /* fmt.Printf(" NodePassphraseSignature length: %d\n", len(moveReq.NodePassphraseSignature)) */ + //fmt.Printf(" NameSignatureEmail: %s\n", moveReq.NameSignatureEmail) + + err = d.executeMoveAPI(ctx, srcLink.LinkID, moveReq) + if err != nil { + return nil, fmt.Errorf("move API call failed: %w", err) + } + + return &model.Object{ + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) reencryptNodePassphrase(ctx context.Context, srcLink *proton.Link, dstParentLinkID string) (string, error) { + // Get source parent link with metadata + srcParentLink, err := d.getLink(ctx, srcLink.ParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get source parent link: %w", err) + } + + // Get source parent keyring using link object + srcParentKR, err := d.getLinkKR(ctx, srcParentLink) + if err != nil { + return "", fmt.Errorf("failed to get source parent keyring: %w", err) + } + + // Get destination parent link with metadata + dstParentLink, err := d.getLink(ctx, dstParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get destination parent link: %w", err) + } + + // Get destination parent keyring using link object + dstParentKR, err := d.getLinkKR(ctx, dstParentLink) + if err != nil { + return "", fmt.Errorf("failed to get destination parent keyring: %w", err) + } + + // Re-encrypt the node passphrase from source parent context to destination parent context + reencryptedPassphrase, err := reencryptKeyPacket(srcParentKR, dstParentKR, d.DefaultAddrKR, srcLink.NodePassphrase) + if err != nil { + return "", fmt.Errorf("failed to re-encrypt key packet: %w", err) + } + + return reencryptedPassphrase, nil +} + +func (d *ProtonDrive) generateNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Get signature verification keyring + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func reencryptKeyPacket(srcKR, dstKR, _ *crypto.KeyRing, passphrase string) (string, error) { // addrKR (3) + oldSplitMessage, err := crypto.NewPGPSplitMessageFromArmored(passphrase) + if err != nil { + return "", err + } + + sessionKey, err := srcKR.DecryptSessionKey(oldSplitMessage.KeyPacket) + if err != nil { + return "", err + } + + newKeyPacket, err := dstKR.EncryptSessionKey(sessionKey) + if err != nil { + return "", err + } + + newSplitMessage := crypto.NewPGPSplitMessage(newKeyPacket, oldSplitMessage.DataPacket) + + return newSplitMessage.GetArmored() +} + +func (d *ProtonDrive) checkCircularMove(ctx context.Context, srcLinkID, dstParentLinkID string) error { + currentLinkID := dstParentLinkID + + for currentLinkID != "" && currentLinkID != d.RootLink.LinkID { + if currentLinkID == srcLinkID { + return fmt.Errorf("cannot move folder into itself or its subfolder") + } + + currentLink, err := d.getLink(ctx, currentLinkID) + if err != nil { + return err + } + currentLinkID = currentLink.ParentLinkID + } + + return nil +} diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index 7f497494502..691874c321e 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -7,12 +7,14 @@ import ( "hash" "io" "net/http" + "strings" "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" streamPkg "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" @@ -36,6 +38,14 @@ func (d *QuarkOrUC) GetAddition() driver.Additional { func (d *QuarkOrUC) Init(ctx context.Context) error { _, err := d.request("/config", http.MethodGet, nil, nil) + if err == nil && d.AdditionVersion != 2 { + d.AdditionVersion = 2 + if !d.UseTransCodingAddress && len(d.DownProxyUrl) == 0 { + d.WebProxy = true + d.WebdavPolicy = "native_proxy" + } + op.MustSaveDriverStorage(d) + } return err } @@ -44,39 +54,23 @@ func (d *QuarkOrUC) Drop(ctx context.Context) error { } func (d *QuarkOrUC) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files, err := d.GetFiles(dir.GetID()) - if err != nil { - return nil, err - } - return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil - }) + return d.GetFiles(dir.GetID()) } func (d *QuarkOrUC) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - data := base.Json{ - "fids": []string{file.GetID()}, - } - var resp DownResp - ua := d.conf.ua - _, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) { - req.SetHeader("User-Agent", ua). - SetBody(data) - }, &resp) - if err != nil { + f := file.(*File) + if d.UseTransCodingAddress && d.config.Name == "Quark" && f.Category == 1 && f.Size > 0 { + link, err := d.getTranscodingLink(file) + if err == nil { + return link, nil + } + if strings.Contains(err.Error(), "plf_invalid") { + log.Warnf("quark transcoding link invalid for %s, fallback to download link: %v", file.GetName(), err) + return d.getDownloadLink(file) + } return nil, err } - - return &model.Link{ - URL: resp.Data[0].DownloadUrl, - Header: http.Header{ - "Cookie": []string{d.Cookie}, - "Referer": []string{d.conf.referer}, - "User-Agent": []string{ua}, - }, - Concurrency: 3, - PartSize: 10 * utils.MB, - }, nil + return d.getDownloadLink(file) } func (d *QuarkOrUC) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { diff --git a/drivers/quark_uc/meta.go b/drivers/quark_uc/meta.go index f3acfe88562..6940c44b9d6 100644 --- a/drivers/quark_uc/meta.go +++ b/drivers/quark_uc/meta.go @@ -8,8 +8,11 @@ import ( type Addition struct { Cookie string `json:"cookie" required:"true"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"none,file_type,file_name,updated_at" default:"none"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + OrderBy string `json:"order_by" type:"select" options:"none,file_type,file_name,updated_at" default:"none"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + UseTransCodingAddress bool `json:"use_transcoding_address" help:"You can watch the transcoded video and support 302 redirection" required:"true" default:"false"` + OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` + AdditionVersion int } type Conf struct { @@ -24,7 +27,6 @@ func init() { return &QuarkOrUC{ config: driver.Config{ Name: "Quark", - OnlyLocal: true, DefaultRoot: "0", NoOverwriteUpload: true, }, @@ -40,7 +42,7 @@ func init() { return &QuarkOrUC{ config: driver.Config{ Name: "UC", - OnlyLocal: true, + OnlyProxy: true, DefaultRoot: "0", NoOverwriteUpload: true, }, diff --git a/drivers/quark_uc/types.go b/drivers/quark_uc/types.go index afbdb3eff89..13bfac2f7d5 100644 --- a/drivers/quark_uc/types.go +++ b/drivers/quark_uc/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" ) type Resp struct { @@ -18,13 +19,13 @@ type File struct { Fid string `json:"fid"` FileName string `json:"file_name"` //PdirFid string `json:"pdir_fid"` - //Category int `json:"category"` + Category int `json:"category"` //FileType int `json:"file_type"` Size int64 `json:"size"` //FormatType string `json:"format_type"` //Status int `json:"status"` //Tags string `json:"tags,omitempty"` - //LCreatedAt int64 `json:"l_created_at"` + LCreatedAt int64 `json:"l_created_at"` LUpdatedAt int64 `json:"l_updated_at"` //NameSpace int `json:"name_space"` //IncludeItems int `json:"include_items,omitempty"` @@ -32,8 +33,8 @@ type File struct { //BackupSign int `json:"backup_sign"` //Duration int `json:"duration"` //FileSource string `json:"file_source"` - File bool `json:"file"` - //CreatedAt int64 `json:"created_at"` + File bool `json:"file"` + CreatedAt int64 `json:"created_at"` UpdatedAt int64 `json:"updated_at"` //PrivateExtra struct {} `json:"_private_extra"` //ObjCategory string `json:"obj_category,omitempty"` @@ -50,6 +51,38 @@ func fileToObj(f File) *model.Object { } } +func (f *File) GetSize() int64 { + return f.Size +} + +func (f *File) GetName() string { + return f.FileName +} + +func (f *File) ModTime() time.Time { + return time.UnixMilli(f.UpdatedAt) +} + +func (f *File) CreateTime() time.Time { + return time.UnixMilli(f.CreatedAt) +} + +func (f *File) IsDir() bool { + return !f.File +} + +func (f *File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f *File) GetID() string { + return f.Fid +} + +func (f *File) GetPath() string { + return "" +} + type SortResp struct { Resp Data struct { @@ -100,6 +133,39 @@ type DownResp struct { //} `json:"metadata"` } +type TranscodingResp struct { + Resp + Data struct { + DefaultResolution string `json:"default_resolution"` + OriginDefaultResolution string `json:"origin_default_resolution"` + VideoList []struct { + Resolution string `json:"resolution"` + VideoInfo struct { + Duration int `json:"duration"` + Size int64 `json:"size"` + Format string `json:"format"` + Width int `json:"width"` + Height int `json:"height"` + Bitrate float64 `json:"bitrate"` + Codec string `json:"codec"` + Fps float64 `json:"fps"` + Rotate int `json:"rotate"` + UpdateTime int64 `json:"update_time"` + URL string `json:"url"` + Resolution string `json:"resolution"` + HlsType string `json:"hls_type"` + Finish bool `json:"finish"` + Resoultion string `json:"resoultion"` + Success bool `json:"success"` + } `json:"video_info,omitempty"` + } `json:"video_list"` + FileName string `json:"file_name"` + NameSpace int `json:"name_space"` + Size int64 `json:"size"` + Thumbnail string `json:"thumbnail"` + } `json:"data"` +} + type UpPreResp struct { Resp Data struct { diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index c5845cc6823..2f99c308f33 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "errors" "fmt" + "html" "io" "net/http" "strconv" @@ -50,20 +51,29 @@ func (d *QuarkOrUC) request(pathname string, method string, callback base.ReqCal d.Cookie = cookie.SetStr(d.Cookie, "__puus", __puus.Value) op.MustSaveDriverStorage(d) } + if d.UseTransCodingAddress && d.config.Name == "Quark" { + __pus := cookie.GetCookie(res.Cookies(), "__pus") + if __pus != nil { + d.Cookie = cookie.SetStr(d.Cookie, "__pus", __pus.Value) + op.MustSaveDriverStorage(d) + } + } if e.Status >= 400 || e.Code != 0 { return nil, errors.New(e.Message) } return res.Body(), nil } -func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { - files := make([]File, 0) +func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { + files := make([]model.Obj, 0) page := 1 size := 100 query := map[string]string{ - "pdir_fid": parent, - "_size": strconv.Itoa(size), - "_fetch_total": "1", + "pdir_fid": parent, + "_size": strconv.Itoa(size), + "_fetch_total": "1", + "fetch_all_file": "1", + "fetch_risk_file_name": "1", } if d.OrderBy != "none" { query["_sort"] = "file_type:asc," + d.OrderBy + ":" + d.OrderDirection @@ -77,7 +87,16 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { if err != nil { return nil, err } - files = append(files, resp.Data.List...) + for _, file := range resp.Data.List { + file.FileName = html.UnescapeString(file.FileName) + if d.OnlyListVideoFile { + if file.IsDir() || file.Category == 1 { + files = append(files, &file) + } + } else { + files = append(files, &file) + } + } if page*size >= resp.Metadata.Total { break } @@ -86,6 +105,62 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { return files, nil } +func (d *QuarkOrUC) getDownloadLink(file model.Obj) (*model.Link, error) { + data := base.Json{ + "fids": []string{file.GetID()}, + } + var resp DownResp + ua := d.conf.ua + _, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", ua). + SetBody(data) + }, &resp) + if err != nil { + return nil, err + } + + return &model.Link{ + URL: resp.Data[0].DownloadUrl, + Header: http.Header{ + "Cookie": []string{d.Cookie}, + "Referer": []string{d.conf.referer}, + "User-Agent": []string{ua}, + }, + Concurrency: 3, + PartSize: 10 * utils.MB, + }, nil +} + +func (d *QuarkOrUC) getTranscodingLink(file model.Obj) (*model.Link, error) { + data := base.Json{ + "fid": file.GetID(), + "resolutions": "low,normal,high,super,2k,4k", + "supports": "fmp4_av,m3u8,dolby_vision", + } + var resp TranscodingResp + ua := d.conf.ua + + _, err := d.request("/file/v2/play/project", http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", ua). + SetBody(data) + }, &resp) + if err != nil { + return nil, err + } + + for _, info := range resp.Data.VideoList { + if info.VideoInfo.URL != "" { + return &model.Link{ + URL: info.VideoInfo.URL, + Concurrency: 3, + PartSize: 10 * utils.MB, + }, nil + } + } + + return nil, errors.New("no link found") +} + func (d *QuarkOrUC) upPre(file model.FileStreamer, parentId string) (UpPreResp, error) { now := time.Now() data := base.Json{ diff --git a/drivers/quqi/types.go b/drivers/quqi/types.go index 32557361532..cade93de885 100644 --- a/drivers/quqi/types.go +++ b/drivers/quqi/types.go @@ -83,7 +83,7 @@ type Group struct { Type int `json:"type"` Name string `json:"name"` IsAdministrator int `json:"is_administrator"` - Role int `json:"role"` + Role []int `json:"role"` Avatar string `json:"avatar_url"` IsStick int `json:"is_stick"` Nickname string `json:"nickname"` diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index b741148983e..896f69b3028 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -15,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/server/common" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -32,6 +33,33 @@ type S3 struct { cron *cron.Cron } +var storageClassLookup = map[string]string{ + "standard": s3.ObjectStorageClassStandard, + "reduced_redundancy": s3.ObjectStorageClassReducedRedundancy, + "glacier": s3.ObjectStorageClassGlacier, + "standard_ia": s3.ObjectStorageClassStandardIa, + "onezone_ia": s3.ObjectStorageClassOnezoneIa, + "intelligent_tiering": s3.ObjectStorageClassIntelligentTiering, + "deep_archive": s3.ObjectStorageClassDeepArchive, + "outposts": s3.ObjectStorageClassOutposts, + "glacier_ir": s3.ObjectStorageClassGlacierIr, + "snow": s3.ObjectStorageClassSnow, + "express_onezone": s3.ObjectStorageClassExpressOnezone, +} + +func (d *S3) resolveStorageClass() *string { + value := strings.TrimSpace(d.StorageClass) + if value == "" { + return nil + } + normalized := strings.ToLower(strings.ReplaceAll(value, "-", "_")) + if v, ok := storageClassLookup[normalized]; ok { + return aws.String(v) + } + log.Warnf("s3: unknown storage class %q, using raw value", d.StorageClass) + return aws.String(value) +} + func (d *S3) Config() driver.Config { return d.config } @@ -41,6 +69,9 @@ func (d *S3) GetAddition() driver.Additional { } func (d *S3) Init(ctx context.Context) error { + if !strings.Contains(d.Storage.Addition, `"use_placeholder"`) { + d.UsePlaceholder = true + } if d.Region == "" { d.Region = "alist" } @@ -123,16 +154,20 @@ func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*mo } func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - return d.Put(ctx, &model.Object{ - Path: stdpath.Join(parentDir.GetPath(), dirName), - }, &stream.FileStream{ - Obj: &model.Object{ - Name: getPlaceholderName(d.Placeholder), - Modified: time.Now(), - }, - Reader: io.NopCloser(bytes.NewReader([]byte{})), - Mimetype: "application/octet-stream", - }, func(float64) {}) + dirPath := stdpath.Join(parentDir.GetPath(), dirName) + if d.UsePlaceholder { + return d.Put(ctx, &model.Object{ + Path: dirPath, + }, &stream.FileStream{ + Obj: &model.Object{ + Name: getPlaceholderName(d.Placeholder), + Modified: time.Now(), + }, + Reader: io.NopCloser(bytes.NewReader([]byte{})), + Mimetype: "application/octet-stream", + }, func(float64) {}) + } + return d.createDirMarker(ctx, dirPath) } func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -179,8 +214,34 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up }), ContentType: &contentType, } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } _, err := uploader.UploadWithContext(ctx, input) return err } -var _ driver.Driver = (*S3)(nil) +func (d *S3) putEmptyObject(ctx context.Context, key string) error { + uploader := s3manager.NewUploader(d.Session) + contentType := "application/octet-stream" + input := &s3manager.UploadInput{ + Bucket: &d.Bucket, + Key: &key, + Body: driver.NewLimitedUploadStream(ctx, bytes.NewReader([]byte{})), + ContentType: &contentType, + } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } + _, err := uploader.UploadWithContext(ctx, input) + return err +} + +func (d *S3) createDirMarker(ctx context.Context, dirPath string) error { + return d.putEmptyObject(ctx, getKey(dirPath, true)) +} + +var ( + _ driver.Driver = (*S3)(nil) + _ driver.Other = (*S3)(nil) +) diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 4de4b60a690..0c675e85646 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -19,8 +19,10 @@ type Addition struct { Placeholder string `json:"placeholder"` ForcePathStyle bool `json:"force_path_style"` ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` + UsePlaceholder bool `json:"use_placeholder" default:"true" help:"Create hidden placeholder file (for example .alist) to keep empty folders."` RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` + StorageClass string `json:"storage_class" type:"select" options:",standard,standard_ia,onezone_ia,intelligent_tiering,glacier,glacier_ir,deep_archive,archive" help:"Storage class for new objects. AWS and Tencent COS support different subsets (COS uses ARCHIVE/DEEP_ARCHIVE)."` } func init() { diff --git a/drivers/s3/other.go b/drivers/s3/other.go new file mode 100644 index 00000000000..e299dae05d1 --- /dev/null +++ b/drivers/s3/other.go @@ -0,0 +1,286 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +const ( + OtherMethodArchive = "archive" + OtherMethodArchiveStatus = "archive_status" + OtherMethodThaw = "thaw" + OtherMethodThawStatus = "thaw_status" +) + +type ArchiveRequest struct { + StorageClass string `json:"storage_class"` +} + +type ThawRequest struct { + Days int64 `json:"days"` + Tier string `json:"tier"` +} + +type ObjectDescriptor struct { + Path string `json:"path"` + Bucket string `json:"bucket"` + Key string `json:"key"` +} + +type ArchiveResponse struct { + Action string `json:"action"` + Object ObjectDescriptor `json:"object"` + StorageClass string `json:"storage_class"` + RequestID string `json:"request_id,omitempty"` + VersionID string `json:"version_id,omitempty"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` +} + +type ThawResponse struct { + Action string `json:"action"` + Object ObjectDescriptor `json:"object"` + RequestID string `json:"request_id,omitempty"` + Status *RestoreStatus `json:"status,omitempty"` +} + +type RestoreStatus struct { + Ongoing bool `json:"ongoing"` + Expiry string `json:"expiry,omitempty"` + Raw string `json:"raw"` +} + +func (d *S3) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + if args.Obj == nil { + return nil, fmt.Errorf("missing object reference") + } + if args.Obj.IsDir() { + return nil, errs.NotSupport + } + + switch strings.ToLower(strings.TrimSpace(args.Method)) { + case "archive": + return d.archive(ctx, args) + case "archive_status": + return d.archiveStatus(ctx, args) + case "thaw": + return d.thaw(ctx, args) + case "thaw_status": + return d.thawStatus(ctx, args) + default: + return nil, errs.NotSupport + } +} + +func (d *S3) archive(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + payload := ArchiveRequest{} + if err := DecodeOtherArgs(args.Data, &payload); err != nil { + return nil, fmt.Errorf("parse archive request: %w", err) + } + if payload.StorageClass == "" { + return nil, fmt.Errorf("storage_class is required") + } + storageClass := NormalizeStorageClass(payload.StorageClass) + input := &s3.CopyObjectInput{ + Bucket: &d.Bucket, + Key: &key, + CopySource: aws.String(url.PathEscape(d.Bucket + "/" + key)), + MetadataDirective: aws.String(s3.MetadataDirectiveCopy), + StorageClass: aws.String(storageClass), + } + copyReq, output := d.client.CopyObjectRequest(input) + copyReq.SetContext(ctx) + if err := copyReq.Send(); err != nil { + return nil, err + } + + resp := ArchiveResponse{ + Action: "archive", + Object: d.describeObject(args.Obj, key), + StorageClass: storageClass, + RequestID: copyReq.RequestID, + } + if output.VersionId != nil { + resp.VersionID = aws.StringValue(output.VersionId) + } + if result := output.CopyObjectResult; result != nil { + resp.ETag = aws.StringValue(result.ETag) + if result.LastModified != nil { + resp.LastModified = result.LastModified.UTC().Format(time.RFC3339) + } + } + if status, err := d.describeObjectStatus(ctx, key); err == nil { + if status.StorageClass != "" { + resp.StorageClass = status.StorageClass + } + } + return resp, nil +} + +func (d *S3) archiveStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + status, err := d.describeObjectStatus(ctx, key) + if err != nil { + return nil, err + } + return ArchiveResponse{ + Action: "archive_status", + Object: d.describeObject(args.Obj, key), + StorageClass: status.StorageClass, + }, nil +} + +func (d *S3) thaw(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + payload := ThawRequest{Days: 1} + if err := DecodeOtherArgs(args.Data, &payload); err != nil { + return nil, fmt.Errorf("parse thaw request: %w", err) + } + if payload.Days <= 0 { + payload.Days = 1 + } + restoreRequest := &s3.RestoreRequest{ + Days: aws.Int64(payload.Days), + } + if tier := NormalizeRestoreTier(payload.Tier); tier != "" { + restoreRequest.GlacierJobParameters = &s3.GlacierJobParameters{Tier: aws.String(tier)} + } + input := &s3.RestoreObjectInput{ + Bucket: &d.Bucket, + Key: &key, + RestoreRequest: restoreRequest, + } + restoreReq, _ := d.client.RestoreObjectRequest(input) + restoreReq.SetContext(ctx) + if err := restoreReq.Send(); err != nil { + return nil, err + } + status, _ := d.describeObjectStatus(ctx, key) + resp := ThawResponse{ + Action: "thaw", + Object: d.describeObject(args.Obj, key), + RequestID: restoreReq.RequestID, + } + if status != nil { + resp.Status = status.Restore + } + return resp, nil +} + +func (d *S3) thawStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + status, err := d.describeObjectStatus(ctx, key) + if err != nil { + return nil, err + } + return ThawResponse{ + Action: "thaw_status", + Object: d.describeObject(args.Obj, key), + Status: status.Restore, + }, nil +} + +func (d *S3) describeObject(obj model.Obj, key string) ObjectDescriptor { + return ObjectDescriptor{ + Path: obj.GetPath(), + Bucket: d.Bucket, + Key: key, + } +} + +type objectStatus struct { + StorageClass string + Restore *RestoreStatus +} + +func (d *S3) describeObjectStatus(ctx context.Context, key string) (*objectStatus, error) { + head, err := d.client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{Bucket: &d.Bucket, Key: &key}) + if err != nil { + return nil, err + } + status := &objectStatus{ + StorageClass: aws.StringValue(head.StorageClass), + Restore: parseRestoreHeader(head.Restore), + } + return status, nil +} + +func parseRestoreHeader(header *string) *RestoreStatus { + if header == nil { + return nil + } + value := strings.TrimSpace(*header) + if value == "" { + return nil + } + status := &RestoreStatus{Raw: value} + parts := strings.Split(value, ",") + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if strings.HasPrefix(part, "ongoing-request=") { + status.Ongoing = strings.Contains(part, "\"true\"") + } + if strings.HasPrefix(part, "expiry-date=") { + expiry := strings.Trim(part[len("expiry-date="):], "\"") + if expiry != "" { + if t, err := time.Parse(time.RFC1123, expiry); err == nil { + status.Expiry = t.UTC().Format(time.RFC3339) + } else { + status.Expiry = expiry + } + } + } + } + return status +} + +func DecodeOtherArgs(data interface{}, target interface{}) error { + if data == nil { + return nil + } + raw, err := json.Marshal(data) + if err != nil { + return err + } + return json.Unmarshal(raw, target) +} + +func NormalizeStorageClass(value string) string { + normalized := strings.ToLower(strings.TrimSpace(strings.ReplaceAll(value, "-", "_"))) + if normalized == "" { + return value + } + if v, ok := storageClassLookup[normalized]; ok { + return v + } + return value +} + +func NormalizeRestoreTier(value string) string { + normalized := strings.ToLower(strings.TrimSpace(value)) + switch normalized { + case "", "default": + return "" + case "bulk": + return s3.TierBulk + case "standard": + return s3.TierStandard + case "expedited": + return s3.TierExpedited + default: + return value + } +} diff --git a/drivers/s3/util.go b/drivers/s3/util.go index e02945a07d2..863b88bcab8 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -8,6 +8,7 @@ import ( "path" "strings" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" @@ -105,17 +106,20 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { files = append(files, &file) } for _, object := range listObjectsResult.Contents { + if strings.HasSuffix(*object.Key, "/") { + continue + } name := path.Base(*object.Key) if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue } - file := model.Object{ + file := &model.Object{ //Id: *object.Key, Name: name, Size: *object.Size, Modified: *object.LastModified, } - files = append(files, &file) + files = append(files, model.WrapObjStorageClass(file, aws.StringValue(object.StorageClass))) } if listObjectsResult.IsTruncated == nil { return nil, errors.New("IsTruncated nil") @@ -164,13 +168,13 @@ func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue } - file := model.Object{ + file := &model.Object{ //Id: *object.Key, Name: name, Size: *object.Size, Modified: *object.LastModified, } - files = append(files, &file) + files = append(files, model.WrapObjStorageClass(file, aws.StringValue(object.StorageClass))) } if !aws.BoolValue(listObjectsResult.IsTruncated) { break @@ -202,15 +206,21 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error { CopySource: aws.String(url.PathEscape(d.Bucket + "/" + srcKey)), Key: &dstKey, } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } _, err := d.client.CopyObject(input) return err } func (d *S3) copyDir(ctx context.Context, src string, dst string) error { - objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) + objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true, Refresh: true}) if err != nil { return err } + if len(objs) == 0 && !d.UsePlaceholder { + return d.createDirMarker(ctx, dst) + } for _, obj := range objs { cSrc := path.Join(src, obj.GetName()) cDst := path.Join(dst, obj.GetName()) @@ -227,8 +237,13 @@ func (d *S3) copyDir(ctx context.Context, src string, dst string) error { } func (d *S3) removeDir(ctx context.Context, src string) error { - objs, err := op.List(ctx, d, src, model.ListArgs{}) + d.cleanupDirArtifacts(src) + + objs, err := op.List(ctx, d, src, model.ListArgs{Refresh: true}) if err != nil { + if errs.IsObjectNotFound(err) { + return nil + } return err } for _, obj := range objs { @@ -242,9 +257,14 @@ func (d *S3) removeDir(ctx context.Context, src string) error { return err } } + d.cleanupDirArtifacts(src) + return nil +} + +func (d *S3) cleanupDirArtifacts(src string) { _ = d.removeFile(path.Join(src, getPlaceholderName(d.Placeholder))) _ = d.removeFile(path.Join(src, d.Placeholder)) - return nil + _ = d.removeFile(getKey(src, true)) } func (d *S3) removeFile(src string) error { diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go new file mode 100644 index 00000000000..9ce32e25b69 --- /dev/null +++ b/drivers/streamtape/driver.go @@ -0,0 +1,543 @@ +package streamtape + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + log "github.com/sirupsen/logrus" +) + +type Streamtape struct { + model.Storage + Addition +} + +var waitMoreSecondsRe = regexp.MustCompile(`wait\s+(\d+)\s+more\s+seconds?`) + +func (d *Streamtape) Config() driver.Config { + return config +} + +func (d *Streamtape) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Streamtape) Init(ctx context.Context) error { + if strings.TrimSpace(d.APILogin) == "" || strings.TrimSpace(d.APIKey) == "" { + return errors.New("api_login and api_key are required") + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + + var account accountInfo + if err := d.callAPI(ctx, "/account/info", nil, &account); err != nil { + return err + } + + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Streamtape) Drop(ctx context.Context) error { + return nil +} + +func (d *Streamtape) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := d.RootFolderID + if dir.GetID() != "" { + folderID = folderIDFromObjID(dir.GetID()) + } + + params := map[string]string{} + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + + var result listFolderResult + if err := d.callAPI(ctx, "/file/listfolder", params, &result); err != nil { + return nil, err + } + + objects := make([]model.Obj, 0, len(result.Folders)+len(result.Files)) + for _, f := range result.Folders { + objects = append(objects, &model.Object{ + ID: encodeFolderID(f.ID), + Name: f.Name, + IsFolder: true, + }) + } + for _, f := range result.Files { + objects = append(objects, buildFileObj(f)) + } + return objects, nil +} + +func (d *Streamtape) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + fileID := fileIDFromObjID(file.GetID()) + if fileID == "" { + return nil, errors.New("empty file id") + } + + var ticket dlTicketResult + if err := d.callAPI(ctx, "/file/dlticket", map[string]string{"file": fileID}, &ticket); err != nil { + return nil, err + } + + var dl dlResult + waitSeconds := ticket.WaitTime + if waitSeconds > 0 { + timer := time.NewTimer(time.Duration(waitSeconds+1) * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + + var err error + for i := 0; i < 3; i++ { + err = d.callAPI(ctx, "/file/dl", map[string]string{ + "file": fileID, + "ticket": ticket.Ticket, + }, &dl) + if err == nil { + break + } + waitSeconds = extractWaitSecondsFromErr(err) + if waitSeconds <= 0 { + return nil, err + } + timer := time.NewTimer(time.Duration(waitSeconds+1) * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + if err != nil { + return nil, err + } + + finalURL := ensureStreamQuery(dl.URL) + log.Infof("streamtape direct link file=%s url=%s", fileID, finalURL) + link := &model.Link{ + URL: finalURL, + Header: http.Header{ + "Referer": []string{"https://streamtape.com/"}, + "Origin": []string{"https://streamtape.com"}, + }, + } + d.applyRangeStrategy(link, file.GetSize()) + return link, nil +} + +func extractWaitSecondsFromErr(err error) int { + if err == nil { + return 0 + } + matches := waitMoreSecondsRe.FindStringSubmatch(strings.ToLower(err.Error())) + if len(matches) < 2 { + return 0 + } + seconds, convErr := strconv.Atoi(matches[1]) + if convErr != nil || seconds < 0 { + return 0 + } + return seconds +} + +func ensureStreamQuery(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + q := u.Query() + if q.Get("stream") == "" { + q.Set("stream", "1") + u.RawQuery = q.Encode() + } + return u.String() +} + +func (d *Streamtape) applyRangeStrategy(link *model.Link, size int64) { + if !d.EnableRangeControl || size <= 0 { + return + } + + mode := strings.ToLower(strings.TrimSpace(d.RangeMode)) + if mode == "" { + mode = "chunk" + } + + switch mode { + case "full": + // Keep single full-tail behavior while still using ranged requests. + link.Concurrency = 1 + link.PartSize = int(size) + case "percent": + percent := d.RangePercent + if percent <= 0 { + percent = 15 + } + if percent > 100 { + percent = 100 + } + partSize := size * int64(percent) / 100 + if partSize < 1*1024*1024 { + partSize = 1 * 1024 * 1024 + } + if partSize > size { + partSize = size + } + link.Concurrency = 1 + link.PartSize = int(partSize) + default: + chunkMB := d.RangeChunkMB + if chunkMB <= 0 { + chunkMB = 8 + } + partSize := int64(chunkMB) * 1024 * 1024 + if partSize > size { + partSize = size + } + concurrency := d.RangeConcurrency + if concurrency <= 0 { + concurrency = 4 + } + link.Concurrency = concurrency + link.PartSize = int(partSize) + } +} + +func (d *Streamtape) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + pid := d.RootFolderID + if parentDir.GetID() != "" { + pid = folderIDFromObjID(parentDir.GetID()) + } + + params := map[string]string{"name": dirName} + if pid != "" && pid != "0" { + params["pid"] = pid + } + + var result createFolderResult + if err := d.callAPI(ctx, "/file/createfolder", params, &result); err != nil { + return nil, err + } + + return &model.Object{ + ID: encodeFolderID(result.FolderID), + Name: dirName, + IsFolder: true, + }, nil +} + +func (d *Streamtape) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, errs.NotImplement + } + fileID := fileIDFromObjID(srcObj.GetID()) + if fileID == "" { + return nil, errors.New("empty file id") + } + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + if folderID == "" || folderID == "0" { + return nil, fmt.Errorf("streamtape move to root is not supported by API") + } + + if err := d.callAPI(ctx, "/file/move", map[string]string{ + "file": fileID, + "folder": folderID, + }, nil); err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: false, + }, nil +} + +func (d *Streamtape) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + endpoint := "/file/rename" + params := map[string]string{"name": newName} + if srcObj.IsDir() { + endpoint = "/file/renamefolder" + params["folder"] = folderIDFromObjID(srcObj.GetID()) + } else { + params["file"] = fileIDFromObjID(srcObj.GetID()) + } + + if err := d.callAPI(ctx, endpoint, params, nil); err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Streamtape) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) Remove(ctx context.Context, obj model.Obj) error { + endpoint := "/file/delete" + params := map[string]string{} + if obj.IsDir() { + endpoint = "/file/deletefolder" + params["folder"] = folderIDFromObjID(obj.GetID()) + } else { + params["file"] = fileIDFromObjID(obj.GetID()) + } + return d.callAPI(ctx, endpoint, params, nil) +} + +func (d *Streamtape) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + params := map[string]string{} + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + if d.Sha256 != "" { + params["sha256"] = d.Sha256 + } + + var uploadURL uploadURLResult + if err := d.callAPI(ctx, "/file/ul", params, &uploadURL); err != nil { + return nil, err + } + + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }) + + res, err := base.RestyClient.R(). + SetContext(ctx). + SetFileReader("file1", file.GetName(), reader). + Post(uploadURL.URL) + if err != nil { + return nil, err + } + if res.StatusCode() >= http.StatusBadRequest { + return nil, fmt.Errorf("streamtape upload failed: http %d", res.StatusCode()) + } + + uploadedID := extractFileIDFromUploadBody(res.Body()) + if uploadedID == "" { + list, listErr := d.List(ctx, &model.Object{ID: encodeFolderID(folderID), IsFolder: true}, model.ListArgs{}) + if listErr == nil { + for _, obj := range list { + if obj.IsDir() { + continue + } + if obj.GetName() == file.GetName() && (file.GetSize() <= 0 || obj.GetSize() == file.GetSize()) { + return obj, nil + } + } + } + return &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil + } + + return &model.Object{ + ID: encodeFileID(uploadedID), + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +// PutURL initiates a remote upload from an external URL +func (d *Streamtape) PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + params := map[string]string{ + "url": url, + } + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + if name != "" { + params["name"] = name + } + + var result remoteDlAddResult + if err := d.callAPI(ctx, "/remotedl/add", params, &result); err != nil { + return nil, err + } + + return &model.Object{ + ID: encodeRemoteUploadID(result.ID), + Name: name, + IsFolder: false, + }, nil +} + +func (d *Streamtape) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch strings.ToLower(args.Method) { + case "remotedl_status": + return d.remoteDlStatus(ctx, args) + case "remotedl_remove": + return d.remoteDlRemove(ctx, args) + case "file_info": + return d.fileInfo(ctx, args) + case "thumbnail": + return d.thumbnail(ctx, args) + case "conversion_status": + return d.conversionStatus(ctx, args) + default: + return nil, errs.NotSupport + } +} + +func (d *Streamtape) extractRemoteUploadID(args model.OtherArgs) (string, error) { + uploadID := remoteUploadIDFromObjID(args.Obj.GetID()) + if uploadID == "" { + if data, ok := args.Data.(map[string]interface{}); ok { + if id, ok := data["id"].(string); ok { + uploadID = id + } + } + } + if uploadID == "" { + return "", fmt.Errorf("remote upload ID required") + } + return uploadID, nil +} + +func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + uploadID, err := d.extractRemoteUploadID(args) + if err != nil { + return nil, err + } + + var result remoteDlStatusResult + if err := d.callAPI(ctx, "/remotedl/status", map[string]string{"id": uploadID}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) remoteDlRemove(ctx context.Context, args model.OtherArgs) (interface{}, error) { + uploadID, err := d.extractRemoteUploadID(args) + if err != nil { + return nil, err + } + + if err := d.callAPI(ctx, "/remotedl/remove", map[string]string{"id": uploadID}, nil); err != nil { + return nil, err + } + return true, nil +} + +func (d *Streamtape) fileInfo(ctx context.Context, args model.OtherArgs) (interface{}, error) { + var fileIDs string + if data, ok := args.Data.(map[string]interface{}); ok { + if ids, ok := data["file_ids"].(string); ok { + fileIDs = ids + } + } + if fileIDs == "" { + fileIDs = fileIDFromObjID(args.Obj.GetID()) + } + if fileIDs == "" { + return nil, fmt.Errorf("file IDs required") + } + + var result fileInfoResult + if err := d.callAPI(ctx, "/file/info", map[string]string{"file": fileIDs}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) thumbnail(ctx context.Context, args model.OtherArgs) (interface{}, error) { + fileID := fileIDFromObjID(args.Obj.GetID()) + if fileID == "" { + return nil, fmt.Errorf("file ID required") + } + + var result string + if err := d.callAPI(ctx, "/file/getsplash", map[string]string{"file": fileID}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) conversionStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + isFailed := false + if data, ok := args.Data.(map[string]interface{}); ok { + if t, ok := data["type"].(string); ok && t == "failed" { + isFailed = true + } + } + + endpoint := "/file/runningconverts" + if isFailed { + endpoint = "/file/failedconverts" + } + + var result conversionResult + if err := d.callAPI(ctx, endpoint, nil, &result); err != nil { + return nil, err + } + return result, nil +} + +var _ driver.Driver = (*Streamtape)(nil) diff --git a/drivers/streamtape/meta.go b/drivers/streamtape/meta.go new file mode 100644 index 00000000000..55e97894dbe --- /dev/null +++ b/drivers/streamtape/meta.go @@ -0,0 +1,39 @@ +package streamtape + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APILogin string `json:"api_login" required:"true" help:"API Login from Streamtape account settings"` + APIKey string `json:"api_key" required:"true" help:"API Key from Streamtape account settings"` + RangeMode string `json:"range_mode" type:"select" options:"chunk,full,percent" default:"chunk" help:"Range strategy for preview: chunk=bounded ranges, full=single full-tail range, percent=part size by file percentage"` + RangeChunkMB int `json:"range_chunk_mb" type:"number" default:"8" help:"Chunk mode part size in MB"` + RangeConcurrency int `json:"range_concurrency" type:"number" default:"4" help:"Chunk mode concurrent upstream requests"` + RangePercent int `json:"range_percent" type:"number" default:"15" help:"Percent mode part size percentage (1-100)"` + EnableRangeControl bool `json:"enable_range_control" default:"true" help:"Enable driver-level range shaping for smoother streaming"` + Sha256 string `json:"sha256" help:"Expected SHA256 hash for upload verification (optional)"` +} + +var config = driver.Config{ + Name: "Streamtape", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: true, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "warning|Moving files to root folder is not supported by Streamtape API", + NoOverwriteUpload: false, + ProxyRangeOption: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Streamtape{} + }) +} diff --git a/drivers/streamtape/types.go b/drivers/streamtape/types.go new file mode 100644 index 00000000000..347e89d8a67 --- /dev/null +++ b/drivers/streamtape/types.go @@ -0,0 +1,97 @@ +package streamtape + +import "encoding/json" + +type apiResponse struct { + Status int `json:"status"` + Msg string `json:"msg"` + Result json.RawMessage `json:"result"` +} + +type accountInfo struct { + APIID string `json:"apiid"` + Email string `json:"email"` + SignupAt string `json:"signup_at"` +} + +type listFolderResult struct { + Folders []folderItem `json:"folders"` + Files []fileItem `json:"files"` +} + +type folderItem struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type fileItem struct { + Name string `json:"name"` + Size int64 `json:"size"` + Link string `json:"link"` + CreatedAt int64 `json:"created_at"` + Downloads int64 `json:"downloads"` + LinkID string `json:"linkid"` + Convert string `json:"convert"` +} + +type dlTicketResult struct { + Ticket string `json:"ticket"` + WaitTime int `json:"wait_time"` +} + +type dlResult struct { + Name string `json:"name"` + Size int64 `json:"size"` + URL string `json:"url"` +} + +type createFolderResult struct { + FolderID string `json:"folderid"` +} + +type uploadURLResult struct { + URL string `json:"url"` +} + +type remoteDlAddResult struct { + ID string `json:"id"` + FolderID string `json:"folderid"` +} + +type remoteDlStatusResult map[string]remoteDlStatusItem + +type remoteDlStatusItem struct { + ID string `json:"id"` + RemoteURL string `json:"remoteurl"` + Status string `json:"status"` + BytesLoaded interface{} `json:"bytes_loaded"` + BytesTotal interface{} `json:"bytes_total"` + FolderID string `json:"folderid"` + Added string `json:"added"` + LastUpdate string `json:"last_update"` + ExtID bool `json:"extid"` + URL bool `json:"url"` +} + +type fileInfoResult map[string]fileInfoItem + +type fileInfoItem struct { + ID string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + Type string `json:"type"` + Converted bool `json:"converted"` + Status int `json:"status"` +} + +type conversionResult []conversionItem + +type conversionItem struct { + Name string `json:"name"` + FolderID string `json:"folderid"` + Status string `json:"status"` + Progress int `json:"progress"` + Retries int `json:"retries"` + Link string `json:"link"` + LinkID string `json:"linkid"` +} diff --git a/drivers/streamtape/util.go b/drivers/streamtape/util.go new file mode 100644 index 00000000000..51ad73d59bb --- /dev/null +++ b/drivers/streamtape/util.go @@ -0,0 +1,163 @@ +package streamtape + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" +) + +const apiBase = "https://api.streamtape.com" + +func (d *Streamtape) callAPI(ctx context.Context, endpoint string, params map[string]string, out any) error { + query := map[string]string{ + "login": d.APILogin, + "key": d.APIKey, + } + for k, v := range params { + if strings.TrimSpace(v) == "" { + continue + } + query[k] = v + } + + var resp apiResponse + r, err := base.RestyClient.R(). + SetContext(ctx). + SetQueryParams(query). + SetResult(&resp). + Get(apiBase + endpoint) + if err != nil { + return err + } + if r.StatusCode() != http.StatusOK { + return fmt.Errorf("streamtape http error: %d", r.StatusCode()) + } + if resp.Status != 200 { + return fmt.Errorf("streamtape api error: status=%d msg=%s", resp.Status, resp.Msg) + } + if out == nil || len(resp.Result) == 0 || string(resp.Result) == "null" { + return nil + } + if err := json.Unmarshal(resp.Result, out); err != nil { + return fmt.Errorf("decode streamtape result failed: %w", err) + } + return nil +} + +func folderIDFromObjID(id string) string { + if id == "" || id == "0" || id == "/" { + return "0" + } + if strings.HasPrefix(id, "d:") { + return strings.TrimPrefix(id, "d:") + } + return id +} + +func fileIDFromObjID(id string) string { + if strings.HasPrefix(id, "f:") { + return strings.TrimPrefix(id, "f:") + } + return id +} + +func encodeFolderID(id string) string { + if id == "" || id == "0" || id == "/" { + return "d:0" + } + return "d:" + id +} + +func encodeFileID(id string) string { + if strings.HasPrefix(id, "f:") { + return id + } + return "f:" + id +} + +func extractFileIDFromLink(link string) string { + if link == "" { + return "" + } + u, err := url.Parse(link) + if err != nil { + return "" + } + parts := strings.Split(strings.Trim(path.Clean(u.Path), "/"), "/") + for i := 0; i < len(parts)-1; i++ { + if parts[i] == "v" { + return parts[i+1] + } + } + return "" +} + +func buildFileObj(f fileItem) model.Obj { + id := f.LinkID + if id == "" { + id = extractFileIDFromLink(f.Link) + } + mod := time.Now() + if f.CreatedAt > 0 { + mod = time.Unix(f.CreatedAt, 0) + } + return &model.Object{ + ID: encodeFileID(id), + Name: f.Name, + Size: f.Size, + Modified: mod, + IsFolder: false, + } +} + +func extractFileIDFromUploadBody(body []byte) string { + if len(body) == 0 { + return "" + } + + var resp apiResponse + if err := json.Unmarshal(body, &resp); err != nil { + return "" + } + if resp.Status != 200 || len(resp.Result) == 0 { + return "" + } + + var result map[string]any + if err := json.Unmarshal(resp.Result, &result); err != nil { + return "" + } + for _, key := range []string{"file", "fileid", "id", "linkid"} { + if v, ok := result[key]; ok { + if s, ok := v.(string); ok && s != "" { + return s + } + } + } + return "" +} + +const remoteUploadPrefix = "ru:" + +func encodeRemoteUploadID(id string) string { + return remoteUploadPrefix + id +} + +func remoteUploadIDFromObjID(id string) string { + if strings.HasPrefix(id, remoteUploadPrefix) { + return strings.TrimPrefix(id, remoteUploadPrefix) + } + return "" +} + +func isRemoteUploadID(id string) bool { + return strings.HasPrefix(id, remoteUploadPrefix) +} diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go new file mode 100644 index 00000000000..175a558dfcc --- /dev/null +++ b/drivers/strm/driver.go @@ -0,0 +1,283 @@ +package strm + +import ( + "context" + "errors" + stdpath "path" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + log "github.com/sirupsen/logrus" +) + +type Strm struct { + model.Storage + Addition + + aliases map[string][]string + autoFlatten bool + singleRootKey string + + mediaExtSet map[string]struct{} + downloadExtSet map[string]struct{} + normalizedMode string + normalizedPrefix string +} + +func (d *Strm) Config() driver.Config { + return config +} + +func (d *Strm) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Strm) Init(ctx context.Context) error { + if strings.TrimSpace(d.Paths) == "" { + return errors.New("paths is required") + } + if d.SaveStrmToLocal && strings.TrimSpace(d.SaveStrmLocalPath) == "" { + return errors.New("SaveStrmLocalPath is required") + } + + d.aliases = parseAliases(d.Paths) + if len(d.aliases) == 0 { + return errors.New("no valid path mapping found") + } + + d.autoFlatten = len(d.aliases) == 1 + d.singleRootKey = "" + if d.autoFlatten { + for k := range d.aliases { + d.singleRootKey = k + } + } + + d.mediaExtSet = parseExtSet(defaultIfEmpty(d.FilterFileTypes, defaultMediaExt)) + d.downloadExtSet = parseExtSet(defaultIfEmpty(d.DownloadFileTypes, defaultDownloadExt)) + d.normalizedPrefix = normalizePrefix(defaultIfEmpty(d.PathPrefix, "/d")) + d.normalizedMode = normalizeSaveMode(d.SaveLocalMode) + + if d.Version != 5 { + d.FilterFileTypes = mergeDefaultExtCSV(d.FilterFileTypes, defaultMediaExt) + d.DownloadFileTypes = mergeDefaultExtCSV(d.DownloadFileTypes, defaultDownloadExt) + d.PathPrefix = "/d" + d.Version = 5 + } + if d.SaveLocalMode == "" { + d.SaveLocalMode = SaveLocalInsertMode + } + if d.SignExpireHours < 0 { + d.SignExpireHours = 0 + } + if d.RotateSignNow { + d.RotateSignNow = false + op.MustSaveDriverStorage(d) + if d.SaveStrmToLocal && strings.TrimSpace(d.SaveStrmLocalPath) != "" { + go func() { + log.Infof("strm: start rotating signs for [%s]", d.MountPath) + d.rotateAllLocal(context.Background()) + log.Infof("strm: finished rotating signs for [%s]", d.MountPath) + }() + } + } + return nil +} + +func (d *Strm) Drop(ctx context.Context) error { + d.aliases = nil + d.mediaExtSet = nil + d.downloadExtSet = nil + return nil +} + +func (Addition) GetRootPath() string { + return "/" +} + +func (d *Strm) Get(ctx context.Context, path string) (model.Obj, error) { + path = cleanPath(path) + root, sub := d.splitVirtualPath(path) + targets, ok := d.aliases[root] + if !ok { + return nil, errs.ObjectNotFound + } + + for _, targetRoot := range targets { + realPath := stdpath.Join(targetRoot, sub) + obj, err := fs.Get(ctx, realPath, &fs.GetArgs{NoLog: true}) + if err != nil { + continue + } + if obj.IsDir() { + return wrapObj(path, obj, 0), nil + } + return wrapObj(realPath, obj, obj.GetSize()), nil + } + + if strings.HasSuffix(strings.ToLower(path), ".strm") { + return nil, errs.NotSupport + } + return nil, errs.ObjectNotFound +} + +func (d *Strm) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + virtualDir := cleanPath(dir.GetPath()) + if virtualDir == "/" && !d.autoFlatten { + objs := d.listVirtualRoots() + d.syncLocalDir(ctx, virtualDir, objs) + return objs, nil + } + + root, sub := d.splitVirtualPath(virtualDir) + targets, ok := d.aliases[root] + if !ok { + return nil, errs.ObjectNotFound + } + + out := make([]model.Obj, 0) + for _, targetRoot := range targets { + realDir := stdpath.Join(targetRoot, sub) + objs, err := fs.List(ctx, realDir, &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) + if err != nil { + continue + } + out = append(out, d.mapListedObjects(ctx, realDir, objs)...) + } + + d.syncLocalDir(ctx, virtualDir, out) + return out, nil +} + +func (d *Strm) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.GetID() == "strm" { + line := d.buildStrmLine(ctx, file.GetPath()) + return &model.Link{MFile: model.NewNopMFile(strings.NewReader(line + "\n"))}, nil + } + return d.linkRealFile(ctx, file.GetPath(), args) +} + +func (d *Strm) listVirtualRoots() []model.Obj { + objs := make([]model.Obj, 0, len(d.aliases)) + for k := range d.aliases { + objs = append(objs, &model.Object{ + Path: "/" + k, + Name: k, + IsFolder: true, + Modified: d.Modified, + }) + } + return objs +} + +func (d *Strm) rotateAllLocal(ctx context.Context) { + for alias, roots := range d.aliases { + virtualRoot := "/" + if !d.autoFlatten { + virtualRoot = "/" + alias + } + for _, realRoot := range roots { + d.walkAndSync(ctx, virtualRoot, realRoot) + } + } +} + +func (d *Strm) walkAndSync(ctx context.Context, virtualDir, realDir string) { + objs, err := fs.List(ctx, realDir, &fs.ListArgs{NoLog: true, Refresh: true}) + if err != nil { + log.Warnf("strm: rotate list failed %s: %v", realDir, err) + return + } + mapped := d.mapListedObjects(ctx, realDir, objs) + d.syncLocalDirWithMode(ctx, virtualDir, mapped, SaveLocalUpdateMode) + for _, obj := range objs { + if !obj.IsDir() { + continue + } + childVirtual := stdpath.Join(virtualDir, obj.GetName()) + childReal := stdpath.Join(realDir, obj.GetName()) + d.walkAndSync(ctx, childVirtual, childReal) + } +} + +func (d *Strm) mapListedObjects(ctx context.Context, realDir string, listed []model.Obj) []model.Obj { + ret := make([]model.Obj, 0, len(listed)) + for _, obj := range listed { + if obj.IsDir() { + ret = append(ret, &model.Object{ + Name: obj.GetName(), + Path: "", + IsFolder: true, + Modified: obj.ModTime(), + }) + continue + } + + realPath := stdpath.Join(realDir, obj.GetName()) + ext := fileExt(obj.GetName()) + + if _, ok := d.downloadExtSet[ext]; ok { + ret = append(ret, d.cloneWithPath(obj, realPath, obj.GetName(), "", obj.GetSize())) + continue + } + if _, ok := d.mediaExtSet[ext]; ok { + strmName := strings.TrimSuffix(obj.GetName(), stdpath.Ext(obj.GetName())) + ".strm" + size := int64(len(d.buildStrmLine(ctx, realPath)) + 1) + ret = append(ret, d.cloneWithPath(obj, realPath, strmName, "strm", size)) + } + } + return ret +} + +func (d *Strm) cloneWithPath(src model.Obj, realPath, name, id string, size int64) model.Obj { + baseObj := model.Object{ + ID: id, + Path: realPath, + Name: name, + Size: size, + Modified: src.ModTime(), + IsFolder: src.IsDir(), + } + thumb, ok := model.GetThumb(src) + if !ok { + return &baseObj + } + return &model.ObjThumb{Object: baseObj, Thumbnail: model.Thumbnail{Thumbnail: thumb}} +} + +func (d *Strm) splitVirtualPath(path string) (string, string) { + if d.autoFlatten { + return d.singleRootKey, path + } + trimmed := strings.TrimPrefix(path, "/") + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) == 1 { + return parts[0], "" + } + return parts[0], parts[1] +} + +func cleanPath(path string) string { + if path == "" { + return "/" + } + return filepath.ToSlash(stdpath.Clean("/" + strings.TrimPrefix(path, "/"))) +} + +func wrapObj(path string, src model.Obj, size int64) model.Obj { + return &model.Object{ + Path: path, + Name: src.GetName(), + Size: size, + Modified: src.ModTime(), + IsFolder: src.IsDir(), + HashInfo: src.GetHash(), + } +} + +var _ driver.Driver = (*Strm)(nil) diff --git a/drivers/strm/hook.go b/drivers/strm/hook.go new file mode 100644 index 00000000000..6fad6993c67 --- /dev/null +++ b/drivers/strm/hook.go @@ -0,0 +1,3 @@ +package strm + +// Local sync is triggered during STRM directory listing. diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go new file mode 100644 index 00000000000..803ed035762 --- /dev/null +++ b/drivers/strm/meta.go @@ -0,0 +1,44 @@ +package strm + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +const ( + SaveLocalInsertMode = "insert" + SaveLocalUpdateMode = "update" + SaveLocalSyncMode = "sync" +) + +type Addition struct { + Paths string `json:"paths" required:"true" type:"text"` + SiteUrl string `json:"siteUrl" type:"text" required:"false" help:"The prefix URL of generated strm file"` + PathPrefix string `json:"PathPrefix" type:"text" required:"false" default:"/d" help:"Path prefix in strm content"` + DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Extensions to download as local files"` + FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Extensions to expose as .strm"` + EncodePath bool `json:"encodePath" default:"true" required:"true" help:"Encode path in strm content"` + WithoutUrl bool `json:"withoutUrl" default:"false" help:"Generate path-only strm content"` + WithSign bool `json:"withSign" default:"false" help:"Append sign query to generated URL"` + SignExpireHours int `json:"SignExpireHours" type:"number" default:"0" help:"Driver-level sign expiration in hours. 0 uses global link_expiration"` + RotateSignNow bool `json:"RotateSignNow" type:"bool" default:"false" help:"Set true and save to rotate signs now (rewrite local STRM), then auto reset to false"` + SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"Save generated files to local disk"` + SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"Local path for generated files"` + SaveLocalMode string `json:"SaveLocalMode" type:"select" help:"Local save mode" options:"insert,update,sync" default:"insert"` + Version int +} + +var config = driver.Config{ + Name: "Strm", + LocalSort: true, + OnlyProxy: true, + NoCache: true, + NoUpload: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Strm{Addition: Addition{EncodePath: true}} + }) +} diff --git a/drivers/strm/util.go b/drivers/strm/util.go new file mode 100644 index 00000000000..2d640bc32fb --- /dev/null +++ b/drivers/strm/util.go @@ -0,0 +1,317 @@ +package strm + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + stdpath "path" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + pkgerr "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +const ( + defaultMediaExt = "mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" + defaultDownloadExt = "ass,srt,vtt,sub,strm" +) + +func parseAliases(raw string) map[string][]string { + aliases := map[string][]string{} + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + name, target := parseAliasLine(line) + aliases[name] = append(aliases[name], cleanPath(target)) + } + return aliases +} + +func parseAliasLine(line string) (string, string) { + if strings.Contains(line, ":") { + parts := strings.SplitN(line, ":", 2) + if !strings.Contains(parts[0], "/") { + return parts[0], parts[1] + } + } + return stdpath.Base(line), line +} + +func parseExtSet(csv string) map[string]struct{} { + ret := map[string]struct{}{} + for _, part := range strings.Split(csv, ",") { + ext := normalizeExt(part) + if ext != "" { + ret[ext] = struct{}{} + } + } + return ret +} + +func mergeDefaultExtCSV(csv, defaults string) string { + base := parseExtSet(csv) + for ext := range parseExtSet(defaults) { + base[ext] = struct{}{} + } + keys := make([]string, 0, len(base)) + for k := range base { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ",") +} + +func normalizeExt(ext string) string { + ext = strings.ToLower(strings.TrimSpace(ext)) + ext = strings.TrimPrefix(ext, ".") + return ext +} + +func fileExt(name string) string { + return normalizeExt(stdpath.Ext(name)) +} + +func defaultIfEmpty(v, fallback string) string { + if strings.TrimSpace(v) == "" { + return fallback + } + return v +} + +func normalizeSaveMode(mode string) string { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "sync": + return SaveLocalSyncMode + case "update": + return SaveLocalUpdateMode + case "insert", "missing": + return SaveLocalInsertMode + default: + return SaveLocalInsertMode + } +} + +func normalizePrefix(prefix string) string { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + return "/d" + } + if !strings.HasPrefix(prefix, "/") { + prefix = "/" + prefix + } + return prefix +} + +func (d *Strm) buildStrmLine(ctx context.Context, realPath string) string { + pathPart := realPath + if d.EncodePath { + pathPart = utils.EncodePath(pathPart, true) + } + if d.WithSign { + sep := "?" + if strings.Contains(pathPart, "?") { + sep = "&" + } + pathPart += sep + "sign=" + d.generateSign(realPath) + } + joined := stdpath.Join(d.normalizedPrefix, pathPart) + if !strings.HasPrefix(joined, "/") { + joined = "/" + joined + } + if d.WithoutUrl { + return joined + } + baseURL := strings.TrimSpace(d.SiteUrl) + if baseURL == "" { + if c, ok := ctx.(*gin.Context); ok { + baseURL = common.GetApiUrl(c.Request) + } else { + baseURL = common.GetApiUrl(nil) + } + } + baseURL = strings.TrimSuffix(baseURL, "/") + return baseURL + joined +} + +func (d *Strm) linkRealFile(ctx context.Context, realPath string, args model.LinkArgs) (*model.Link, error) { + storage, actualPath, err := op.GetStorageAndActualPath(realPath) + if err != nil { + return nil, err + } + if !args.Redirect { + link, _, linkErr := op.Link(ctx, storage, actualPath, args) + return link, linkErr + } + obj, err := fs.Get(ctx, realPath, &fs.GetArgs{NoLog: true}) + if err != nil { + return nil, err + } + if common.ShouldProxy(storage, obj.GetName()) { + api := common.GetApiUrl(args.HttpReq) + if api == "" { + api = strings.TrimSuffix(strings.TrimSpace(d.SiteUrl), "/") + } + if api == "" { + api = common.GetApiUrl(nil) + } + return &model.Link{URL: fmt.Sprintf("%s/p%s?sign=%s", api, utils.EncodePath(realPath, true), d.generateSign(realPath))}, nil + } + link, _, linkErr := op.Link(ctx, storage, actualPath, args) + return link, linkErr +} + +func (d *Strm) syncLocalDir(ctx context.Context, virtualDir string, objs []model.Obj) { + d.syncLocalDirWithMode(ctx, virtualDir, objs, d.normalizedMode) +} + +func (d *Strm) syncLocalDirWithMode(ctx context.Context, virtualDir string, objs []model.Obj, mode string) { + if !d.SaveStrmToLocal || strings.TrimSpace(d.SaveStrmLocalPath) == "" { + return + } + baseDir := filepath.Clean(d.SaveStrmLocalPath) + localDir := baseDir + if virtualDir != "/" { + localDir = filepath.Join(baseDir, filepath.FromSlash(strings.TrimPrefix(virtualDir, "/"))) + } + if err := os.MkdirAll(localDir, 0o755); err != nil { + log.Warnf("strm: mkdir failed %s: %v", localDir, err) + return + } + + expected := map[string]bool{} + for _, obj := range objs { + name := obj.GetName() + expected[name] = obj.IsDir() + localPath := filepath.Join(localDir, name) + if obj.IsDir() { + _ = os.MkdirAll(localPath, 0o755) + continue + } + payload, err := d.localPayload(ctx, obj) + if err != nil { + log.Warnf("strm: build local payload failed %s: %v", localPath, err) + continue + } + if err = d.writeLocal(localPath, payload, mode); err != nil { + log.Warnf("strm: write local failed %s: %v", localPath, err) + } + } + + if mode == SaveLocalSyncMode { + d.syncDeleteExtras(localDir, expected) + } +} + +func (d *Strm) localPayload(ctx context.Context, obj model.Obj) ([]byte, error) { + if obj.GetID() == "strm" { + return []byte(d.buildStrmLine(ctx, obj.GetPath()) + "\n"), nil + } + link, err := d.linkRealFile(ctx, obj.GetPath(), model.LinkArgs{Redirect: true}) + if err != nil { + return nil, err + } + return readLinkBytes(ctx, link) +} + +func readLinkBytes(ctx context.Context, link *model.Link) ([]byte, error) { + if link.MFile != nil { + defer link.MFile.Close() + return io.ReadAll(link.MFile) + } + if link.RangeReadCloser != nil { + rc, err := link.RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) + if err == nil && rc != nil { + defer rc.Close() + return io.ReadAll(rc) + } + } + if link.URL == "" { + return nil, fmt.Errorf("empty link") + } + url := link.URL + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + api := common.GetApiUrl(nil) + if api == "" { + return nil, fmt.Errorf("relative url without site url: %s", url) + } + url = strings.TrimSuffix(api, "/") + url + } + res, err := base.RestyClient.R().SetContext(ctx).SetDoNotParseResponse(true).Get(url) + if err != nil { + return nil, err + } + defer res.RawBody().Close() + if res.StatusCode() >= http.StatusBadRequest { + return nil, fmt.Errorf("read url failed: status=%d", res.StatusCode()) + } + return io.ReadAll(res.RawBody()) +} + +func (d *Strm) writeLocal(path string, payload []byte, mode string) error { + if mode == SaveLocalInsertMode && utils.Exists(path) { + return nil + } + if st, err := os.Stat(path); err == nil && st.IsDir() { + if mode != SaveLocalSyncMode { + return nil + } + if err = os.RemoveAll(path); err != nil { + return err + } + } + if mode != SaveLocalInsertMode { + if old, err := os.ReadFile(path); err == nil { + if bytes.Equal(old, payload) { + return nil + } + } + } + f, err := utils.CreateNestedFile(path) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(payload) + return err +} + +func (d *Strm) syncDeleteExtras(localDir string, expected map[string]bool) { + entries, err := os.ReadDir(localDir) + if err != nil { + if pkgerr.Cause(err) != os.ErrNotExist { + log.Warnf("strm: read local dir failed %s: %v", localDir, err) + } + return + } + for _, e := range entries { + expectDir, ok := expected[e.Name()] + full := filepath.Join(localDir, e.Name()) + if !ok || expectDir != e.IsDir() { + _ = os.RemoveAll(full) + } + } +} + +func (d *Strm) generateSign(path string) string { + if d.SignExpireHours > 0 { + return sign.WithDuration(path, time.Duration(d.SignExpireHours)*time.Hour) + } + return sign.Sign(path) +} diff --git a/drivers/webdav/meta.go b/drivers/webdav/meta.go index 2294d482a6e..d66499bc3f9 100644 --- a/drivers/webdav/meta.go +++ b/drivers/webdav/meta.go @@ -11,7 +11,6 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` driver.RootPath - TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" default:"false"` } var config = driver.Config{ diff --git a/drivers/webdav/util.go b/drivers/webdav/util.go index 23dc909ff88..dfd6e5b2457 100644 --- a/drivers/webdav/util.go +++ b/drivers/webdav/util.go @@ -6,6 +6,7 @@ import ( "net/http/cookiejar" "github.com/alist-org/alist/v3/drivers/webdav/odrvcookie" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/gowebdav" ) @@ -20,7 +21,7 @@ func (d *WebDav) setClient() error { c := gowebdav.NewClient(d.Address, d.Username, d.Password) c.SetTransport(&http.Transport{ Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: d.TlsInsecureSkipVerify}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, }) if d.isSharepoint() { cookie, err := odrvcookie.GetCookie(d.Username, d.Password, d.Address) diff --git a/drivers/wukong/driver.go b/drivers/wukong/driver.go new file mode 100644 index 00000000000..cb5dee1ae40 --- /dev/null +++ b/drivers/wukong/driver.go @@ -0,0 +1,1116 @@ +package wukong + +import ( + "context" + "crypto/hmac" + "crypto/md5" + crand "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash/crc32" + "io" + "net/http" + "net/url" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" +) + +const ( + wukongBaseURL = "https://api.wkbrowser.com" + webReferer = "https://pan.wkbrowser.com/" + vodBaseURL = "https://vod.bytedanceapi.com" + vodRegion = "cn-north-1" + vodService = "vod" + videoSpaceName = "wukong_netdisk_ugc" + minUploadSubmitSuccess = 2000 + multipartChunkSize = int64(5 * 1024 * 1024) +) + +type Wukong struct { + model.Storage + Addition + client *resty.Client +} + +func (d *Wukong) Config() driver.Config { + return config +} + +func (d *Wukong) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Wukong) Init(ctx context.Context) error { + d.client = base.NewRestyClient(). + SetBaseURL(wukongBaseURL). + SetHeader("accept", "application/json, text/plain, */*"). + SetHeader("content-type", "application/json"). + SetHeader("referer", webReferer). + SetHeader("origin", "https://pan.wkbrowser.com") + if d.Cookie != "" { + d.client.SetHeader("cookie", d.Cookie) + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + if strings.TrimSpace(d.Aid) == "" { + d.Aid = "590353" + } + if strings.TrimSpace(d.Language) == "" { + d.Language = "zh" + } + if d.PageSize <= 0 { + d.PageSize = 100 + } + return nil +} + +func (d *Wukong) Drop(ctx context.Context) error { + return nil +} + +func (d *Wukong) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + fatherID := dir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + offset := 0 + limit := d.PageSize + objs := make([]model.Obj, 0) + for { + var resp filterFileResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "offset": strconv.Itoa(offset), + "limit": strconv.Itoa(limit), + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "father_id": asIDValue(fatherID), + "filter_type": 2, + "is_desc": 1, + "file_type": 0, + }). + SetResult(&resp). + Post("/netdisk/user_file/filter_file") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong list failed: code=%d message=%s", resp.Code, resp.Message) + } + + for _, item := range resp.Data.FileList { + objs = append(objs, &model.Object{ + ID: strconv.FormatInt(item.FileID, 10), + Path: strconv.FormatInt(item.FatherID, 10), + Name: item.FileName, + Size: item.Size, + Modified: parseUnix(item.UpdatedAt), + Ctime: parseUnix(item.CreatedAt), + IsFolder: item.IsDirectory == 1, + }) + } + + if !hasMore(resp.Data.HasMore) || len(resp.Data.FileList) == 0 { + break + } + offset += len(resp.Data.FileList) + } + return objs, nil +} + +func (d *Wukong) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + fileID := file.GetID() + if fileID == "" { + return nil, errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(fileID)}, + }). + SetResult(&resp). + Post("/netdisk/user_file/detail") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong detail failed: code=%d message=%s", resp.Code, resp.Message) + } + + url := extractDetailMainURL(resp.Data) + if url == "" { + url = extractURL(resp.Data) + } + if url == "" { + return nil, errs.NotImplement + } + + return &model.Link{ + URL: url, + Header: http.Header{ + "Referer": []string{webReferer}, + "Cookie": []string{d.Cookie}, + }, + }, nil +} + +func (d *Wukong) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + fatherID := parentDir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "father_id": asIDValue(fatherID), + "file_name": dirName, + }). + SetResult(&resp). + Post("/netdisk/user_file/create_directory") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong create directory failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + srcID := srcObj.GetID() + if srcID == "" { + return errors.New("missing source file id") + } + + dstID := dstDir.GetID() + if dstID == "" { + dstID = d.RootFolderID + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(srcID)}, + "new_father_id": asIDValue(dstID), + }). + SetResult(&resp). + Post("/netdisk/user_file/move_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong move failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + srcID := srcObj.GetID() + if srcID == "" { + return errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id": asIDValue(srcID), + "new_name": newName, + }). + SetResult(&resp). + Post("/netdisk/user_file/rename_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong rename failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Remove(ctx context.Context, obj model.Obj) error { + fileID := obj.GetID() + if fileID == "" { + return errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(fileID)}, + }). + SetResult(&resp). + Post("/netdisk/user_file/delete_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong delete failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + fatherID := dstDir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return err + } + defer tempFile.Close() + if _, err = tempFile.Seek(0, io.SeekStart); err != nil { + return err + } + + md5Hex, crc32Hex, err := calcFileMD5AndCRC32(tempFile) + if err != nil { + return err + } + ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(file.GetName())), ".") + fileType := detectWukongFileType(file.GetMimetype(), file.GetName()) + size := file.GetSize() + up(5) + + uploadType := detectUploadType(file.GetMimetype(), file.GetName()) + authToken, err := d.getUploadAuthToken(ctx, uploadType) + if err != nil { + return err + } + up(10) + + candidates, err := d.getUploadCandidates(ctx, authToken) + if err != nil { + return err + } + bestHosts := collectCandidateHosts(candidates) + if len(bestHosts) == 0 { + return errors.New("wukong upload candidates is empty") + } + up(20) + + applyResp, err := d.applyUploadInner(ctx, authToken, uploadType, size, strings.Join(bestHosts, ",")) + if err != nil { + return err + } + if len(applyResp.Result.InnerUploadAddress.UploadNodes) == 0 || + len(applyResp.Result.InnerUploadAddress.UploadNodes[0].StoreInfos) == 0 { + return errors.New("wukong apply upload inner returns empty upload node") + } + node := applyResp.Result.InnerUploadAddress.UploadNodes[0] + store := node.StoreInfos[0] + up(30) + + if size > multipartChunkSize { + if err = d.uploadToTOSMultipart(ctx, node.UploadHost, store.StoreURI, store.Auth, getStorageUserID(store.StorageHeader), tempFile, size, up); err != nil { + return err + } + } else { + if _, err = tempFile.Seek(0, io.SeekStart); err != nil { + return err + } + reader := &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{Reader: tempFile, Size: size}, + UpdateProgress: func(percent float64) { + up(30 + percent*0.5) + }, + } + if err = d.uploadToTOS(ctx, node.UploadHost, store.StoreURI, store.Auth, getStorageUserID(store.StorageHeader), crc32Hex, file.GetName(), reader, size); err != nil { + return err + } + } + up(85) + + videoVid, err := d.commitUploadInner(ctx, authToken, chooseCommitSpace(uploadType, authToken.SpaceName), node.SessionKey) + if err != nil { + return err + } + up(92) + + if fileType == 3000 && videoVid == "" { + return errors.New("wukong video upload missing vid in commit response") + } + if err = d.uploadSubmit(ctx, fatherID, file.GetName(), ext, fileType, size, md5Hex, store.StoreURI, videoVid); err != nil { + return err + } + up(100) + return nil +} + +func (d *Wukong) getUploadAuthToken(ctx context.Context, uploadType string) (*uploadAuthTokenResp, error) { + var resp uploadAuthTokenResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "upload_source": uploadSourceByType(uploadType), + "type": uploadType, + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetResult(&resp). + Get("/toutiao/upload/auth_token/v1/") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong get upload auth token failed: code=%d message=%s", resp.Code, resp.Message) + } + return &resp, nil +} + +func (d *Wukong) getUploadCandidates(ctx context.Context, auth *uploadAuthTokenResp) (*getUploadCandidatesResp, error) { + q := map[string]string{ + "Action": "GetUploadCandidates", + "Version": "2020-11-19", + "SpaceName": videoSpaceName, + } + var resp getUploadCandidatesResp + if err := d.vodRequest(ctx, http.MethodGet, q, nil, auth, &resp); err != nil { + return nil, err + } + if resp.ResponseMetadata.Error.Code != "" { + return nil, fmt.Errorf("wukong get upload candidates failed: %s", resp.ResponseMetadata.Error.Message) + } + return &resp, nil +} + +func (d *Wukong) applyUploadInner(ctx context.Context, auth *uploadAuthTokenResp, uploadType string, fileSize int64, bestHosts string) (*applyUploadInnerResp, error) { + spaceName := auth.SpaceName + if uploadType == "video" { + spaceName = videoSpaceName + } + q := map[string]string{ + "Action": "ApplyUploadInner", + "Version": "2020-11-19", + "SpaceName": spaceName, + "FileType": uploadType, + "IsInner": "1", + "ClientBestHosts": bestHosts, + "NeedFallback": "true", + "FileSize": strconv.FormatInt(fileSize, 10), + "s": randomString(8), + } + var resp applyUploadInnerResp + if err := d.vodRequest(ctx, http.MethodGet, q, nil, auth, &resp); err != nil { + return nil, err + } + if resp.ResponseMetadata.Error.Code != "" { + return nil, fmt.Errorf("wukong apply upload inner failed: %s", resp.ResponseMetadata.Error.Message) + } + return &resp, nil +} + +func (d *Wukong) commitUploadInner(ctx context.Context, auth *uploadAuthTokenResp, spaceName, sessionKey string) (string, error) { + q := map[string]string{ + "Action": "CommitUploadInner", + "Version": "2020-11-19", + "SpaceName": spaceName, + } + body, _ := json.Marshal(map[string]any{ + "SessionKey": sessionKey, + "Functions": []any{}, + }) + var resp commitUploadInnerResp + if err := d.vodRequest(ctx, http.MethodPost, q, body, auth, &resp); err != nil { + return "", err + } + if resp.ResponseMetadata.Error.Code != "" { + return "", fmt.Errorf("wukong commit upload inner failed: %s", resp.ResponseMetadata.Error.Message) + } + if len(resp.Result.Results) > 0 { + status := resp.Result.Results[0].URIStatus + if status != 0 && status != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong commit upload inner failed: uri_status=%d", status) + } + } + return extractVideoVid(&resp), nil +} + +func (d *Wukong) uploadSubmit(ctx context.Context, fatherID, fileName, ext string, fileType int, size int64, md5Hex, storeURI, videoVid string) error { + var resp uploadSubmitResp + body := map[string]any{ + "base_info": map[string]any{ + "father_id": asIDValue(fatherID), + "file_type": fileType, + "size": size, + "extension": ext, + "file_name": fileName, + "is_directory": 0, + "md5": md5Hex, + "slice_md5": md5Hex, + }, + } + switch fileType { + case 3000: + if videoVid != "" { + body["video_info"] = map[string]any{"vid": videoVid} + } + case 2000: + body["image_info"] = map[string]any{"uri": storeURI} + default: + body["general_info"] = map[string]any{"key": storeURI} + } + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(body). + SetResult(&resp). + Post("/netdisk/upload_submit/") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong upload submit failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func extractVideoVid(resp *commitUploadInnerResp) string { + for _, item := range resp.Result.Results { + if item.Vid != "" { + return item.Vid + } + } + if resp.Result.PluginResult != nil { + if vid := findStringByKey(resp.Result.PluginResult, "Vid"); vid != "" { + return vid + } + if vid := findStringByKey(resp.Result.PluginResult, "vid"); vid != "" { + return vid + } + } + return "" +} + +func findStringByKey(v any, key string) string { + switch cur := v.(type) { + case map[string]any: + if val, ok := cur[key]; ok { + if s, ok := val.(string); ok && s != "" { + return s + } + } + for _, child := range cur { + if s := findStringByKey(child, key); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findStringByKey(child, key); s != "" { + return s + } + } + } + return "" +} + +func (d *Wukong) uploadToTOS(ctx context.Context, host, storeURI, auth, storageUser, crc32Hex, fileName string, body io.Reader, size int64) error { + var resp tosUploadResp + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Crc32", crc32Hex). + SetHeader("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, url.QueryEscape(fileName))). + SetHeader("Content-Length", strconv.FormatInt(size, 10)). + SetBody(body). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + _, err := req.Post(uploadURL) + if err != nil { + return err + } + if resp.Code != minUploadSubmitSuccess { + return fmt.Errorf("wukong upload to tos failed: code=%d message=%s", resp.Code, resp.Message) + } + if resp.Data.Crc32 != "" && !strings.EqualFold(resp.Data.Crc32, crc32Hex) { + return fmt.Errorf("wukong upload to tos crc32 mismatch: local=%s remote=%s", crc32Hex, resp.Data.Crc32) + } + return nil +} + +func (d *Wukong) uploadToTOSMultipart(ctx context.Context, host, storeURI, auth, storageUser string, tempFile model.File, size int64, up driver.UpdateProgress) error { + uploadID, err := d.initMultipartUpload(ctx, host, storeURI, auth, storageUser) + if err != nil { + return err + } + + totalParts := int((size + multipartChunkSize - 1) / multipartChunkSize) + if totalParts <= 0 { + return errors.New("invalid multipart parts") + } + parts := make([]string, 0, totalParts) + for i := 0; i < totalParts; i++ { + partNumber := i + 1 + offset := int64(i) * multipartChunkSize + partSize := multipartChunkSize + if remain := size - offset; remain < partSize { + partSize = remain + } + buf := make([]byte, partSize) + n, readErr := tempFile.ReadAt(buf, offset) + if readErr != nil && readErr != io.EOF { + return readErr + } + buf = buf[:n] + crc32Hex := fmt.Sprintf("%08x", crc32.ChecksumIEEE(buf)) + remoteCRC32, err := d.uploadMultipartPart(ctx, host, storeURI, auth, storageUser, uploadID, partNumber, buf, crc32Hex) + if err != nil { + return err + } + if remoteCRC32 != "" && !strings.EqualFold(remoteCRC32, crc32Hex) { + return fmt.Errorf("multipart part crc32 mismatch: part=%d local=%s remote=%s", partNumber, crc32Hex, remoteCRC32) + } + parts = append(parts, fmt.Sprintf("%d:%s", partNumber, crc32Hex)) + up(30 + float64(partNumber)/float64(totalParts)*50) + } + + return d.finishMultipartUpload(ctx, host, storeURI, auth, storageUser, uploadID, strings.Join(parts, ",")) +} + +func (d *Wukong) initMultipartUpload(ctx context.Context, host, storeURI, auth, storageUser string) (string, error) { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetQueryParams(map[string]string{ + "uploadmode": "part", + "phase": "init", + }). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return "", err + } + if resp.Code != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong init multipart upload failed: code=%d message=%s", resp.Code, resp.Message) + } + if resp.Data.UploadID == "" { + return "", errors.New("wukong init multipart upload returns empty uploadid") + } + return resp.Data.UploadID, nil +} + +func (d *Wukong) uploadMultipartPart(ctx context.Context, host, storeURI, auth, storageUser, uploadID string, partNumber int, data []byte, crc32Hex string) (string, error) { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Crc32", crc32Hex). + SetHeader("Content-Length", strconv.Itoa(len(data))). + SetQueryParams(map[string]string{ + "uploadid": uploadID, + "part_number": strconv.Itoa(partNumber), + "phase": "transfer", + }). + SetBody(data). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return "", err + } + if resp.Code != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong multipart transfer failed: code=%d message=%s part=%d", resp.Code, resp.Message, partNumber) + } + return resp.Data.Crc32, nil +} + +func (d *Wukong) finishMultipartUpload(ctx context.Context, host, storeURI, auth, storageUser, uploadID, body string) error { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetQueryParams(map[string]string{ + "uploadid": uploadID, + "phase": "finish", + "uploadmode": "part", + }). + SetBody(body). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return err + } + if resp.Code != minUploadSubmitSuccess && resp.Code != 4024 { + return fmt.Errorf("wukong multipart finish failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) vodRequest(ctx context.Context, method string, query map[string]string, body []byte, auth *uploadAuthTokenResp, resp any) error { + reqURL := vodBaseURL + "/" + amzDate := time.Now().UTC().Format("20060102T150405Z") + dateStamp := amzDate[:8] + headers := map[string]string{ + "x-amz-date": amzDate, + "x-amz-security-token": auth.SessionToken, + } + if method == http.MethodPost { + headers["x-amz-content-sha256"] = hashSHA256Bytes(body) + } + authorization := buildVodAuthorization(method, "/", query, headers, body, auth, dateStamp) + + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Authorization", authorization). + SetHeader("x-amz-date", amzDate). + SetHeader("x-amz-security-token", auth.SessionToken). + SetQueryParams(query). + SetResult(resp) + if method == http.MethodPost { + req.SetHeader("x-amz-content-sha256", headers["x-amz-content-sha256"]) + req.SetHeader("Content-Type", "text/plain;charset=UTF-8") + req.SetBody(body) + } + _, err := req.Execute(method, reqURL) + return err +} + +func buildVodAuthorization(method, canonicalURI string, query map[string]string, headers map[string]string, body []byte, auth *uploadAuthTokenResp, dateStamp string) string { + canonicalQueryString := getCanonicalQueryStringFromMap(query) + canonicalHeaders, signedHeaders := getCanonicalHeaders(headers) + payloadHash := hashSHA256Bytes(body) + canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQueryString + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash + credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", dateStamp, vodRegion, vodService) + stringToSign := "AWS4-HMAC-SHA256\n" + headers["x-amz-date"] + "\n" + credentialScope + "\n" + hashSHA256String(canonicalRequest) + signingKey := getSigningKey(auth.SecretAccessKey, dateStamp, vodRegion, vodService) + signature := hmacSHA256Hex(signingKey, stringToSign) + return fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", auth.AccessKeyID, credentialScope, signedHeaders, signature) +} + +func getCanonicalQueryStringFromMap(query map[string]string) string { + if len(query) == 0 { + return "" + } + keys := make([]string, 0, len(query)) + for k := range query { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, awsURLEncode(k)+"="+awsURLEncode(query[k])) + } + return strings.Join(parts, "&") +} + +func getCanonicalHeaders(headers map[string]string) (string, string) { + keys := make([]string, 0, len(headers)) + for k := range headers { + keys = append(keys, strings.ToLower(k)) + } + sort.Strings(keys) + var h strings.Builder + for _, k := range keys { + h.WriteString(k) + h.WriteString(":") + h.WriteString(strings.TrimSpace(headers[k])) + h.WriteString("\n") + } + return h.String(), strings.Join(keys, ";") +} + +func awsURLEncode(s string) string { + s = url.QueryEscape(s) + return strings.ReplaceAll(s, "+", "%20") +} + +func hashSHA256Bytes(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func hashSHA256String(s string) string { + return hashSHA256Bytes([]byte(s)) +} + +func hmacSHA256(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + _, _ = h.Write([]byte(data)) + return h.Sum(nil) +} + +func hmacSHA256Hex(key []byte, data string) string { + return hex.EncodeToString(hmacSHA256(key, data)) +} + +func getSigningKey(secret, dateStamp, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secret), dateStamp) + kRegion := hmacSHA256(kDate, region) + kService := hmacSHA256(kRegion, service) + return hmacSHA256(kService, "aws4_request") +} + +func collectCandidateHosts(resp *getUploadCandidatesResp) []string { + seen := map[string]struct{}{} + hosts := make([]string, 0, len(resp.Result.Domains)) + add := func(domain vodDomain) { + if domain.Name == "" { + return + } + if _, ok := seen[domain.Name]; ok { + return + } + seen[domain.Name] = struct{}{} + hosts = append(hosts, domain.Name) + } + for _, candidate := range resp.Result.Candidates { + for _, domain := range candidate.Domains { + add(domain) + } + } + for _, domain := range resp.Result.Domains { + add(domain) + } + return hosts +} + +func getStorageUserID(header map[string]any) string { + if header == nil { + return "" + } + if s, ok := header["USER_ID"].(string); ok { + return s + } + if f, ok := header["USER_ID"].(float64); ok { + return strconv.FormatInt(int64(f), 10) + } + return "" +} + +func calcFileMD5AndCRC32(f model.File) (string, string, error) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", "", err + } + md5Hasher := md5.New() + crc := crc32.NewIEEE() + _, err := io.Copy(io.MultiWriter(md5Hasher, crc), f) + if err != nil { + return "", "", err + } + if _, err = f.Seek(0, io.SeekStart); err != nil { + return "", "", err + } + return hex.EncodeToString(md5Hasher.Sum(nil)), fmt.Sprintf("%08x", crc.Sum32()), nil +} + +func detectWukongFileType(mimetype, fileName string) int { + lowerName := strings.ToLower(fileName) + switch { + case strings.HasPrefix(mimetype, "image/"): + return 2000 + case strings.HasPrefix(mimetype, "video/"), strings.HasSuffix(lowerName, ".flv"), strings.HasSuffix(lowerName, ".mkv"): + return 3000 + case strings.HasPrefix(mimetype, "audio/"), strings.HasSuffix(lowerName, ".mp3"), strings.HasSuffix(lowerName, ".m4a"), strings.HasSuffix(lowerName, ".wav"): + return 4000 + case strings.HasSuffix(lowerName, ".zip"), strings.HasSuffix(lowerName, ".rar"), strings.HasSuffix(lowerName, ".7z"), strings.HasSuffix(lowerName, ".tar"), strings.HasSuffix(lowerName, ".gz"), strings.HasSuffix(lowerName, ".tgz"): + return 6000 + default: + return 5000 + } +} + +func randomString(n int) string { + const letters = "abcdefghijklmnopqrstuvwxyz0123456789" + if n <= 0 { + return "" + } + buf := make([]byte, n) + if _, err := crand.Read(buf); err == nil { + for i := range buf { + buf[i] = letters[int(buf[i])%len(letters)] + } + return string(buf) + } + + now := uint64(time.Now().UnixNano()) + b := make([]byte, n) + for i := range b { + now = now*6364136223846793005 + 1 + b[i] = letters[int(now%uint64(len(letters)))] + } + return string(b) +} + +func uploadSourceByType(uploadType string) string { + switch uploadType { + case "video": + return "10150001" + case "image": + return "20150001" + default: + return "50150001" + } +} + +func detectUploadType(mimetype, fileName string) string { + lowerName := strings.ToLower(fileName) + if strings.HasPrefix(mimetype, "video/") || strings.HasPrefix(mimetype, "audio/") || + strings.HasSuffix(lowerName, ".flv") || strings.HasSuffix(lowerName, ".mkv") || + strings.HasSuffix(lowerName, ".mp3") || strings.HasSuffix(lowerName, ".m4a") || strings.HasSuffix(lowerName, ".wav") { + return "video" + } + if strings.HasPrefix(mimetype, "image/") { + return "image" + } + return "object" +} + +func chooseCommitSpace(uploadType, authSpace string) string { + if uploadType == "video" { + return videoSpaceName + } + return authSpace +} + +func asIDValue(id string) any { + if n, err := strconv.ParseInt(id, 10, 64); err == nil { + return n + } + return id +} + +func parseUnix(ts int64) time.Time { + if ts <= 0 { + return time.Time{} + } + if ts > 1e12 { + return time.UnixMilli(ts) + } + return time.Unix(ts, 0) +} + +func hasMore(v any) bool { + switch val := v.(type) { + case bool: + return val + case float64: + return val != 0 + case int: + return val != 0 + case int64: + return val != 0 + case string: + return val == "1" || strings.EqualFold(val, "true") + default: + return false + } +} + +func extractURL(data map[string]any) string { + priority := []string{ + "download_url", + "main_url", + "MainUrl", + "MainHTTPUrl", + "url", + "source_url", + "play_url", + "backup_url", + "BackupUrl", + "BackupHTTPUrl", + } + for _, key := range priority { + if url := findURLByKey(data, key); url != "" { + return url + } + } + return findAnyHTTPURL(data) +} + +func extractDetailMainURL(data map[string]any) string { + rawList, ok := data["list"] + if !ok { + return "" + } + list, ok := rawList.([]any) + if !ok || len(list) == 0 { + return "" + } + first, ok := list[0].(map[string]any) + if !ok { + return "" + } + generalInfo, ok := first["general_info"].(map[string]any) + if !ok { + return "" + } + mainURL, ok := generalInfo["main_url"].(string) + if !ok || !isHTTPURL(mainURL) { + return "" + } + return mainURL +} + +func findURLByKey(v any, key string) string { + switch cur := v.(type) { + case map[string]any: + if val, ok := cur[key]; ok { + if s, ok := val.(string); ok && isHTTPURL(s) { + return s + } + if decoded := tryDecodeJSONAny(val); decoded != nil { + if s := findURLByKey(decoded, key); s != "" { + return s + } + } + } + for _, child := range cur { + if s := findURLByKey(child, key); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findURLByKey(child, key); s != "" { + return s + } + } + case string: + if decoded := tryDecodeJSONString(cur); decoded != nil { + return findURLByKey(decoded, key) + } + } + return "" +} + +func findAnyHTTPURL(v any) string { + switch cur := v.(type) { + case string: + if isHTTPURL(cur) { + return cur + } + if decoded := tryDecodeJSONString(cur); decoded != nil { + return findAnyHTTPURL(decoded) + } + case map[string]any: + for _, child := range cur { + if s := findAnyHTTPURL(child); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findAnyHTTPURL(child); s != "" { + return s + } + } + } + return "" +} + +func isHTTPURL(v string) bool { + return strings.HasPrefix(v, "http://") || strings.HasPrefix(v, "https://") +} + +func tryDecodeJSONAny(v any) any { + s, ok := v.(string) + if !ok { + return nil + } + return tryDecodeJSONString(s) +} + +func tryDecodeJSONString(s string) any { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + if !(strings.HasPrefix(s, "{") || strings.HasPrefix(s, "[")) { + return nil + } + var out any + if err := json.Unmarshal([]byte(s), &out); err != nil { + return nil + } + return out +} + +var _ driver.Driver = (*Wukong)(nil) diff --git a/drivers/wukong/meta.go b/drivers/wukong/meta.go new file mode 100644 index 00000000000..451fd7942b4 --- /dev/null +++ b/drivers/wukong/meta.go @@ -0,0 +1,34 @@ +package wukong + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Cookie string `json:"cookie" type:"text" required:"true" help:"Cookie from https://pan.wkbrowser.com/"` + Aid string `json:"aid" default:"590353" help:"aid query param used by web requests"` + Language string `json:"language" default:"zh"` + PageSize int `json:"page_size" type:"number" default:"100"` +} + +var config = driver.Config{ + Name: "WuKongNetdisk", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Wukong{} + }) +} diff --git a/drivers/wukong/types.go b/drivers/wukong/types.go new file mode 100644 index 00000000000..86ada25e06d --- /dev/null +++ b/drivers/wukong/types.go @@ -0,0 +1,113 @@ +package wukong + +type filterFileResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + FileList []wukongFile `json:"file_list"` + HasMore any `json:"has_more"` + } `json:"data"` +} + +type wukongFile struct { + FileID int64 `json:"file_id"` + FatherID int64 `json:"father_id"` + IsDirectory int `json:"is_directory"` + FileType int `json:"file_type"` + Size int64 `json:"size"` + FileName string `json:"file_name"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +type rawResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data map[string]any `json:"data"` +} + +type uploadAuthTokenResp struct { + Code int `json:"code"` + Message string `json:"message"` + CurrentTime int64 `json:"current_time"` + ExpireTime int64 `json:"expire_time"` + SpaceName string `json:"space_name"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + SessionToken string `json:"session_token"` +} + +type vodResponseMetadata struct { + RequestID string `json:"RequestId"` + Action string `json:"Action"` + Version string `json:"Version"` + Service string `json:"Service"` + Region string `json:"Region"` + Error struct { + CodeN int `json:"CodeN,omitempty"` + Code string `json:"Code,omitempty"` + Message string `json:"Message,omitempty"` + } `json:"Error,omitempty"` +} + +type vodDomain struct { + Name string `json:"Name"` + Sign string `json:"Sign"` + StoreID string `json:"StoreID"` +} + +type getUploadCandidatesResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + Candidates []struct { + Domains []vodDomain `json:"Domains"` + } `json:"Candidates"` + Domains []vodDomain `json:"Domains"` + } `json:"Result"` +} + +type applyUploadInnerResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + InnerUploadAddress struct { + UploadNodes []struct { + StoreInfos []struct { + StoreURI string `json:"StoreUri"` + Auth string `json:"Auth"` + UploadID string `json:"UploadID"` + StorageHeader map[string]any `json:"StorageHeader"` + } `json:"StoreInfos"` + UploadHost string `json:"UploadHost"` + SessionKey string `json:"SessionKey"` + } `json:"UploadNodes"` + } `json:"InnerUploadAddress"` + } `json:"Result"` +} + +type tosUploadResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Crc32 string `json:"crc32"` + UploadID string `json:"uploadid"` + PartNumber string `json:"part_number"` + Etag string `json:"etag"` + } `json:"data"` +} + +type commitUploadInnerResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + Results []struct { + URI string `json:"Uri"` + URIStatus int `json:"UriStatus"` + Vid string `json:"Vid"` + } `json:"Results"` + PluginResult any `json:"PluginResult"` + } `json:"Result"` +} + +type uploadSubmitResp struct { + Code int `json:"code"` + Message string `json:"message"` +} diff --git a/drivers/yunpan360/driver.go b/drivers/yunpan360/driver.go new file mode 100644 index 00000000000..c92e918a0ad --- /dev/null +++ b/drivers/yunpan360/driver.go @@ -0,0 +1,286 @@ +package yunpan360 + +import ( + "context" + "errors" + stdpath "path" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Yunpan360 struct { + model.Storage + Addition + + authMu sync.Mutex + cachedOpenAuth *OpenAuthInfo + openAuthExpire time.Time + + cachedCookieSession *CookieDownloadSession + cookieSessionExpire time.Time +} + +func (d *Yunpan360) Config() driver.Config { + return config +} + +func (d *Yunpan360) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Yunpan360) Init(ctx context.Context) error { + if d.PageSize <= 0 { + d.PageSize = 100 + } + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + if d.RootFolderPath == "" { + d.RootFolderPath = "/" + } + d.OrderDirection = strings.ToLower(strings.TrimSpace(d.OrderDirection)) + if d.OrderDirection != "desc" { + d.OrderDirection = "asc" + } + d.AuthType = strings.ToLower(strings.TrimSpace(d.AuthType)) + if d.AuthType == "" { + d.AuthType = authTypeCookie + } + d.SubChannel = strings.TrimSpace(d.SubChannel) + if d.SubChannel == "" { + d.SubChannel = defaultSubChannel + } + d.EcsEnv = strings.ToLower(strings.TrimSpace(d.EcsEnv)) + if d.EcsEnv == "" { + d.EcsEnv = openEnvProd + } + d.Cookie = strings.TrimSpace(d.Cookie) + d.APIKey = strings.TrimSpace(d.APIKey) + d.OwnerQID = strings.TrimSpace(d.OwnerQID) + d.DownloadToken = strings.TrimSpace(d.DownloadToken) + d.cachedOpenAuth = nil + d.openAuthExpire = time.Time{} + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} + + switch d.authMode() { + case authTypeAPIKey: + if d.APIKey == "" { + return errors.New("api_key is empty") + } + _, err := d.openUserInfo(ctx) + return err + case authTypeCookie: + if d.Cookie == "" { + return errors.New("cookie is empty") + } + // Web download URLs require browser-session headers; force local proxying + // so AList can forward Referer/Origin instead of exposing a bare 302 URL. + d.WebProxy = true + _, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + return err + default: + return errors.New("invalid auth_type") + } +} + +func (d *Yunpan360) Drop(ctx context.Context) error { + d.cachedOpenAuth = nil + d.openAuthExpire = time.Time{} + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} + return nil +} + +func (d *Yunpan360) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + dirPath := dir.GetPath() + if dirPath == "" { + dirPath = d.RootFolderPath + } + + objs := make([]model.Obj, 0, d.PageSize) + for page := 0; ; page++ { + resp, err := d.listPage(ctx, dirPath, page, d.PageSize) + if err != nil { + return nil, err + } + pageObjs := resp.Objects(dirPath) + for _, item := range pageObjs { + objs = append(objs, item) + } + if len(pageObjs) == 0 { + break + } + if d.authMode() == authTypeAPIKey { + if len(pageObjs) < d.PageSize { + break + } + continue + } + if !resp.GetHasNextPage() { + break + } + } + return objs, nil +} + +func (d *Yunpan360) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if d.authMode() == authTypeCookie { + resp, err := d.cookieDownloadURL(ctx, file) + if err != nil { + return nil, err + } + downloadURL := strings.TrimSpace(resp.GetURL()) + if downloadURL == "" { + return nil, errors.New("download url is empty") + } + return &model.Link{ + URL: downloadURL, + Header: map[string][]string{ + "Accept": {"text/javascript, text/html, application/xml, text/xml, */*"}, + "Origin": {baseURL}, + "Referer": {baseURL + indexPath}, + }, + }, nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + resp, err := d.openDownloadURL(ctx, file) + if err != nil { + return nil, err + } + downloadURL := strings.TrimSpace(resp.GetURL()) + if downloadURL == "" { + return nil, errors.New("download url is empty") + } + return &model.Link{URL: downloadURL}, nil +} + +func (d *Yunpan360) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if d.authMode() == authTypeCookie { + fullPath := ensureDirAPIPath(stdpath.Join(parentDir.GetPath(), dirName)) + resp, err := d.cookieMakeDir(ctx, fullPath) + if err != nil { + return nil, err + } + return &YunpanObject{ + Object: model.Object{ + ID: resp.Data.NID, + Path: normalizeRemotePath(fullPath), + Name: dirName, + Size: 0, + Modified: time.Now(), + IsFolder: true, + }, + }, nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + fullPath := ensureDirAPIPath(stdpath.Join(parentDir.GetPath(), dirName)) + resp, err := d.openMakeDir(ctx, fullPath) + if err != nil { + return nil, err + } + obj := &model.Object{ + ID: resp.Data.NID, + Path: normalizeRemotePath(fullPath), + Name: dirName, + Size: 0, + Modified: time.Now(), + IsFolder: true, + } + return obj, nil +} + +func (d *Yunpan360) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.authMode() == authTypeCookie { + srcPath := apiPathForObj(srcObj) + dstPath := ensureDirAPIPath(dstDir.GetPath()) + if err := d.cookieMove(ctx, srcPath, dstPath); err != nil { + return nil, err + } + return cloneObj(srcObj, stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.GetName()), nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + srcPath := apiPathForObj(srcObj) + dstPath := ensureDirAPIPath(dstDir.GetPath()) + if err := d.openMove(ctx, srcPath, dstPath); err != nil { + return nil, err + } + return cloneObj(srcObj, stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.GetName()), nil +} + +func (d *Yunpan360) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.authMode() == authTypeCookie { + targetName := strings.TrimSuffix(strings.TrimSpace(newName), "/") + if targetName == "" { + return nil, errors.New("new name is empty") + } + if err := d.cookieRename(ctx, srcObj, targetName); err != nil { + return nil, err + } + parentPath := stdpath.Dir(srcObj.GetPath()) + if parentPath == "." { + parentPath = "/" + } + return cloneObj(srcObj, stdpath.Join(parentPath, targetName), targetName), nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + srcPath := apiPathForObj(srcObj) + targetName := newName + if srcObj.IsDir() { + targetName = ensureDirSuffix(newName) + } + if err := d.openRename(ctx, srcPath, targetName); err != nil { + return nil, err + } + + parentPath := stdpath.Dir(srcObj.GetPath()) + if parentPath == "." { + parentPath = "/" + } + return cloneObj(srcObj, stdpath.Join(parentPath, strings.TrimSuffix(newName, "/")), strings.TrimSuffix(newName, "/")), nil +} + +func (d *Yunpan360) Remove(ctx context.Context, obj model.Obj) error { + if d.authMode() == authTypeCookie { + return d.cookieRecycle(ctx, obj) + } + if d.authMode() != authTypeAPIKey { + return errs.NotImplement + } + return d.openDelete(ctx, apiPathForObj(obj)) +} + +func (d *Yunpan360) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if d.authMode() == authTypeCookie { + return nil, errs.NotImplement + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + return d.putOpenFile(ctx, dstDir, file, up) +} + +func (d *Yunpan360) authMode() string { + if d.AuthType == authTypeAPIKey { + return authTypeAPIKey + } + return authTypeCookie +} + +var _ driver.Driver = (*Yunpan360)(nil) diff --git a/drivers/yunpan360/meta.go b/drivers/yunpan360/meta.go new file mode 100644 index 00000000000..5c4e1c8d245 --- /dev/null +++ b/drivers/yunpan360/meta.go @@ -0,0 +1,34 @@ +package yunpan360 + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + AuthType string `json:"auth_type" type:"select" options:"cookie,api_key" default:"cookie"` + Cookie string `json:"cookie" type:"text" help:"Cookie copied from a logged-in yunpan.com session; used when auth_type=cookie"` + OwnerQID string `json:"owner_qid" type:"text" help:"Optional owner_qid for cookie-mode download; leave empty to auto-detect"` + DownloadToken string `json:"download_token" type:"text" help:"Optional web token for cookie-mode download; leave empty to auto-detect"` + APIKey string `json:"api_key" type:"text" help:"360 AI YunPan API key; used when auth_type=api_key"` + EcsEnv string `json:"ecs_env" type:"select" options:"prod,test,hgtest" default:"prod"` + SubChannel string `json:"sub_channel" default:"open"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + PageSize int `json:"page_size" type:"number" default:"100" help:"List page size"` +} + +var config = driver.Config{ + Name: "360AIYunPan", + LocalSort: false, + CheckStatus: true, + NoUpload: false, + DefaultRoot: "/", + Alert: "info|api_key mode supports list/link/upload/mkdir/rename/move/delete; cookie mode supports list/link/mkdir/rename/move/delete only, and forces web proxy because direct download URLs require web headers.", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Yunpan360{} + }) +} diff --git a/drivers/yunpan360/types.go b/drivers/yunpan360/types.go new file mode 100644 index 00000000000..f5c1b7a2135 --- /dev/null +++ b/drivers/yunpan360/types.go @@ -0,0 +1,462 @@ +package yunpan360 + +import ( + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + authTypeCookie = "cookie" + authTypeAPIKey = "api_key" + openEnvProd = "prod" + defaultSubChannel = "open" + openSignSecret = "e7b24b112a44fdd9ee93bdf998c6ca0e" + openClientID = "e4757e933b6486c08ed206ecb6d5d9e684fcb4e2" + openClientSecret = "885fd3231f1c1e37c9f462261a09b8c38cde0c2b" + openClientSecretQA = "b11b8fff1c75a5d227c8cc93aaeb0bb70c8eee47" +) + +type BaseResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` +} + +type CookieDownloadSession struct { + OwnerQID string + Token string +} + +type ListResp interface { + Objects(parentPath string) []model.Obj + GetHasNextPage() bool +} + +type CookieListResp struct { + BaseResp + Token string `json:"token"` + OwnerQid string `json:"owner_qid"` + Qid string `json:"qid"` + Data []ListItem `json:"data"` + HasNextPage bool `json:"has_next_page"` +} + +func (r *CookieListResp) Objects(parentPath string) []model.Obj { + ownerQID := r.GetOwnerQID() + return utils.MustSliceConvert(r.Data, func(src ListItem) model.Obj { + return src.toObj(parentPath, ownerQID, r.Token) + }) +} + +func (r *CookieListResp) GetHasNextPage() bool { + return r.HasNextPage +} + +func (r *CookieListResp) GetOwnerQID() string { + return firstNonEmpty(r.OwnerQid, r.Qid) +} + +type ListItem struct { + NID string `json:"nid"` + FileName string `json:"file_name"` + FilePath string `json:"file_path"` + FileSize string `json:"file_size"` + IsDir bool `json:"is_dir"` + Fhash string `json:"fhash"` + CreateTime string `json:"create_time"` + ModifyTime string `json:"modify_time"` + Mtime string `json:"mtime"` + ServerTime string `json:"server_time"` + Preview string `json:"preview"` + Thumb string `json:"thumb"` + SrcPic string `json:"srcpic"` + OwnerQid string `json:"owner_qid"` + Qid string `json:"qid"` + Token string `json:"token"` +} + +func (i ListItem) toObj(parentPath, ownerQID, token string) model.Obj { + objPath := normalizeRemotePath(i.FilePath) + if objPath == "" || !pathLooksLikeObject(objPath, i.FileName) { + objPath = joinRemotePath(parentPath, i.FileName) + } + thumb := "" + if !i.IsDir { + thumb = absoluteURL(firstNonEmpty(i.Thumb, i.SrcPic, i.Preview)) + } + + return &YunpanObject{ + Object: model.Object{ + ID: i.NID, + Path: objPath, + Name: i.FileName, + Size: parseSize(i.FileSize), + Modified: parseYunpanTime(i.ModifyTime, i.Mtime), + Ctime: parseYunpanTime(i.CreateTime, i.ServerTime), + IsFolder: i.IsDir, + HashInfo: parseHash(i.Fhash), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + OwnerQID: firstNonEmpty(i.OwnerQid, i.Qid, ownerQID), + DownloadToken: firstNonEmpty(i.Token, token), + } +} + +func parseSize(raw string) int64 { + size, _ := strconv.ParseInt(raw, 10, 64) + return size +} + +func parseHash(raw string) utils.HashInfo { + if len(raw) == 40 { + return utils.NewHashInfo(utils.SHA1, raw) + } + return utils.HashInfo{} +} + +func parseYunpanTime(unixStr, text string) time.Time { + if t := parseUnixTime(unixStr); !t.IsZero() { + return t + } + return parseTextTime(text) +} + +func parseUnixTime(raw string) time.Time { + if raw != "" { + sec, err := strconv.ParseInt(raw, 10, 64) + if err == nil && sec > 0 { + return time.Unix(sec, 0) + } + } + return time.Time{} +} + +func parseTextTime(text string) time.Time { + if text == "" { + return time.Time{} + } + t, err := time.ParseInLocation("2006-01-02 15:04:05", text, utils.CNLoc) + if err == nil { + return t + } + return time.Time{} +} + +func normalizeRemotePath(p string) string { + if p == "" { + return "" + } + if p != "/" { + p = strings.TrimSuffix(p, "/") + } + return utils.FixAndCleanPath(p) +} + +type OpenAuthResp struct { + BaseResp + Data struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + AccessTokenExpire int64 `json:"access_token_expire"` + Qid string `json:"qid"` + } `json:"data"` +} + +type OpenAuthInfo struct { + AccessToken string + Qid string + Token string + SubChannel string +} + +type OpenListResp struct { + BaseResp + Data struct { + NodeList []OpenNode `json:"node_list"` + List []OpenNode `json:"list"` + Data []OpenNode `json:"data"` + TotalCount int `json:"total_count"` + Total int `json:"total"` + PageNum int `json:"page_num"` + } `json:"data"` +} + +func (r *OpenListResp) Objects(parentPath string) []model.Obj { + nodes := r.Data.NodeList + if len(nodes) == 0 { + nodes = r.Data.List + } + if len(nodes) == 0 { + nodes = r.Data.Data + } + return utils.MustSliceConvert(nodes, func(src OpenNode) model.Obj { + return src.toObj(parentPath) + }) +} + +func (r *OpenListResp) GetHasNextPage() bool { + total := r.Data.TotalCount + if total <= 0 { + total = r.Data.Total + } + if total <= 0 { + return false + } + loaded := len(r.Data.NodeList) + if loaded == 0 { + loaded = len(r.Data.List) + } + if loaded == 0 { + loaded = len(r.Data.Data) + } + return loaded > 0 && loaded < total +} + +type OpenNode struct { + NID string `json:"nid"` + Name string `json:"name"` + FName string `json:"fname"` + Path string `json:"path"` + FPath string `json:"fpath"` + Type interface{} `json:"type"` + IsDir interface{} `json:"is_dir"` + CountSize interface{} `json:"count_size"` + Size interface{} `json:"size"` + CreateTime interface{} `json:"create_time"` + ModifyTime interface{} `json:"modify_time"` + MTime interface{} `json:"mtime"` + FileHash string `json:"file_hash"` + Fhash string `json:"fhash"` + Thumb string `json:"thumb"` + Preview string `json:"preview"` + SrcPic string `json:"srcpic"` +} + +func (n OpenNode) toObj(parentPath string) model.Obj { + name := firstNonEmpty(strings.TrimSpace(n.Name), strings.TrimSpace(n.FName)) + objPath := normalizeRemotePath(firstNonEmpty(n.FPath, n.Path)) + if objPath == "" || !pathLooksLikeObject(objPath, name) { + objPath = joinRemotePath(parentPath, name) + } + isDir := parseOpenDir(n.IsDir, n.Type) + thumb := "" + if !isDir { + thumb = absoluteURL(firstNonEmpty(n.Thumb, n.SrcPic, n.Preview)) + } + + return &YunpanObject{ + Object: model.Object{ + ID: n.NID, + Path: objPath, + Name: name, + Size: parseAnySize(n.CountSize, n.Size), + Modified: parseAnyTime(n.ModifyTime, n.MTime), + Ctime: parseAnyTime(n.CreateTime), + IsFolder: isDir, + HashInfo: parseHash(firstNonEmpty(n.FileHash, n.Fhash)), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + } +} + +type OpenUserInfoResp struct { + BaseResp + Data map[string]interface{} `json:"data"` +} + +type OpenMkdirResp struct { + BaseResp + Data struct { + NID string `json:"nid"` + } `json:"data"` +} + +type CookieMkdirResp struct { + BaseResp + Data struct { + NID string `json:"nid"` + } `json:"data"` +} + +type CookieMoveResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + IsAsync bool `json:"is_async"` + } `json:"data"` +} + +type CookieRecycleResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + IsAsync bool `json:"is_async"` + } `json:"data"` +} + +type CookieAsyncQueryResp struct { + BaseResp + Data map[string]CookieAsyncTask `json:"data"` +} + +type CookieAsyncTask struct { + MessageID string `json:"message_id"` + SendTime string `json:"send_time"` + Status int `json:"status"` + Action string `json:"action"` + Errno int `json:"errno"` + Errstr string `json:"errstr"` + Result string `json:"result"` +} + +type CookieDownloadResp struct { + BaseResp + Data struct { + DownloadURL string `json:"download_url"` + Store string `json:"store"` + Host string `json:"host"` + } `json:"data"` +} + +func (r *CookieDownloadResp) GetURL() string { + return r.Data.DownloadURL +} + +type OpenDownloadResp struct { + BaseResp + Data struct { + DownloadURL string `json:"downloadUrl"` + } `json:"data"` + DownloadURL string `json:"downloadUrl"` +} + +func (r *OpenDownloadResp) GetURL() string { + return firstNonEmpty(r.Data.DownloadURL, r.DownloadURL) +} + +type YunpanObject struct { + model.Object + model.Thumbnail + OwnerQID string + DownloadToken string +} + +func parseAnySize(values ...interface{}) int64 { + for _, value := range values { + switch v := value.(type) { + case string: + if v == "" { + continue + } + size, err := strconv.ParseInt(v, 10, 64) + if err == nil { + return size + } + case float64: + return int64(v) + case int64: + return v + case int: + return int64(v) + } + } + return 0 +} + +func parseAnyTime(values ...interface{}) time.Time { + for _, value := range values { + switch v := value.(type) { + case string: + if t := parseUnixTime(v); !t.IsZero() { + return t + } + if t := parseTextTime(v); !t.IsZero() { + return t + } + case float64: + if v > 0 { + return time.Unix(int64(v), 0) + } + case int64: + if v > 0 { + return time.Unix(v, 0) + } + case int: + if v > 0 { + return time.Unix(int64(v), 0) + } + } + } + return time.Time{} +} + +func parseOpenDir(values ...interface{}) bool { + for _, value := range values { + switch v := value.(type) { + case bool: + if v { + return true + } + case string: + switch strings.ToLower(strings.TrimSpace(v)) { + case "1", "true", "dir", "folder": + return true + } + case float64: + if int64(v) == 1 { + return true + } + case int64: + if v == 1 { + return true + } + case int: + if v == 1 { + return true + } + } + } + return false +} + +func pathLooksLikeObject(objPath, name string) bool { + if objPath == "" || name == "" { + return false + } + return strings.TrimSuffix(stdPathBase(objPath), "/") == strings.TrimSuffix(name, "/") +} + +func stdPathBase(p string) string { + if p == "/" { + return "/" + } + idx := strings.LastIndex(strings.TrimSuffix(p, "/"), "/") + if idx < 0 { + return p + } + return p[idx+1:] +} + +func joinRemotePath(parentPath, name string) string { + parentPath = normalizeRemotePath(parentPath) + if parentPath == "" { + parentPath = "/" + } + return normalizeRemotePath(strings.TrimSuffix(parentPath, "/") + "/" + strings.TrimPrefix(name, "/")) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} diff --git a/drivers/yunpan360/upload.go b/drivers/yunpan360/upload.go new file mode 100644 index 00000000000..38c3fded57b --- /dev/null +++ b/drivers/yunpan360/upload.go @@ -0,0 +1,926 @@ +package yunpan360 + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/textproto" + "net/url" + stdpath "path" + "runtime" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + aliststream "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + yunpanUploadChunkSize = int64(512 * 1024) + yunpanUploadBoundary = "WebKitFormBoundaryQ5OJVvzZwEkg4ttY" + yunpanUploadVersion = "1.0.1" + yunpanUploadDevType = "ecs_openapi" + yunpanUploadDevName = "EYUN_WEB_UPLOAD" +) + +type openUploadPlan struct { + DirPath string + TargetPath string + FileName string + Size int64 + FileHash string + FileSHA1 string + FileSum string + CreatedAt int64 + DeviceID string + Chunks []openUploadChunk +} + +type openUploadChunk struct { + Index int + Offset int64 + Size int64 + Hash string +} + +type openUploadDetectResp struct { + BaseResp + Data struct { + Exists []openUploadDuplicate `json:"exists"` + IsSlice int `json:"is_slice"` + } `json:"data"` +} + +type openUploadDuplicate struct { + FullName string `json:"fullName"` +} + +type openUploadAddressResp struct { + BaseResp + Data struct { + HTTP string `json:"http"` + Addr1 string `json:"addr_1"` + Addr2 string `json:"addr_2"` + Backup string `json:"backup"` + TK string `json:"tk"` + GroupSize string `json:"group_size"` + AutoCommit interface{} `json:"autoCommit"` + IsHTTPS interface{} `json:"is_https"` + } `json:"data"` +} + +type openUploadRequestResp struct { + BaseResp + Data struct { + Tid string `json:"tid"` + BlockInfo []map[string]interface{} `json:"block_info"` + } `json:"data"` +} + +type openUploadFinalizeResp struct { + BaseResp + Data map[string]interface{} `json:"data"` +} + +type uploadEnvelope struct { + Errno *int `json:"errno"` + Errmsg string `json:"errmsg"` + Data json.RawMessage `json:"data"` +} + +func (d *Yunpan360) putOpenFile(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return nil, err + } + + dirPath := dstDir.GetPath() + if dirPath == "" { + dirPath = d.RootFolderPath + } + dirPath = ensureDirAPIPath(dirPath) + targetPath := joinRemotePath(dirPath, file.GetName()) + + cached, err := d.cacheUploadSource(ctx, file, progressRange(up, 0, 5)) + if err != nil { + return nil, err + } + defer func() { + _, _ = cached.Seek(0, io.SeekStart) + }() + + plan, err := d.buildUploadPlan(ctx, cached, targetPath, file.GetSize(), file.ModTime(), progressRange(up, 5, 10)) + if err != nil { + return nil, err + } + + detectResp, err := d.openDetectUpload(ctx, auth, plan) + if err != nil { + return nil, err + } + if detectResp.Data.IsSlice == 0 { + detectResp.Data.IsSlice = 1 + } + + addrResp, err := d.openGetUploadAddress(ctx, auth, plan) + if err != nil { + return nil, err + } + + finalResp := &openUploadFinalizeResp{BaseResp: BaseResp{Errno: 0}, Data: map[string]interface{}{}} + if strings.TrimSpace(addrResp.Data.HTTP) == "" { + finalResp.Data = map[string]interface{}{"autoCommit": true} + if tk := strings.TrimSpace(addrResp.Data.TK); tk != "" { + finalResp.Data["tk"] = tk + finalResp.Data["autoCommit"] = false + } + } else { + reqResp, err := d.openRequestUpload(ctx, auth, plan, addrResp) + if err != nil { + return nil, err + } + if err := d.openUploadBlocks(ctx, auth, cached, plan, addrResp, reqResp, up); err != nil { + return nil, err + } + finalResp, err = d.openCommitUpload(ctx, auth, plan, addrResp, reqResp) + if err != nil { + return nil, err + } + } + + if err := d.openFinalizeUpload(ctx, auth, finalResp); err != nil { + return nil, err + } + if up != nil { + up(100) + } + + obj, err := d.findUploadedObject(ctx, targetPath) + if err == nil { + return obj, nil + } + if !errors.Is(err, errs.ObjectNotFound) { + return nil, err + } + + return &model.Object{ + Path: normalizeRemotePath(targetPath), + Name: file.GetName(), + Size: file.GetSize(), + Modified: time.Now(), + Ctime: time.Now(), + HashInfo: utils.NewHashInfo(utils.SHA1, firstNonEmpty(plan.FileSHA1, plan.FileHash)), + }, nil +} + +func (d *Yunpan360) cacheUploadSource(ctx context.Context, file model.FileStreamer, up driver.UpdateProgress) (model.File, error) { + if cached := file.GetFile(); cached != nil { + _, _ = cached.Seek(0, io.SeekStart) + if up != nil { + up(100) + } + return cached, nil + } + if up == nil { + return file.CacheFullInTempFile() + } + return aliststream.CacheFullInTempFileAndUpdateProgress(file, up) +} + +func (d *Yunpan360) buildUploadPlan(ctx context.Context, cached model.File, targetPath string, size int64, modTime time.Time, up driver.UpdateProgress) (*openUploadPlan, error) { + if _, err := cached.Seek(0, io.SeekStart); err != nil { + return nil, err + } + + createdAt := time.Now().Unix() + if !modTime.IsZero() { + createdAt = modTime.Unix() + } + plan := &openUploadPlan{ + DirPath: ensureDirAPIPath(stdpath.Dir(targetPath)), + TargetPath: normalizeRemotePath(targetPath), + FileName: stdpath.Base(targetPath), + Size: size, + CreatedAt: createdAt, + DeviceID: sha1HexString("node-sdk-" + runtime.Version()), + } + + if plan.DirPath == "./" || plan.DirPath == "." { + plan.DirPath = "/" + } + + totalChunks := 0 + if size > 0 { + totalChunks = int((size + yunpanUploadChunkSize - 1) / yunpanUploadChunkSize) + } + chunks := make([]openUploadChunk, 0, totalChunks) + var hashConcat strings.Builder + var hashed int64 + + for idx := 0; idx < totalChunks; idx++ { + if err := ctx.Err(); err != nil { + return nil, err + } + offset := int64(idx) * yunpanUploadChunkSize + chunkSize := yunpanUploadChunkSize + if remain := size - offset; remain < chunkSize { + chunkSize = remain + } + + chunkHash, err := sha1HexReader(io.NewSectionReader(cached, offset, chunkSize)) + if err != nil { + return nil, err + } + + chunks = append(chunks, openUploadChunk{ + Index: idx + 1, + Offset: offset, + Size: chunkSize, + Hash: chunkHash, + }) + hashConcat.WriteString(chunkHash) + hashed += chunkSize + reportByteProgress(up, hashed, size) + } + + plan.Chunks = chunks + plan.FileHash = sha1HexString(hashConcat.String()) + if _, err := cached.Seek(0, io.SeekStart); err != nil { + return nil, err + } + sha1Hasher := sha1.New() + md5Hasher := md5.New() + if _, err := io.Copy(io.MultiWriter(sha1Hasher, md5Hasher), cached); err != nil { + return nil, err + } + plan.FileSHA1 = hex.EncodeToString(sha1Hasher.Sum(nil)) + plan.FileSum = hex.EncodeToString(md5Hasher.Sum(nil)) + if size == 0 && up != nil { + up(100) + } + return plan, nil +} + +func (d *Yunpan360) openDetectUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan) (*openUploadDetectResp, error) { + payload, err := json.Marshal([]map[string]interface{}{ + {"fname": plan.FileName, "fsize": plan.Size}, + }) + if err != nil { + return nil, err + } + + signParams := map[string]string{ + "data": string(payload), + "path": plan.DirPath, + } + body, contentType, err := createMultipartForm("", map[string]string{ + "qid": auth.Qid, + "data": string(payload), + "path": plan.DirPath, + "sign": openSign(auth.AccessToken, auth.Qid, "Sync.detectFileExists", signParams), + }, nil) + if err != nil { + return nil, err + } + + var resp openUploadDetectResp + err = d.uploadRequest(ctx, http.MethodPost, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.detectFileExists", nil), auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openGetUploadAddress(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan) (*openUploadAddressResp, error) { + query := d.uploadCookieParams(auth, plan, "") + signParams := map[string]string{ + "access_token": auth.AccessToken, + "fhash": plan.FileHash, + "fname": plan.TargetPath, + "fsize": strconv.FormatInt(plan.Size, 10), + } + query["sign"] = openSign(auth.AccessToken, auth.Qid, "Sync.getUploadFileAddr", signParams) + + var resp openUploadAddressResp + err := d.uploadRequest(ctx, http.MethodGet, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.getUploadFileAddr", query), auth.AccessToken, "", nil, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openRequestUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan, addrResp *openUploadAddressResp) (*openUploadRequestResp, error) { + chunkInfos := make([]map[string]interface{}, 0, len(plan.Chunks)) + for _, chunk := range plan.Chunks { + chunkInfos = append(chunkInfos, map[string]interface{}{ + "bhash": chunk.Hash, + "bidx": chunk.Index, + "boffset": chunk.Offset, + "bsize": chunk.Size, + }) + } + payload, err := json.Marshal(map[string]interface{}{ + "request": map[string]interface{}{ + "block_info": chunkInfos, + }, + }) + if err != nil { + return nil, err + } + + body, contentType, err := createMultipartForm( + yunpanUploadBoundary, + d.uploadCookieParams(auth, plan, strings.TrimSpace(addrResp.Data.TK)), + &multipartFile{ + FieldName: "file", + FileName: "file.dat", + ContentType: "application/octet-stream", + Content: payload, + }, + ) + if err != nil { + return nil, err + } + + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.request4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var resp openUploadRequestResp + err = d.uploadRequest(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openUploadBlocks(ctx context.Context, auth *OpenAuthInfo, cached model.File, plan *openUploadPlan, addrResp *openUploadAddressResp, reqResp *openUploadRequestResp, up driver.UpdateProgress) error { + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.block4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var uploaded int64 + for _, chunk := range plan.Chunks { + info := reqResp.blockInfoForChunk(chunk.Index) + if info.found() > 0 { + uploaded += chunk.Size + reportUploadProgress(up, uploaded, plan.Size) + continue + } + + chunkBytes := make([]byte, chunk.Size) + if _, err := cached.ReadAt(chunkBytes, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { + return err + } + + fields := map[string]string{ + "bhash": chunk.Hash, + "bidx": strconv.Itoa(chunk.Index), + "boffset": strconv.FormatInt(chunk.Offset, 10), + "bsize": strconv.FormatInt(chunk.Size, 10), + "filename": plan.TargetPath, + "filesize": strconv.FormatInt(plan.Size, 10), + "q": info.stringValue("q"), + "t": info.stringValue("t"), + "token": auth.Token, + "tid": info.stringValue("tid"), + } + for key, value := range info.extraFields() { + fields[key] = value + } + + body, contentType, err := createMultipartForm( + yunpanUploadBoundary, + fields, + &multipartFile{ + FieldName: "file", + FileName: "file.dat", + ContentType: "application/octet-stream", + Content: chunkBytes, + }, + ) + if err != nil { + return err + } + + chunkStart := uploaded + chunkSize := chunk.Size + err = d.uploadRequestWithProgress(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, func(p float64) { + done := chunkStart + int64(float64(chunkSize)*(p/100.0)) + reportUploadProgress(up, done, plan.Size) + }, nil) + if err != nil { + return err + } + + uploaded += chunk.Size + reportUploadProgress(up, uploaded, plan.Size) + } + return nil +} + +func (d *Yunpan360) openCommitUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan, addrResp *openUploadAddressResp, reqResp *openUploadRequestResp) (*openUploadFinalizeResp, error) { + body, contentType, err := createMultipartForm("", map[string]string{ + "q": "", + "t": "", + "token": auth.Token, + "tid": strings.TrimSpace(reqResp.Data.Tid), + }, nil) + if err != nil { + return nil, err + } + + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.commit4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var resp openUploadFinalizeResp + err = d.uploadRequest(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openFinalizeUpload(ctx context.Context, auth *OpenAuthInfo, resp *openUploadFinalizeResp) error { + if resp == nil || resp.autoCommit() { + return nil + } + tk := strings.TrimSpace(resp.stringValue("tk")) + if tk == "" { + return errors.New("upload tk is empty") + } + + signParams := map[string]string{ + "tk": tk, + } + body, contentType, err := createMultipartForm("", map[string]string{ + "qid": auth.Qid, + "tk": tk, + "sign": openSign(auth.AccessToken, auth.Qid, "Sync.addFileToApi", signParams), + }, nil) + if err != nil { + return err + } + + return d.uploadRequest(ctx, http.MethodPost, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.addFileToApi", nil), auth.AccessToken, contentType, body, nil) +} + +func (d *Yunpan360) findUploadedObject(ctx context.Context, targetPath string) (model.Obj, error) { + targetPath = normalizeRemotePath(targetPath) + parentPath := normalizeRemotePath(stdpath.Dir(targetPath)) + if parentPath == "." || parentPath == "" { + parentPath = "/" + } + targetName := stdpath.Base(targetPath) + + for page := 0; ; page++ { + resp, err := d.listPage(ctx, parentPath, page, d.PageSize) + if err != nil { + return nil, err + } + pageObjs := resp.Objects(parentPath) + for _, obj := range pageObjs { + if normalizeRemotePath(obj.GetPath()) == targetPath || obj.GetName() == targetName { + return obj, nil + } + } + if len(pageObjs) == 0 || len(pageObjs) < d.PageSize { + break + } + } + return nil, errs.ObjectNotFound +} + +func (d *Yunpan360) uploadCookieParams(auth *OpenAuthInfo, plan *openUploadPlan, uploadTK string) map[string]string { + params := map[string]string{ + "owner_qid": auth.Qid, + "fname": plan.TargetPath, + "fsize": strconv.FormatInt(plan.Size, 10), + "fctime": strconv.FormatInt(plan.CreatedAt, 10), + "fmtime": strconv.FormatInt(plan.CreatedAt, 10), + "fhash": plan.FileHash, + "qid": auth.Qid, + "fattr": "0", + "token": auth.Token, + "devtype": yunpanUploadDevType, + } + if uploadTK != "" { + params["tk"] = uploadTK + } + return params +} + +func (d *Yunpan360) uploadDataParams(auth *OpenAuthInfo, plan *openUploadPlan) map[string]string { + return map[string]string{ + "owner_qid": auth.Qid, + "qid": auth.Qid, + "devtype": yunpanUploadDevType, + "devid": plan.DeviceID, + "v": yunpanUploadVersion, + "ofmt": "json", + "devname": yunpanUploadDevName, + "rtick": strconv.FormatInt(time.Now().UnixMilli(), 10), + } +} + +func (d *Yunpan360) uploadBaseURL(addrResp *openUploadAddressResp) string { + host := "" + isHTTPS := false + if addrResp != nil { + host = strings.TrimSpace(addrResp.Data.HTTP) + isHTTPS = parseOpenDir(addrResp.Data.IsHTTPS) + } + scheme := "http" + if isHTTPS { + scheme = "https" + } + if host == "" { + return openAPIURL(d.EcsEnv) + } + return fmt.Sprintf("%s://%s/intf.php", scheme, host) +} + +func (d *Yunpan360) uploadRequest(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, out interface{}) error { + return d.uploadRequestWithProgress(ctx, method, reqURL, accessToken, contentType, body, nil, out) +} + +func (d *Yunpan360) uploadRequestWithProgress(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, progress driver.UpdateProgress, out interface{}) error { + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + if err := sleepWithContext(ctx, time.Duration(attempt)*500*time.Millisecond); err != nil { + return err + } + } + err := d.doUploadRequest(ctx, method, reqURL, accessToken, contentType, body, progress, out) + if err == nil { + return nil + } + lastErr = err + if ctx.Err() != nil { + return ctx.Err() + } + } + return lastErr +} + +func (d *Yunpan360) doUploadRequest(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, progress driver.UpdateProgress, out interface{}) error { + var bodyReader io.ReadCloser + if body != nil { + reader := &driver.SimpleReaderWithSize{ + Reader: bytes.NewReader(body), + Size: int64(len(body)), + } + if progress != nil { + bodyReader = driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: reader, + UpdateProgress: progress, + }) + } else { + bodyReader = driver.NewLimitedUploadStream(ctx, reader) + } + } + + req, err := http.NewRequestWithContext(ctx, method, reqURL, bodyReader) + if err != nil { + if bodyReader != nil { + _ = bodyReader.Close() + } + return err + } + if body != nil { + req.ContentLength = int64(len(body)) + } + req.Header.Set("Accept", "application/json") + if accessToken != "" { + req.Header.Set("Access-Token", accessToken) + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode >= http.StatusBadRequest { + return fmt.Errorf("yunpan upload request failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(respBody))) + } + return decodeUploadResp(respBody, out) +} + +func decodeUploadResp(body []byte, out interface{}) error { + var env uploadEnvelope + if err := utils.Json.Unmarshal(body, &env); err != nil { + return err + } + if env.Errno != nil && *env.Errno != 0 { + if env.Errmsg == "" { + return fmt.Errorf("yunpan upload request failed: errno=%d", *env.Errno) + } + return errors.New(env.Errmsg) + } + if env.Errno == nil && strings.TrimSpace(env.Errmsg) != "" && len(env.Data) > 0 && string(env.Data) == "[]" { + return errors.New(env.Errmsg) + } + if out == nil { + return nil + } + if err := utils.Json.Unmarshal(body, out); err != nil { + if strings.TrimSpace(env.Errmsg) != "" { + return errors.New(env.Errmsg) + } + return err + } + return nil +} + +type multipartFile struct { + FieldName string + FileName string + ContentType string + Content []byte +} + +func createMultipartForm(boundary string, fields map[string]string, file *multipartFile) ([]byte, string, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + if boundary != "" { + if err := writer.SetBoundary(boundary); err != nil { + return nil, "", err + } + } + + for key, value := range fields { + if err := writer.WriteField(key, value); err != nil { + return nil, "", err + } + } + + if file != nil { + partHeader := make(textproto.MIMEHeader) + partHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, file.FieldName, file.FileName)) + partHeader.Set("Content-Type", firstNonEmpty(file.ContentType, "application/octet-stream")) + part, err := writer.CreatePart(partHeader) + if err != nil { + return nil, "", err + } + if _, err := part.Write(file.Content); err != nil { + return nil, "", err + } + } + + if err := writer.Close(); err != nil { + return nil, "", err + } + return body.Bytes(), writer.FormDataContentType(), nil +} + +func buildJSQueryURL(baseURL, method string, params map[string]string) string { + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sort.Strings(keys) + + var builder strings.Builder + builder.WriteString(baseURL) + if strings.Contains(baseURL, "?") { + builder.WriteByte('&') + } else { + builder.WriteByte('?') + } + builder.WriteString("method=") + builder.WriteString(jsQueryEscape(method)) + for _, key := range keys { + value := params[key] + if value == "" { + continue + } + builder.WriteByte('&') + builder.WriteString(key) + builder.WriteByte('=') + builder.WriteString(jsQueryEscape(value)) + } + return builder.String() +} + +func buildQueryURL(baseURL string, params map[string]string) string { + u, err := url.Parse(baseURL) + if err != nil { + return baseURL + } + u.RawQuery = encodeSortedQuery(params) + return u.String() +} + +func encodeSortedQuery(params map[string]string) string { + if len(params) == 0 { + return "" + } + q := make(url.Values, len(params)) + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + q.Set(key, params[key]) + } + return q.Encode() +} + +func appendHostQuery(rawURL, host string) string { + host = strings.TrimSpace(host) + if host == "" { + return rawURL + } + return rawURL + "&host=" + jsQueryEscape(host) +} + +func jsQueryEscape(raw string) string { + replacer := strings.NewReplacer( + "+", "%20", + "%21", "!", + "%27", "'", + "%28", "(", + "%29", ")", + "%2A", "*", + "%7E", "~", + ) + return replacer.Replace(url.QueryEscape(raw)) +} + +func sha1HexReader(r io.Reader) (string, error) { + h := sha1.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func sha1HexString(raw string) string { + sum := sha1.Sum([]byte(raw)) + return hex.EncodeToString(sum[:]) +} + +func progressRange(up driver.UpdateProgress, start, end float64) driver.UpdateProgress { + if up == nil { + return nil + } + return model.UpdateProgressWithRange(up, start, end) +} + +func reportByteProgress(up driver.UpdateProgress, done, total int64) { + if up == nil { + return + } + if total <= 0 { + up(100) + return + } + up(float64(done) / float64(total) * 100) +} + +func reportUploadProgress(up driver.UpdateProgress, done, total int64) { + if up == nil { + return + } + if total <= 0 { + up(100) + return + } + if done > total { + done = total + } + up(10 + float64(done)/float64(total)*90) +} + +func sleepWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +type blockInfoMap map[string]interface{} + +func (r *openUploadRequestResp) blockInfoForChunk(index int) blockInfoMap { + if index <= 0 || index > len(r.Data.BlockInfo) { + return blockInfoMap{} + } + return blockInfoMap(r.Data.BlockInfo[index-1]) +} + +func (m blockInfoMap) stringValue(key string) string { + value, ok := m[key] + if !ok || value == nil { + return "" + } + switch v := value.(type) { + case string: + return v + case float64: + return strconv.FormatInt(int64(v), 10) + case int: + return strconv.Itoa(v) + case int64: + return strconv.FormatInt(v, 10) + case bool: + if v { + return "1" + } + return "0" + default: + return fmt.Sprint(v) + } +} + +func (m blockInfoMap) found() int64 { + raw := strings.TrimSpace(m.stringValue("found")) + if raw == "" { + return 0 + } + value, _ := strconv.ParseInt(raw, 10, 64) + return value +} + +func (m blockInfoMap) extraFields() map[string]string { + extras := make(map[string]string) + for key := range m { + switch key { + case "bhash", "bidx", "boffset", "bsize", "filename", "filesize", "q", "t", "token", "tid", "found", "url": + continue + } + value := strings.TrimSpace(m.stringValue(key)) + if value != "" { + extras[key] = value + } + } + return extras +} + +func (r *openUploadFinalizeResp) autoCommit() bool { + raw, ok := r.Data["autoCommit"] + if !ok { + return false + } + switch v := raw.(type) { + case bool: + return v + case string: + return strings.EqualFold(v, "true") || v == "1" + case float64: + return int64(v) == 1 + case int: + return v == 1 + case int64: + return v == 1 + default: + return false + } +} + +func (r *openUploadFinalizeResp) stringValue(key string) string { + if r == nil || r.Data == nil { + return "" + } + value, ok := r.Data[key] + if !ok || value == nil { + return "" + } + switch v := value.(type) { + case string: + return v + case float64: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(v, 10) + case int: + return strconv.Itoa(v) + default: + return fmt.Sprint(v) + } +} diff --git a/drivers/yunpan360/util.go b/drivers/yunpan360/util.go new file mode 100644 index 00000000000..2718f9c99cc --- /dev/null +++ b/drivers/yunpan360/util.go @@ -0,0 +1,909 @@ +package yunpan360 + +import ( + "context" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "html" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + baseURL = "https://www.yunpan.com" + indexPath = "/file/index" + listPath = "/file/list" + downloadPath = "/file/download" + openAPIProdURL = "https://openapi.eyun.360.cn/intf.php" + openAPITestURL = "https://qaopen.eyun.360.cn/intf.php" + openAPIHGTestURL = "https://hg-openapi.eyun.360.cn/intf.php" +) + +func (d *Yunpan360) listPage(ctx context.Context, dirPath string, page, pageSize int) (ListResp, error) { + if d.authMode() == authTypeAPIKey { + return d.listOpenPage(ctx, dirPath, page, pageSize) + } + return d.listCookiePage(ctx, dirPath, page, pageSize) +} + +func (d *Yunpan360) listCookiePage(ctx context.Context, dirPath string, page, pageSize int) (*CookieListResp, error) { + var resp CookieListResp + err := d.cookieRequestForm(ctx, listPath, map[string]string{ + "path": requestPath(dirPath), + "page": strconv.Itoa(page), + "page_size": strconv.Itoa(pageSize), + "order": requestOrder(d.OrderDirection), + "field": "file_name", + "focus_nid": "0", + }, &resp) + if err != nil { + return nil, err + } + d.cacheCookieDownloadSession(resp.GetOwnerQID(), resp.Token) + return &resp, nil +} + +func (d *Yunpan360) cookieRequestForm(ctx context.Context, apiPath string, form map[string]string, out interface{}) error { + req := base.RestyClient.R(). + SetContext(ctx). + SetHeaders(map[string]string{ + "Accept": "text/javascript, text/html, application/xml, text/xml, */*", + "Content-Type": "application/x-www-form-urlencoded", + "Cookie": d.Cookie, + "Origin": baseURL, + "Referer": baseURL + "/file/index", + "X-Requested-With": "XMLHttpRequest", + }). + SetFormData(form) + + res, err := req.Execute(http.MethodPost, baseURL+apiPath) + if err != nil { + return err + } + + var baseResp BaseResp + if err := utils.Json.Unmarshal(res.Body(), &baseResp); err != nil { + return err + } + if baseResp.Errno != 0 { + if baseResp.Errmsg == "" { + return fmt.Errorf("yunpan request failed: errno=%d", baseResp.Errno) + } + return errors.New(baseResp.Errmsg) + } + if out == nil { + return nil + } + return utils.Json.Unmarshal(res.Body(), out) +} + +func requestPath(dirPath string) string { + path := normalizeRemotePath(dirPath) + if path == "" { + return "/" + } + return path +} + +func requestOrder(order string) string { + if strings.EqualFold(order, "desc") { + return "desc" + } + return "asc" +} + +func (d *Yunpan360) cookiePage(ctx context.Context, pagePath string) ([]byte, error) { + req := base.RestyClient.R(). + SetContext(ctx). + SetHeaders(map[string]string{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Cookie": d.Cookie, + "Referer": baseURL + indexPath, + }) + res, err := req.Get(baseURL + pagePath) + if err != nil { + return nil, err + } + return res.Body(), nil +} + +func openAPIURL(env string) string { + switch env { + case "test": + return openAPITestURL + case "hgtest": + return openAPIHGTestURL + default: + return openAPIProdURL + } +} + +func openClientSecretForEnv(env string) string { + if env == "test" { + return openClientSecretQA + } + return openClientSecret +} + +func phpQueryEscape(raw string) string { + escaped := url.QueryEscape(raw) + return strings.ReplaceAll(escaped, "~", "%7E") +} + +func openSign(accessToken, qid, method string, extra map[string]string) string { + params := map[string]string{ + "access_token": accessToken, + "method": method, + "qid": qid, + } + for key, value := range extra { + if value != "" { + params[key] = value + } + } + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sortStrings(keys) + pairs := make([]string, 0, len(keys)) + for _, key := range keys { + pairs = append(pairs, key+"="+phpQueryEscape(params[key])) + } + sum := md5.Sum([]byte(strings.Join(pairs, "&") + openSignSecret)) + return hex.EncodeToString(sum[:]) +} + +func sortStrings(values []string) { + for i := 0; i < len(values); i++ { + for j := i + 1; j < len(values); j++ { + if values[j] < values[i] { + values[i], values[j] = values[j], values[i] + } + } + } +} + +func (d *Yunpan360) getOpenAuth(ctx context.Context) (*OpenAuthInfo, error) { + d.authMu.Lock() + defer d.authMu.Unlock() + + if d.cachedOpenAuth != nil && time.Now().Before(d.openAuthExpire) { + auth := *d.cachedOpenAuth + return &auth, nil + } + + reqURL := openAPIURL(d.EcsEnv) + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Accept", "application/json"). + SetHeader("api_key", d.APIKey). + SetQueryParams(map[string]string{ + "method": "Oauth.getAccessTokenByApiKey", + "client_id": openClientID, + "client_secret": openClientSecretForEnv(d.EcsEnv), + "grant_type": "authorization_code", + "sub_channel": d.SubChannel, + "api_key": d.APIKey, + }) + + res, err := req.Get(reqURL) + if err != nil { + return nil, err + } + + var resp OpenAuthResp + if err := utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + if resp.Errno != 0 { + if resp.Errmsg == "" { + return nil, fmt.Errorf("yunpan auth failed: errno=%d", resp.Errno) + } + return nil, errors.New(resp.Errmsg) + } + + auth := &OpenAuthInfo{ + AccessToken: resp.Data.AccessToken, + Qid: resp.Data.Qid, + Token: resp.Data.Token, + SubChannel: d.SubChannel, + } + d.cachedOpenAuth = auth + d.openAuthExpire = time.Now().Add(50 * time.Minute) + + copied := *auth + return &copied, nil +} + +func (d *Yunpan360) openBaseParams(auth *OpenAuthInfo, method string, signParams map[string]string, withSign bool) map[string]string { + params := map[string]string{ + "method": method, + "access_token": auth.AccessToken, + "qid": auth.Qid, + "sub_channel": auth.SubChannel, + } + if withSign { + params["sign"] = openSign(auth.AccessToken, auth.Qid, method, signParams) + } else { + params["sign"] = "" + } + return params +} + +func (d *Yunpan360) openGET(ctx context.Context, method string, signParams map[string]string, query map[string]string, out interface{}, withSign bool) error { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return err + } + params := d.openBaseParams(auth, method, signParams, withSign) + for key, value := range query { + params[key] = value + } + + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Access-Token", auth.AccessToken). + SetQueryParams(params) + res, err := req.Get(openAPIURL(d.EcsEnv)) + if err != nil { + return err + } + return decodeBaseResp(res.Body(), out) +} + +func (d *Yunpan360) openPOST(ctx context.Context, method string, signParams map[string]string, query, body map[string]string, out interface{}, withSign bool) error { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return err + } + queryParams := map[string]string{} + for key, value := range query { + queryParams[key] = value + } + bodyParams := map[string]string{} + for key, value := range body { + bodyParams[key] = value + } + + baseParams := d.openBaseParams(auth, method, signParams, withSign) + if len(queryParams) == 0 { + bodyParams = mergeStringMaps(baseParams, bodyParams) + } else { + queryParams = mergeStringMaps(baseParams, queryParams) + } + + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Access-Token", auth.AccessToken). + SetHeader("Content-Type", "application/x-www-form-urlencoded") + if len(queryParams) > 0 { + req.SetQueryParams(queryParams) + } + if len(bodyParams) > 0 { + req.SetFormData(bodyParams) + } + res, err := req.Post(openAPIURL(d.EcsEnv)) + if err != nil { + return err + } + return decodeBaseResp(res.Body(), out) +} + +func decodeBaseResp(body []byte, out interface{}) error { + var baseResp BaseResp + if err := utils.Json.Unmarshal(body, &baseResp); err != nil { + return err + } + if baseResp.Errno != 0 { + if baseResp.Errmsg == "" { + return fmt.Errorf("yunpan request failed: errno=%d", baseResp.Errno) + } + return errors.New(baseResp.Errmsg) + } + if out == nil { + return nil + } + return utils.Json.Unmarshal(body, out) +} + +func mergeStringMaps(baseMap, extra map[string]string) map[string]string { + merged := map[string]string{} + for key, value := range baseMap { + merged[key] = value + } + for key, value := range extra { + merged[key] = value + } + return merged +} + +func (d *Yunpan360) cookieDownloadURL(ctx context.Context, file model.Obj) (*CookieDownloadResp, error) { + resp, err := d.cookieDownloadURLOnce(ctx, file, false) + if err == nil { + return resp, nil + } + + d.invalidateCookieDownloadSession() + return d.cookieDownloadURLOnce(ctx, file, true) +} + +func (d *Yunpan360) cookieDownloadURLOnce(ctx context.Context, file model.Obj, refresh bool) (*CookieDownloadResp, error) { + nid := strings.TrimSpace(file.GetID()) + if nid == "" { + return nil, errors.New("missing file id") + } + + fname := normalizeRemotePath(file.GetPath()) + if fname == "" { + return nil, errors.New("missing file path") + } + + ownerQID, token, err := d.resolveCookieDownloadParams(ctx, file, refresh) + if err != nil { + return nil, err + } + + var resp CookieDownloadResp + err = d.cookieRequestForm(ctx, downloadPath, map[string]string{ + "nid": nid, + "fname": fname, + "owner_qid": ownerQID, + "token": token, + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) cookieRename(ctx context.Context, srcObj model.Obj, newName string) error { + path := normalizeRemotePath(srcObj.GetPath()) + if path == "" { + return errors.New("missing object path") + } + nid := strings.TrimSpace(srcObj.GetID()) + if nid == "" { + return errors.New("missing object id") + } + + ownerQID, err := d.resolveCookieOwnerQID(ctx, srcObj, false) + if err != nil { + return err + } + + return d.cookieRequestForm(ctx, "/file/rename", map[string]string{ + "path": path, + "nid": nid, + "newpath": strings.TrimSuffix(strings.TrimSpace(newName), "/"), + "owner_qid": ownerQID, + }, nil) +} + +func (d *Yunpan360) resolveCookieDownloadParams(ctx context.Context, file model.Obj, refresh bool) (string, string, error) { + ownerQID := sanitizeOwnerQID(d.OwnerQID) + token := strings.TrimSpace(d.DownloadToken) + + if obj, ok := file.(*YunpanObject); ok { + ownerQID = firstNonEmpty(sanitizeOwnerQID(obj.OwnerQID), ownerQID) + token = firstNonEmpty(strings.TrimSpace(obj.DownloadToken), token) + } + + ownerQID = firstNonEmpty(ownerQID, + sanitizeOwnerQID(getCookieValue(d.Cookie, "owner_qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "ownerQid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "QID")), + ) + token = firstNonEmpty(token, + getCookieValue(d.Cookie, "download_token"), + getCookieValue(d.Cookie, "token"), + getCookieValue(d.Cookie, "Token"), + ) + + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + return ownerQID, token, nil + } + + if !refresh { + if cached := d.getCachedCookieDownloadSession(); cached != nil { + ownerQID = firstNonEmpty(ownerQID, cached.OwnerQID) + token = firstNonEmpty(token, cached.Token) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + return ownerQID, token, nil + } + } + + resp, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + if err == nil && resp != nil { + ownerQID = firstNonEmpty(ownerQID, resp.GetOwnerQID()) + token = firstNonEmpty(token, strings.TrimSpace(resp.Token)) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + d.cacheCookieDownloadSession(ownerQID, token) + return ownerQID, token, nil + } + + pageSession, err := d.getCookieDownloadSessionFromPage(ctx) + if err == nil && pageSession != nil { + ownerQID = firstNonEmpty(ownerQID, pageSession.OwnerQID) + token = firstNonEmpty(token, pageSession.Token) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID == "" || token == "" { + return "", "", errors.New("missing owner_qid or download_token for cookie mode") + } + + d.cacheCookieDownloadSession(ownerQID, token) + return ownerQID, token, nil +} + +func (d *Yunpan360) resolveCookieOwnerQID(ctx context.Context, file model.Obj, refresh bool) (string, error) { + ownerQID := sanitizeOwnerQID(d.OwnerQID) + + if obj, ok := file.(*YunpanObject); ok { + ownerQID = firstNonEmpty(sanitizeOwnerQID(obj.OwnerQID), ownerQID) + } + + ownerQID = firstNonEmpty(ownerQID, + sanitizeOwnerQID(getCookieValue(d.Cookie, "owner_qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "ownerQid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "QID")), + ) + if ownerQID != "" { + return ownerQID, nil + } + + if !refresh { + if cached := d.getCachedCookieDownloadSession(); cached != nil { + ownerQID = firstNonEmpty(ownerQID, cached.OwnerQID) + } + if ownerQID != "" { + return ownerQID, nil + } + } + + resp, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + if err == nil && resp != nil { + ownerQID = firstNonEmpty(ownerQID, resp.GetOwnerQID()) + } + if ownerQID != "" { + return ownerQID, nil + } + + pageSession, err := d.getCookieDownloadSessionFromPage(ctx) + if err == nil && pageSession != nil { + ownerQID = firstNonEmpty(ownerQID, pageSession.OwnerQID) + } + if ownerQID == "" { + return "", errors.New("missing owner_qid for cookie mode") + } + return ownerQID, nil +} + +func (d *Yunpan360) getCookieDownloadSessionFromPage(ctx context.Context) (*CookieDownloadSession, error) { + body, err := d.cookiePage(ctx, indexPath) + if err != nil { + return nil, err + } + session := parseCookieDownloadSessionFromText(string(body)) + if session == nil { + return nil, errors.New("failed to parse cookie download session from page") + } + d.cacheCookieSession(session) + return session, nil +} + +func (d *Yunpan360) getCachedCookieDownloadSession() *CookieDownloadSession { + d.authMu.Lock() + defer d.authMu.Unlock() + + if d.cachedCookieSession == nil || time.Now().After(d.cookieSessionExpire) { + return nil + } + session := *d.cachedCookieSession + return &session +} + +func (d *Yunpan360) cacheCookieDownloadSession(ownerQID, token string) { + d.cacheCookieSession(&CookieDownloadSession{ + OwnerQID: ownerQID, + Token: token, + }) +} + +func (d *Yunpan360) cacheCookieSession(session *CookieDownloadSession) { + if session == nil { + return + } + + cached := &CookieDownloadSession{ + OwnerQID: sanitizeOwnerQID(session.OwnerQID), + Token: strings.TrimSpace(session.Token), + } + if cached.OwnerQID == "" && cached.Token != "" { + cached.OwnerQID = ownerQIDFromToken(cached.Token) + } + if cached.OwnerQID == "" || cached.Token == "" { + return + } + + d.authMu.Lock() + defer d.authMu.Unlock() + + d.cachedCookieSession = cached + d.cookieSessionExpire = time.Now().Add(10 * time.Minute) +} + +func (d *Yunpan360) invalidateCookieDownloadSession() { + d.authMu.Lock() + defer d.authMu.Unlock() + + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} +} + +func getCookieValue(rawCookie, name string) string { + for _, item := range strings.Split(rawCookie, ";") { + part := strings.TrimSpace(item) + if part == "" { + continue + } + key, value, ok := strings.Cut(part, "=") + if !ok || key != name { + continue + } + value = strings.TrimSpace(value) + value = strings.Trim(value, "\"") + unescaped, err := url.QueryUnescape(value) + if err == nil { + return strings.TrimSpace(unescaped) + } + return value + } + return "" +} + +func ownerQIDFromToken(token string) string { + parts := strings.Split(strings.TrimSpace(token), ".") + if len(parts) < 4 { + return "" + } + qid := strings.TrimSpace(parts[3]) + for _, ch := range qid { + if ch < '0' || ch > '9' { + return "" + } + } + return qid +} + +func sanitizeOwnerQID(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "0" { + return "" + } + return raw +} + +func parseCookieDownloadSessionFromText(text string) *CookieDownloadSession { + token := extractFirstMatch(text, + `(?i)["']download_token["']\s*[:=]\s*["']([^"'<>]+)["']`, + `(?i)["']token["']\s*[:=]\s*["']([^"'<>]+)["']`, + `(?i)\btoken\s*[:=]\s*["']([^"'<>]+)["']`, + ) + ownerQID := extractFirstMatch(text, + `(?i)["']owner_qid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)["']ownerQid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)["']qid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)\bowner_qid\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)\bqid\s*[:=]\s*["']?([0-9]+)["']?`, + ) + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID == "" || token == "" { + return nil + } + return &CookieDownloadSession{ + OwnerQID: ownerQID, + Token: token, + } +} + +func extractFirstMatch(text string, patterns ...string) string { + return extractFirstValidatedMatch(nil, text, patterns...) +} + +func extractFirstValidatedMatch(validate func(string) bool, text string, patterns ...string) string { + for _, pattern := range patterns { + re := regexp.MustCompile(pattern) + for _, matches := range re.FindAllStringSubmatch(text, -1) { + if len(matches) < 2 { + continue + } + value := html.UnescapeString(strings.TrimSpace(matches[1])) + value = strings.Trim(value, "\"'") + if value == "" { + continue + } + if validate == nil || validate(value) { + return value + } + } + } + return "" +} + +func (d *Yunpan360) listOpenPage(ctx context.Context, dirPath string, page, pageSize int) (*OpenListResp, error) { + var resp OpenListResp + path := ensureDirAPIPath(dirPath) + params := map[string]string{ + "path": path, + "page": strconv.Itoa(page), + "page_size": strconv.Itoa(pageSize), + } + err := d.openGET(ctx, "File.getList", params, params, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openUserInfo(ctx context.Context) (*OpenUserInfoResp, error) { + var resp OpenUserInfoResp + err := d.openGET(ctx, "User.getUserDetail", nil, nil, &resp, false) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openDownloadURL(ctx context.Context, file model.Obj) (*OpenDownloadResp, error) { + var resp OpenDownloadResp + signParams := map[string]string{} + body := map[string]string{} + + if file.GetPath() != "" { + signParams["fpath"] = normalizeRemotePath(file.GetPath()) + body["fpath"] = signParams["fpath"] + } else if file.GetID() != "" { + signParams["nid"] = file.GetID() + body["nid"] = file.GetID() + } else { + return nil, errors.New("missing file path and id") + } + + err := d.openPOST(ctx, "MCP.getDownLoadUrl", signParams, nil, body, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) cookieMakeDir(ctx context.Context, fullPath string) (*CookieMkdirResp, error) { + var resp CookieMkdirResp + body := map[string]string{ + "path": ensureDirAPIPath(fullPath), + "owner_qid": "0", + } + err := d.cookieRequestForm(ctx, "/file/mkdir", body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openMakeDir(ctx context.Context, fullPath string) (*OpenMkdirResp, error) { + var resp OpenMkdirResp + body := map[string]string{"fname": ensureDirAPIPath(fullPath)} + err := d.openPOST(ctx, "File.mkdir", body, nil, body, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openRename(ctx context.Context, srcName, newName string) error { + signParams := map[string]string{ + "src_name": srcName, + "new_name": newName, + } + return d.openPOST(ctx, "File.rename", signParams, nil, signParams, nil, true) +} + +func (d *Yunpan360) cookieMove(ctx context.Context, srcPath, dstPath string) error { + var resp CookieMoveResp + body := map[string]string{ + "path[]": srcPath, + "newpath": ensureDirAPIPath(dstPath), + } + if err := d.cookieRequestForm(ctx, "/file/move", body, &resp); err != nil { + return err + } + if !resp.Data.IsAsync { + return nil + } + return d.waitCookieAsyncTask(ctx, resp.Data.TaskID) +} + +func (d *Yunpan360) cookieRecycle(ctx context.Context, obj model.Obj) error { + path := apiPathForObj(obj) + if path == "" { + return errors.New("missing object path") + } + ownerQID, err := d.resolveCookieOwnerQID(ctx, obj, false) + if err != nil { + return err + } + + var resp CookieRecycleResp + if err := d.cookieRequestForm(ctx, "/file/recycle", map[string]string{ + "path[]": path, + "owner_qid": ownerQID, + }, &resp); err != nil { + return err + } + if !resp.Data.IsAsync { + return nil + } + return d.waitCookieAsyncTask(ctx, resp.Data.TaskID, 3008) +} + +func (d *Yunpan360) cookieAsyncQuery(ctx context.Context, taskID string) (*CookieAsyncQueryResp, error) { + var resp CookieAsyncQueryResp + err := d.cookieRequestForm(ctx, "/async/query", map[string]string{ + "task_id": strings.TrimSpace(taskID), + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) waitCookieAsyncTask(ctx context.Context, taskID string, toleratedErrnos ...int) error { + taskID = strings.TrimSpace(taskID) + if taskID == "" { + return nil + } + + tolerated := map[int]struct{}{} + for _, errno := range toleratedErrnos { + tolerated[errno] = struct{}{} + } + + for attempt := 0; attempt < 15; attempt++ { + resp, err := d.cookieAsyncQuery(ctx, taskID) + if err == nil && resp != nil { + if task, ok := resp.Data[taskID]; ok { + done, taskErr := checkCookieAsyncTask(task, tolerated) + if done { + return taskErr + } + } + } + if attempt == 14 { + break + } + if err := sleepWithContext(ctx, 300*time.Millisecond); err != nil { + return err + } + } + + // Keep prior behavior when the async task is still pending after the probe window. + return nil +} + +func checkCookieAsyncTask(task CookieAsyncTask, toleratedErrnos map[int]struct{}) (bool, error) { + if task.Status != 10 { + return false, nil + } + if task.Errno == 0 { + return true, nil + } + if _, ok := toleratedErrnos[task.Errno]; ok { + return true, nil + } + if strings.TrimSpace(task.Errstr) != "" { + return true, errors.New(task.Errstr) + } + if strings.TrimSpace(task.Action) != "" { + return true, fmt.Errorf("yunpan async task %s failed: errno=%d", task.Action, task.Errno) + } + return true, fmt.Errorf("yunpan async task failed: errno=%d", task.Errno) +} + +func (d *Yunpan360) openMove(ctx context.Context, srcName, dstPath string) error { + signParams := map[string]string{ + "src_name": srcName, + "new_name": dstPath, + } + return d.openPOST(ctx, "File.move", signParams, nil, signParams, nil, true) +} + +func (d *Yunpan360) openDelete(ctx context.Context, targetPath string) error { + return d.openPOST(ctx, "File.delete", nil, nil, map[string]string{ + "fname": targetPath, + }, nil, true) +} + +func apiPathForObj(obj model.Obj) string { + if obj.IsDir() { + return ensureDirAPIPath(obj.GetPath()) + } + return normalizeRemotePath(obj.GetPath()) +} + +func ensureDirSuffix(name string) string { + name = strings.TrimSpace(name) + if name == "" || strings.HasSuffix(name, "/") { + return name + } + return name + "/" +} + +func ensureDirAPIPath(p string) string { + p = normalizeRemotePath(p) + if p == "" || p == "/" { + return "/" + } + return p + "/" +} + +func cloneObj(src model.Obj, newPath, newName string) model.Obj { + obj := model.Object{ + ID: src.GetID(), + Path: normalizeRemotePath(newPath), + Name: newName, + Size: src.GetSize(), + Modified: src.ModTime(), + Ctime: src.CreateTime(), + IsFolder: src.IsDir(), + HashInfo: src.GetHash(), + } + if raw, ok := src.(*YunpanObject); ok { + return &YunpanObject{ + Object: obj, + Thumbnail: raw.Thumbnail, + OwnerQID: raw.OwnerQID, + DownloadToken: raw.DownloadToken, + } + } + return &obj +} + +func absoluteURL(raw string) string { + if raw == "" { + return "" + } + if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") { + return raw + } + if strings.HasPrefix(raw, "/") { + return baseURL + raw + } + return baseURL + "/" + raw +} diff --git a/go.mod b/go.mod index e8afe0e7a62..46d43d9d116 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,15 @@ module github.com/alist-org/alist/v3 -go 1.23.4 +go 1.25.0 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 - github.com/SheltonZhu/115driver v1.0.34 + github.com/ProtonMail/gopenpgp/v2 v2.7.4 + github.com/SheltonZhu/115driver v1.2.3-1 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 @@ -26,22 +29,26 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dlclark/regexp2 v1.11.4 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 + github.com/fatedier/frp v0.68.0 github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 github.com/go-resty/resty/v2 v2.14.0 github.com/go-webauthn/webauthn v0.11.1 - github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/hekmon/transmissionrpc/v3 v3.0.0 + github.com/henrybear327/Proton-API-Bridge v1.0.0 + github.com/henrybear327/go-proton-api v1.0.0 github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 github.com/kdomanski/iso9660 v0.4.0 - github.com/larksuite/oapi-sdk-go/v3 v3.3.1 + github.com/larksuite/oapi-sdk-go/v3 v3.6.1 + github.com/mark3labs/mcp-go v0.48.0 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.27.2 github.com/mholt/archives v0.1.0 @@ -56,7 +63,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 @@ -65,11 +72,11 @@ require ( github.com/xhofe/wopan-sdk-go v0.1.3 github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.36.0 + golang.org/x/crypto v0.46.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 - golang.org/x/net v0.38.0 - golang.org/x/oauth2 v0.22.0 + golang.org/x/net v0.48.0 + golang.org/x/oauth2 v0.34.0 golang.org/x/time v0.8.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 @@ -80,9 +87,54 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect + github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect + github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect + github.com/ProtonMail/go-srp v0.0.7 // indirect + github.com/PuerkitoBio/goquery v1.8.1 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect + github.com/bradenaw/juniper v0.15.2 // indirect + github.com/coreos/go-oidc/v3 v3.14.1 // indirect + github.com/cronokirby/saferith v0.33.0 // indirect + github.com/emersion/go-message v0.18.0 // indirect + github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect + github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect + github.com/fatedier/golib v0.5.1 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/google/jsonschema-go v0.4.2 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect + github.com/klauspost/reedsolomon v1.12.0 // indirect + github.com/pion/dtls/v2 v2.2.7 // indirect + github.com/pion/logging v0.2.2 // indirect + github.com/pion/stun/v2 v2.0.0 // indirect + github.com/pion/transport/v2 v2.2.1 // indirect + github.com/pion/transport/v3 v3.0.1 // indirect + github.com/pires/go-proxyproto v0.7.0 // indirect + github.com/quic-go/quic-go v0.55.0 // indirect + github.com/relvacode/iso8601 v1.3.0 // indirect + github.com/samber/lo v1.47.0 // indirect + github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/templexxx/cpu v0.1.1 // indirect + github.com/templexxx/xorsimd v0.4.3 // indirect + github.com/tjfoc/gmsm v1.4.1 // indirect + github.com/vishvananda/netlink v1.3.0 // indirect + github.com/vishvananda/netns v0.0.4 // indirect + github.com/xtaci/kcp-go/v5 v5.6.13 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect + golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apimachinery v0.28.8 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) require ( @@ -107,9 +159,8 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/pgzip v1.2.6 // indirect - github.com/kr/text v0.2.0 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 @@ -170,7 +221,7 @@ require ( github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/go-webauthn/x v0.1.12 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -181,8 +232,8 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -236,7 +287,7 @@ require ( github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/shirou/gopsutil/v3 v3.24.4 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/shoenig/go-m1cpu v0.2.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -252,15 +303,15 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.12.0 - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 - golang.org/x/tools v0.24.0 // indirect + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 + golang.org/x/tools v0.39.0 // indirect google.golang.org/api v0.169.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/grpc v1.66.0 - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/grpc v1.79.3 + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect @@ -268,4 +319,8 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect ) -// replace github.com/xhofe/115-sdk-go => ../../xhofe/115-sdk-go +replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton-api v1.0.0 + +replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed + +replace github.com/SheltonZhu/115driver => github.com/okatu-loli/115driver v1.2.3-1 diff --git a/go.sum b/go.sum index 6fbaeb2b3ef..0d42e1fe1f2 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,8 @@ cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= -cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= -cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -21,27 +21,52 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= +github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= +github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 h1:ikwCzeqoqN6wvBHOB9OI6dde/jbV7EoTMpUcxtYl5Po= github.com/KirCute/ftpserverlib-pasvportmap v1.25.0/go.mod h1:v0NgMtKDDi/6CM6r4P+daCljCW3eO9yS+Z+pZDTKo1E= github.com/KirCute/sftpd-alist v0.0.12 h1:GNVM5QLbQLAfXP4wGUlXFA2IO6fVek0n0IsGnOuISdg= github.com/KirCute/sftpd-alist v0.0.12/go.mod h1:2wNK7yyW2XfjyJq10OY6xB4COLac64hOwfV6clDJn6s= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= +github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo= +github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= +github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= +github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= +github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk= +github.com/ProtonMail/gopenpgp/v2 v2.7.4 h1:Vz/8+HViFFnf2A6XX8JOvZMrA6F5puwNvvF21O1mRlo= +github.com/ProtonMail/gopenpgp/v2 v2.7.4/go.mod h1:IhkNEDaxec6NyzSI0PlxapinnwPVIESk8/76da3Ct3g= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.0.34 h1:zhMLp4vgq7GksqvSxQQDOVfK6EOHldQl4b2n8tnZ+EE= -github.com/SheltonZhu/115driver v1.0.34/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= @@ -63,6 +88,11 @@ github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAP github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -128,6 +158,9 @@ github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bradenaw/juniper v0.15.2 h1:0JdjBGEF2jP1pOxmlNIrPhAoQN7Ng5IMAY5D0PHMW4U= +github.com/bradenaw/juniper v0.15.2/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= @@ -158,6 +191,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= @@ -165,14 +199,16 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk= +github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -194,10 +230,22 @@ github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj6 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= +github.com/emersion/go-message v0.18.0 h1:7LxAXHRpSeoO/Wom3ZApVZYG7c3d17yCScYce8WiXA8= +github.com/emersion/go-message v0.18.0/go.mod h1:Zi69ACvzaoV/MBnrxfVBPV3xWEuCmC2nEN39oJF4B8A= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fatedier/frp v0.68.0 h1:woKC31EpgCLQDirRAOAUgHP3IRxLu2mBHS6zGpcw7tw= +github.com/fatedier/frp v0.68.0/go.mod h1:qFdez6Z+RDqoqF1xJhh48+IW91SVQ+pNWnrmcl43Wjs= +github.com/fatedier/golib v0.5.1 h1:hcKAnaw5mdI/1KWRGejxR+i1Hn/NvbY5UsMKDr7o13M= +github.com/fatedier/golib v0.5.1/go.mod h1:W6kIYkIFxHsTzbgqg5piCxIiDo4LzwgTY6R5W8l9NFQ= github.com/fclairamb/go-log v0.5.0 h1:Gz9wSamEaA6lta4IU2cjJc2xSq5sV5VYSB5w/SUHhVc= github.com/fclairamb/go-log v0.5.0/go.mod h1:XoRO1dYezpsGmLLkZE9I+sHqpqY65p8JA+Vqblb7k40= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -206,6 +254,8 @@ github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/sr github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= @@ -223,13 +273,15 @@ github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -255,10 +307,11 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -275,6 +328,12 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -283,6 +342,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -290,11 +351,14 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= +github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -313,6 +377,8 @@ github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUh github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -330,10 +396,16 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI0= github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= github.com/hekmon/transmissionrpc/v3 v3.0.0/go.mod h1:38SlNhFzinVUuY87wGj3acOmRxeYZAZfrj6Re7UgCDg= +github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0LTKWQciUm8PMZb0= +github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= +github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= +github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -347,12 +419,12 @@ github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2 github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= -github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.0 h1:T/dI+2TvmI2H8s/KH1/lXIbz1CUFk3gn5oTjr0/mBsE= +github.com/jackc/pgx/v5 v5.9.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -388,6 +460,8 @@ github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuV github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/reedsolomon v1.12.0 h1:I5FEp3xSwVCcEh3F5A7dofEfhXdF/bWhQWPH+XwBFno= +github.com/klauspost/reedsolomon v1.12.0/go.mod h1:EPLZJeh4l27pUGC3aXOjheaoh1I9yut7xTURiW3LQ9Y= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -398,8 +472,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/larksuite/oapi-sdk-go/v3 v3.3.1 h1:DLQQEgHUAGZB6RVlceB1f6A94O206exxW2RIMH+gMUc= -github.com/larksuite/oapi-sdk-go/v3 v3.3.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/larksuite/oapi-sdk-go/v3 v3.6.1 h1:vAdu+sX9yXNkKnKnYQeIv6yBkjP37Q1JEJHmMa2eCjQ= +github.com/larksuite/oapi-sdk-go/v3 v3.6.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= @@ -415,6 +491,8 @@ github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIg github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mark3labs/mcp-go v0.48.0 h1:o+MXuGW/HCeR2ny5LcAcZQn2bo6I2xaZMEHnpRG+dtw= +github.com/mark3labs/mcp-go v0.48.0/go.mod h1:JKTC7R2LLVagkEWK7Kwu7DbmA6iIvnNAod6yrHiQMag= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= @@ -483,6 +561,8 @@ github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/okatu-loli/115driver v1.2.3-1 h1:UoBEREqh6RD6WlxiJ2Z29JxNZ/UcoChvdHn9r9Tx7nI= +github.com/okatu-loli/115driver v1.2.3-1/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= @@ -492,6 +572,20 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0= +github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= +github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= +github.com/pires/go-proxyproto v0.7.0 h1:IukmRewDQFWC7kfnb66CSomk2q/seBuilHBYFwyq0Hs= +github.com/pires/go-proxyproto v0.7.0/go.mod h1:Vz/1JPY/OACxWGQNIRY2BeyDmpoaWmEP40O9LbuiFR4= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -518,8 +612,12 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk= +github.com/quic-go/quic-go v0.55.0/go.mod h1:DR51ilwU1uE164KuWXhinFcKWGlEjzys2l8zUl5Ss1U= github.com/rclone/rclone v1.67.0 h1:yLRNgHEG2vQ60HCuzFqd0hYwKCRuWuvPUhvhMJ2jI5E= github.com/rclone/rclone v1.67.0/go.mod h1:Cb3Ar47M/SvwfhAjZTbVXdtrP/JLtPFCq2tkdtBVC6w= +github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= +github.com/relvacode/iso8601 v1.3.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -534,22 +632,28 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= +github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -557,6 +661,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -574,16 +680,23 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543/go.mod h1:jpwqYA8KUVEvSUJHkCXsnBRJCSKP1BMa81QZ6kvRpow= +github.com/templexxx/cpu v0.1.1 h1:isxHaxBXpYFWnk2DReuKkigaZyrjs2+9ypIdGP4h+HI= +github.com/templexxx/cpu v0.1.1/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= +github.com/templexxx/xorsimd v0.4.3 h1:9AQTFHd7Bhk3dIT7Al2XeBX5DWOvsUPZCuhyAtNbHjU= +github.com/templexxx/xorsimd v0.4.3/go.mod h1:oZQcD6RFDisW2Am58dSAGwwL6rHjbzrlu25VDqfWkQg= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= +github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= +github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= @@ -608,6 +721,10 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d h1:xS9QTPgKl9ewGsAOPc+xW7DeStJDqYPfisDmeSCcbco= github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk= +github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -620,10 +737,16 @@ github.com/xhofe/tache v0.1.5 h1:ezDcgim7tj7KNMXliQsmf8BJQbaZtitfyQA9Nt+B4WM= github.com/xhofe/tache v0.1.5/go.mod h1:PYt6I/XUKliSg1uHlgsk6ha+le/f6PAvjUtFZAVl3a8= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= +github.com/xtaci/kcp-go/v5 v5.6.13 h1:FEjtz9+D4p8t2x4WjciGt/jsIuhlWjjgPCCWjrVR4Hk= +github.com/xtaci/kcp-go/v5 v5.6.13/go.mod h1:75S1AKYYzNUSXIv30h+jPKJYZUwqpfvLshu63nCNSOM= +github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37 h1:EWU6Pktpas0n8lLQwDsRyZfmkPeRbdgPtW609es+/9E= +github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37/go.mod h1:HpMP7DB2CyokmAh4lp0EQnnWhmycP/TvwBGzvuie+H0= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 h1:K8gF0eekWPEX+57l30ixxzGhHH/qscI3JCnuhbN6V4M= github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9/go.mod h1:9BnoKCcgJ/+SLhfAXj15352hTOuVmG5Gzo8xNRINfqI= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -641,14 +764,24 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -662,17 +795,20 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -709,6 +845,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -723,9 +861,11 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -734,22 +874,22 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= -golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= -golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -762,8 +902,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -784,6 +924,7 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -796,15 +937,17 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -812,13 +955,15 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -831,12 +976,13 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -874,12 +1020,18 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -910,21 +1062,29 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= -google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -932,6 +1092,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= @@ -955,11 +1117,17 @@ gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDa gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 h1:TbRPT0HtzFP3Cno1zZo7yPzEEnfu8EjLfl6IU9VfqkQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259/go.mod h1:AVgIgHMwK63XvmAzWG9vLQ41YnVHN0du0tEC46fI7yY= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/apimachinery v0.28.8 h1:hi/nrxHwk4QLV+W/SHve1bypTE59HCDorLY1stBIxKQ= +k8s.io/apimachinery v0.28.8/go.mod h1:cBnwIM3fXoRo28SqbV/Ihxf/iviw85KyXOrzxvZQ83U= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= @@ -969,4 +1137,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/archive/archives/archives.go b/internal/archive/archives/archives.go index 0a42cd0c512..e3a48b15757 100644 --- a/internal/archive/archives/archives.go +++ b/internal/archive/archives/archives.go @@ -1,10 +1,12 @@ package archives import ( + "fmt" "io" "io/fs" "os" stdpath "path" + "path/filepath" "strings" "github.com/alist-org/alist/v3/internal/archive/tool" @@ -106,7 +108,7 @@ func (Archives) Decompress(ss []*stream.SeekableStream, outputPath string, args } if stat.IsDir() { isDir = true - outputPath = stdpath.Join(outputPath, stat.Name()) + outputPath = filepath.Join(outputPath, stat.Name()) err = os.Mkdir(outputPath, 0700) if err != nil { return filterPassword(err) @@ -118,18 +120,46 @@ func (Archives) Decompress(ss []*stream.SeekableStream, outputPath string, args if err != nil { return err } + if p == path { + if d.IsDir() { + return nil + } + } relPath := strings.TrimPrefix(p, path+"/") - dstPath := stdpath.Join(outputPath, relPath) + if relPath == "" || relPath == "." { + if d.IsDir() { + return nil + } + } + dstPath, err := tool.SecureJoin(outputPath, relPath) + if err != nil { + return err + } if d.IsDir() { - err = os.MkdirAll(dstPath, 0700) - } else { - dir := stdpath.Dir(dstPath) - err = decompress(fsys, p, dir, func(_ float64) {}) + return os.MkdirAll(dstPath, 0700) } - return err + info, err := d.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, p) + } + if err := os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + return decompress(fsys, p, dstPath, func(_ float64) {}) }) } else { - err = decompress(fsys, path, outputPath, up) + entryName := stdpath.Base(path) + dstPath, e := tool.SecureJoin(outputPath, entryName) + if e != nil { + return e + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = decompress(fsys, path, dstPath, up) } return filterPassword(err) } diff --git a/internal/archive/archives/utils.go b/internal/archive/archives/utils.go index 2f499a10feb..5249862ce4b 100644 --- a/internal/archive/archives/utils.go +++ b/internal/archive/archives/utils.go @@ -1,12 +1,13 @@ package archives import ( + "fmt" "io" fs2 "io/fs" "os" - stdpath "path" "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" @@ -59,7 +60,7 @@ func filterPassword(err error) error { return err } -func decompress(fsys fs2.FS, filePath, targetPath string, up model.UpdateProgress) error { +func decompress(fsys fs2.FS, filePath, dstPath string, up model.UpdateProgress) error { rc, err := fsys.Open(filePath) if err != nil { return err @@ -69,7 +70,10 @@ func decompress(fsys fs2.FS, filePath, targetPath string, up model.UpdateProgres if err != nil { return err } - f, err := os.OpenFile(stdpath.Join(targetPath, stat.Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if !stat.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, filePath) + } + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } diff --git a/internal/archive/iso9660/iso9660.go b/internal/archive/iso9660/iso9660.go index be107d7b4c4..7de8da6f393 100644 --- a/internal/archive/iso9660/iso9660.go +++ b/internal/archive/iso9660/iso9660.go @@ -1,14 +1,14 @@ package iso9660 import ( + "io" + "os" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" "github.com/kdomanski/iso9660" - "io" - "os" - stdpath "path" ) type ISO9660 struct { @@ -78,7 +78,10 @@ func (ISO9660) Decompress(ss []*stream.SeekableStream, outputPath string, args m } if obj.IsDir() { if args.InnerPath != "/" { - outputPath = stdpath.Join(outputPath, obj.Name()) + outputPath, err = tool.SecureJoin(outputPath, obj.Name()) + if err != nil { + return err + } if err = os.MkdirAll(outputPath, 0700); err != nil { return err } diff --git a/internal/archive/iso9660/utils.go b/internal/archive/iso9660/utils.go index 0e4cfb1caf3..dd2e00354f8 100644 --- a/internal/archive/iso9660/utils.go +++ b/internal/archive/iso9660/utils.go @@ -1,10 +1,12 @@ package iso9660 import ( + "io" "os" - stdpath "path" + "path/filepath" "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" @@ -62,15 +64,26 @@ func toModelObj(file *iso9660.File) model.Obj { } func decompress(f *iso9660.File, path string, up model.UpdateProgress) error { - file, err := os.OpenFile(stdpath.Join(path, f.Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + return decompressEntry(f.Reader(), f.Size(), path, f.Name(), up) +} + +func decompressEntry(reader io.Reader, size int64, path, entryName string, up model.UpdateProgress) error { + dstPath, err := tool.SecureJoin(path, entryName) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + file, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } defer file.Close() _, err = utils.CopyWithBuffer(file, &stream.ReaderUpdatingProgress{ Reader: &stream.SimpleReaderWithSize{ - Reader: f.Reader(), - Size: f.Size(), + Reader: reader, + Size: size, }, UpdateProgress: up, }) @@ -84,7 +97,10 @@ func decompressAll(children []*iso9660.File, path string) error { if err != nil { return err } - nextPath := stdpath.Join(path, child.Name()) + nextPath, err := tool.SecureJoin(path, child.Name()) + if err != nil { + return err + } if err = os.MkdirAll(nextPath, 0700); err != nil { return err } diff --git a/internal/archive/rardecode/rardecode.go b/internal/archive/rardecode/rardecode.go index cd31d1a40e0..2848c704bee 100644 --- a/internal/archive/rardecode/rardecode.go +++ b/internal/archive/rardecode/rardecode.go @@ -1,15 +1,18 @@ package rardecode import ( + "fmt" + "io" + "os" + stdpath "path" + "path/filepath" + "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" "github.com/nwaples/rardecode/v2" - "io" - "os" - stdpath "path" - "strings" ) type RarDecoder struct{} @@ -85,7 +88,11 @@ func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, arg if header.IsDir { name = name + "/" } - err = decompress(reader, header, name, outputPath) + dstPath, e := tool.SecureJoin(outputPath, name) + if e != nil { + return e + } + err = decompress(reader, header, dstPath) if err != nil { return err } @@ -94,6 +101,7 @@ func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, arg innerPath := strings.TrimPrefix(args.InnerPath, "/") innerBase := stdpath.Base(innerPath) createdBaseDir := false + var baseDirPath string for { var header *rardecode.FileHeader header, err = reader.Next() @@ -108,22 +116,55 @@ func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, arg name = name + "/" } if name == innerPath { - err = _decompress(reader, header, outputPath, up) + if header.IsDir { + if !createdBaseDir { + baseDirPath, err = tool.SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + if err = os.MkdirAll(baseDirPath, 0700); err != nil { + return err + } + createdBaseDir = true + } + continue + } + if !header.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, header.Name) + } + dstPath, e := tool.SecureJoin(outputPath, stdpath.Base(innerPath)) + if e != nil { + return e + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(reader, header, dstPath, up) if err != nil { return err } break } else if strings.HasPrefix(name, innerPath+"/") { - targetPath := stdpath.Join(outputPath, innerBase) if !createdBaseDir { - err = os.Mkdir(targetPath, 0700) + baseDirPath, err = tool.SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + err = os.MkdirAll(baseDirPath, 0700) if err != nil { return err } createdBaseDir = true } restPath := strings.TrimPrefix(name, innerPath+"/") - err = decompress(reader, header, restPath, targetPath) + if restPath == "" || restPath == "." { + continue + } + dstPath, e := tool.SecureJoin(baseDirPath, restPath) + if e != nil { + return e + } + err = decompress(reader, header, dstPath) if err != nil { return err } diff --git a/internal/archive/rardecode/utils.go b/internal/archive/rardecode/utils.go index 5790ec58a22..e3612b363df 100644 --- a/internal/archive/rardecode/utils.go +++ b/internal/archive/rardecode/utils.go @@ -2,18 +2,20 @@ package rardecode import ( "fmt" - "github.com/alist-org/alist/v3/internal/archive/tool" - "github.com/alist-org/alist/v3/internal/errs" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/nwaples/rardecode/v2" "io" "io/fs" "os" stdpath "path" + "path/filepath" "sort" "strings" "time" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/nwaples/rardecode/v2" ) type VolumeFile struct { @@ -179,27 +181,21 @@ func getReader(ss []*stream.SeekableStream, password string) (*rardecode.Reader, return &rc.Reader, nil } -func decompress(reader *rardecode.Reader, header *rardecode.FileHeader, filePath, outputPath string) error { - targetPath := outputPath - dir, base := stdpath.Split(filePath) - if dir != "" { - targetPath = stdpath.Join(targetPath, dir) - err := os.MkdirAll(targetPath, 0700) - if err != nil { - return err - } +func decompress(reader *rardecode.Reader, header *rardecode.FileHeader, dstPath string) error { + if header.IsDir { + return os.MkdirAll(dstPath, 0700) } - if base != "" { - err := _decompress(reader, header, targetPath, func(_ float64) {}) - if err != nil { - return err - } + if !header.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, header.Name) } - return nil + if err := os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + return _decompress(reader, header, dstPath, func(_ float64) {}) } -func _decompress(reader *rardecode.Reader, header *rardecode.FileHeader, targetPath string, up model.UpdateProgress) error { - f, err := os.OpenFile(stdpath.Join(targetPath, stdpath.Base(header.Name)), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) +func _decompress(reader *rardecode.Reader, header *rardecode.FileHeader, dstPath string, up model.UpdateProgress) error { + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } diff --git a/internal/archive/tool/helper.go b/internal/archive/tool/helper.go index 20da34467b0..80254b8fd04 100644 --- a/internal/archive/tool/helper.go +++ b/internal/archive/tool/helper.go @@ -1,10 +1,12 @@ package tool import ( + "fmt" "io" "io/fs" "os" stdpath "path" + "path/filepath" "strings" "github.com/alist-org/alist/v3/internal/model" @@ -119,7 +121,30 @@ func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args mode if args.InnerPath == "/" { for i, file := range files { name := file.Name() - err = decompress(file, name, outputPath, args.Password) + info := file.FileInfo() + if info.IsDir() { + var dirPath string + dirPath, err = SecureJoin(outputPath, name) + if err != nil { + return err + } + if err = os.MkdirAll(dirPath, 0700); err != nil { + return err + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(outputPath, name) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, func(_ float64) {}) if err != nil { return err } @@ -129,25 +154,80 @@ func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args mode innerPath := strings.TrimPrefix(args.InnerPath, "/") innerBase := stdpath.Base(innerPath) createdBaseDir := false + var baseDirPath string for _, file := range files { name := file.Name() if name == innerPath { - err = _decompress(file, outputPath, args.Password, up) + info := file.FileInfo() + if info.IsDir() { + if !createdBaseDir { + baseDirPath, err = SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + if err = os.MkdirAll(baseDirPath, 0700); err != nil { + return err + } + createdBaseDir = true + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(outputPath, stdpath.Base(innerPath)) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, up) if err != nil { return err } break } else if strings.HasPrefix(name, innerPath+"/") { - targetPath := stdpath.Join(outputPath, innerBase) if !createdBaseDir { - err = os.Mkdir(targetPath, 0700) + baseDirPath, err = SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + err = os.MkdirAll(baseDirPath, 0700) if err != nil { return err } createdBaseDir = true } restPath := strings.TrimPrefix(name, innerPath+"/") - err = decompress(file, restPath, targetPath, args.Password) + if restPath == "" || restPath == "." { + continue + } + info := file.FileInfo() + if info.IsDir() { + var dirPath string + dirPath, err = SecureJoin(baseDirPath, restPath) + if err != nil { + return err + } + if err = os.MkdirAll(dirPath, 0700); err != nil { + return err + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(baseDirPath, restPath) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, func(_ float64) {}) if err != nil { return err } @@ -157,26 +237,7 @@ func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args mode return nil } -func decompress(file SubFile, filePath, outputPath, password string) error { - targetPath := outputPath - dir, base := stdpath.Split(filePath) - if dir != "" { - targetPath = stdpath.Join(targetPath, dir) - err := os.MkdirAll(targetPath, 0700) - if err != nil { - return err - } - } - if base != "" { - err := _decompress(file, targetPath, password, func(_ float64) {}) - if err != nil { - return err - } - } - return nil -} - -func _decompress(file SubFile, targetPath, password string, up model.UpdateProgress) error { +func _decompress(file SubFile, dstPath, password string, up model.UpdateProgress) error { if encrypt, ok := file.(CanEncryptSubFile); ok && encrypt.IsEncrypted() { encrypt.SetPassword(password) } @@ -185,7 +246,7 @@ func _decompress(file SubFile, targetPath, password string, up model.UpdateProgr return err } defer func() { _ = rc.Close() }() - f, err := os.OpenFile(stdpath.Join(targetPath, file.FileInfo().Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } diff --git a/internal/archive/tool/securepath.go b/internal/archive/tool/securepath.go new file mode 100644 index 00000000000..f9bd89a914a --- /dev/null +++ b/internal/archive/tool/securepath.go @@ -0,0 +1,63 @@ +package tool + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" +) + +// ErrArchiveIllegalPath indicates an archive entry path is unsafe for extraction. +var ErrArchiveIllegalPath = errors.New("archive entry has illegal path") + +// SecureJoin returns a safe extraction path for an archive entry. +// It rejects absolute paths, traversal, Windows drive/UNC paths, and NUL bytes. +func SecureJoin(baseDir, entryName string) (string, error) { + if strings.Contains(entryName, "\x00") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + normalized := strings.ReplaceAll(entryName, "\\", "/") + if strings.HasPrefix(normalized, "//") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + cleaned := path.Clean(normalized) + + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + if strings.HasPrefix(cleaned, "/") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + rel := filepath.FromSlash(cleaned) + if filepath.IsAbs(rel) || filepath.VolumeName(rel) != "" { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + if strings.HasPrefix(rel, `\\`) { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + base := filepath.Clean(baseDir) + dst := filepath.Join(base, rel) + + baseAbs, err := filepath.Abs(base) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + dstAbs, err := filepath.Abs(dst) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + + relCheck, err := filepath.Rel(baseAbs, dstAbs) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + if relCheck == ".." || strings.HasPrefix(relCheck, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + return dst, nil +} diff --git a/internal/archive/tool/securepath_test.go b/internal/archive/tool/securepath_test.go new file mode 100644 index 00000000000..78be52ee5d5 --- /dev/null +++ b/internal/archive/tool/securepath_test.go @@ -0,0 +1,48 @@ +package tool + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestSecureJoin(t *testing.T) { + baseDir := t.TempDir() + tests := []struct { + name string + entry string + wantErr bool + }{ + {name: "ok", entry: "a/b/c.txt", wantErr: false}, + {name: "parent", entry: "../evil.txt", wantErr: true}, + {name: "parent-backslash", entry: "..\\evil.txt", wantErr: true}, + {name: "abs", entry: "/tmp/evil.txt", wantErr: true}, + {name: "drive", entry: "C:\\evil.txt", wantErr: true}, + {name: "unc", entry: "\\\\server\\share\\evil.txt", wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dst, err := SecureJoin(baseDir, tc.entry) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for %q, got nil", tc.entry) + } + if !strings.Contains(err.Error(), tc.entry) { + t.Fatalf("error should include entry name %q, got %q", tc.entry, err.Error()) + } + return + } + if err != nil { + t.Fatalf("unexpected error for %q: %v", tc.entry, err) + } + rel, err := filepath.Rel(baseDir, dst) + if err != nil { + t.Fatalf("Rel failed: %v", err) + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + t.Fatalf("path escaped baseDir: %q", dst) + } + }) + } +} diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index db3e20942b6..ac36059a076 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -70,6 +70,15 @@ func InitConfig() { if !conf.Conf.Force { confFromEnv() } + if conf.Conf.TlsInsecureSkipVerify { + log.Warn("SECURITY WARNING / 安全警告:") + log.Warn("TLS certificate verification is disabled.") + log.Warn("TLS 证书校验已被禁用。") + log.Warn("This exposes all storage traffic to MitM attacks and may leak credentials or allow data tampering.") + log.Warn("这会使所有存储通信暴露于中间人攻击（MitM），可能导致凭据泄露和数据被篡改。") + log.Warn("Only use this setting if you fully understand the risks.") + log.Warn("仅在你完全理解风险的情况下使用该配置。") + } // convert abs path if !filepath.IsAbs(conf.Conf.TempDir) { absPath, err := filepath.Abs(conf.Conf.TempDir) diff --git a/internal/bootstrap/data/data.go b/internal/bootstrap/data/data.go index c2170d2f479..1f0a5909a58 100644 --- a/internal/bootstrap/data/data.go +++ b/internal/bootstrap/data/data.go @@ -3,6 +3,7 @@ package data import "github.com/alist-org/alist/v3/cmd/flags" func InitData() { + initRoles() initUser() initSettings() initTasks() diff --git a/internal/bootstrap/data/dev.go b/internal/bootstrap/data/dev.go index f6296c9e96a..74097dbd8b7 100644 --- a/internal/bootstrap/data/dev.go +++ b/internal/bootstrap/data/dev.go @@ -26,7 +26,7 @@ func initDevData() { Username: "Noah", Password: "hsu", BasePath: "/data", - Role: 0, + Role: nil, Permission: 512, }) if err != nil { diff --git a/internal/bootstrap/data/role.go b/internal/bootstrap/data/role.go new file mode 100644 index 00000000000..a82fa2afcc3 --- /dev/null +++ b/internal/bootstrap/data/role.go @@ -0,0 +1,52 @@ +package data + +// initRoles creates the default admin and guest roles if missing. +// These roles are essential and must not be modified or removed. + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +func initRoles() { + guestRole, err := op.GetRoleByName("guest") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + guestRole = &model.Role{ + ID: uint(model.GUEST), + Name: "guest", + Description: "Guest", + PermissionScopes: []model.PermissionEntry{ + {Path: "/", Permission: 0}, + }, + } + if err := op.CreateRole(guestRole); err != nil { + utils.Log.Fatalf("[init role] Failed to create guest role: %v", err) + } + } else { + utils.Log.Fatalf("[init role] Failed to get guest role: %v", err) + } + } + + _, err = op.GetRoleByName("admin") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + adminRole := &model.Role{ + ID: uint(model.ADMIN), + Name: "admin", + Description: "Administrator", + PermissionScopes: []model.PermissionEntry{ + {Path: "/", Permission: 0xFFFF}, + }, + } + if err := op.CreateRole(adminRole); err != nil { + utils.Log.Fatalf("[init role] Failed to create admin role: %v", err) + } + } else { + utils.Log.Fatalf("[init role] Failed to get admin role: %v", err) + } + } +} diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 407a5c64e17..ad3d181405e 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -91,6 +91,7 @@ func InitialSettings() []model.SettingItem { } else { token = random.Token() } + defaultRoleID := strconv.Itoa(model.GUEST) initialSettingItems = []model.SettingItem{ // site settings {Key: conf.VERSION, Value: conf.Version, Type: conf.TypeString, Group: model.SITE, Flag: model.READONLY}, @@ -99,10 +100,15 @@ func InitialSettings() []model.SettingItem { {Key: conf.SiteTitle, Value: "AList", Type: conf.TypeString, Group: model.SITE}, {Key: conf.Announcement, Value: "### repo\nhttps://github.com/alist-org/alist", Type: conf.TypeText, Group: model.SITE}, {Key: "pagination_type", Value: "all", Type: conf.TypeSelect, Options: "all,pagination,load_more,auto_load_more", Group: model.SITE}, - {Key: "default_page_size", Value: "30", Type: conf.TypeNumber, Group: model.SITE}, + {Key: "default_page_size", Value: "50", Type: conf.TypeNumber, Group: model.SITE}, {Key: conf.AllowIndexed, Value: "false", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, + {Key: conf.AllowRegister, Value: "false", Type: conf.TypeBool, Group: model.SITE}, + {Key: conf.DefaultRole, Value: defaultRoleID, Type: conf.TypeSelect, Group: model.SITE}, + // newui settings + {Key: conf.UseNewui, Value: "false", Type: conf.TypeBool, Group: model.SITE}, + {Key: conf.FrontendRememberSort, Value: "false", Type: conf.TypeBool, Group: model.SITE, Help: "Persist frontend list sorting in the browser. When disabled, backend/driver order is used until the user sorts manually in the current session."}, // style settings {Key: conf.Logo, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeText, Group: model.STYLE}, {Key: conf.Favicon, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.STYLE}, @@ -141,6 +147,7 @@ func InitialSettings() []model.SettingItem { {Key: "audio_cover", Value: "https://jsd.nn.ci/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW}, {Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.ThumbnailSize, Value: "144", Type: conf.TypeNumber, Group: model.PREVIEW, Help: "Thumbnail width in pixels. Height is scaled proportionally."}, {Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, @@ -155,11 +162,15 @@ func InitialSettings() []model.SettingItem { ([[:xdigit:]]{1,4}(?::[[:xdigit:]]{1,4}){7}|::|:(?::[[:xdigit:]]{1,4}){1,6}|[[:xdigit:]]{1,4}:(?::[[:xdigit:]]{1,4}){1,5}|(?:[[:xdigit:]]{1,4}:){2}(?::[[:xdigit:]]{1,4}){1,4}|(?:[[:xdigit:]]{1,4}:){3}(?::[[:xdigit:]]{1,4}){1,3}|(?:[[:xdigit:]]{1,4}:){4}(?::[[:xdigit:]]{1,4}){1,2}|(?:[[:xdigit:]]{1,4}:){5}:[[:xdigit:]]{1,4}|(?:[[:xdigit:]]{1,4}:){1,6}:) (?U)access_token=(.*)&`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, - {Key: conf.OcrApi, Value: "https://api.nn.ci/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, + {Key: conf.OcrApi, Value: "https://api.alistgo.com/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL}, {Key: conf.ForwardDirectLinkParams, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL}, {Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, + {Key: conf.MaxDevices, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL}, + {Key: conf.DeviceEvictPolicy, Value: "deny", Type: conf.TypeSelect, Options: "deny,evict_oldest", Group: model.GLOBAL}, + {Key: conf.DeviceSessionTTL, Value: "86400", Type: conf.TypeNumber, Group: model.GLOBAL}, + {Key: conf.MetaNotFoundCacheExpire, Value: "60", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE, Help: "Negative cache expiration for missing meta records, in seconds. Set 0 to disable."}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, @@ -211,6 +222,21 @@ func InitialSettings() []model.SettingItem { {Key: conf.FTPTLSPrivateKeyPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPTLSPublicCertPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + // frp settings + {Key: conf.FRPEnabled, Value: "false", Type: conf.TypeBool, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPServerAddr, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPServerPort, Value: "7000", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPAuthToken, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPProxyName, Value: "alist", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPProxyType, Value: "http", Type: conf.TypeSelect, Options: "http,https,tcp,stcp", Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPCustomDomain, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPSubdomain, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPRemotePort, Value: "0", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPLocalPort, Value: "5244", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPTLSEnable, Value: "false", Type: conf.TypeBool, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPSTCPSecretKey, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE, Help: "Required for stcp proxy type"}, + {Key: conf.FRPStatus, Value: "stopped", Type: conf.TypeString, Group: model.FRP, Flag: model.READONLY}, + // traffic settings {Key: conf.TaskOfflineDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Download.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskOfflineDownloadTransferThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Transfer.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index 9c3f8962ad3..6851118668b 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -1,10 +1,10 @@ package data import ( + "github.com/alist-org/alist/v3/internal/db" "os" "github.com/alist-org/alist/v3/cmd/flags" - "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" @@ -14,6 +14,28 @@ import ( ) func initUser() { + guest, err := op.GetGuest() + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + salt := random.String(16) + guestRole, _ := op.GetRoleByName("guest") + guest = &model.User{ + Username: "guest", + PwdHash: model.TwoHashPwd("guest", salt), + Salt: salt, + Role: model.Roles{int(guestRole.ID)}, + BasePath: "/", + Permission: 0, + Disabled: true, + Authn: "[]", + } + if err := db.CreateUser(guest); err != nil { + utils.Log.Fatalf("[init user] Failed to create guest user: %v", err) + } + } else { + utils.Log.Fatalf("[init user] Failed to get guest user: %v", err) + } + } admin, err := op.GetAdmin() adminPassword := random.String(8) envpass := os.Getenv("ALIST_ADMIN_PASSWORD") @@ -25,15 +47,16 @@ func initUser() { if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { salt := random.String(16) + adminRole, _ := op.GetRoleByName("admin") admin = &model.User{ Username: "admin", Salt: salt, PwdHash: model.TwoHashPwd(adminPassword, salt), - Role: model.ADMIN, + Role: model.Roles{int(adminRole.ID)}, BasePath: "/", Authn: "[]", // 0(can see hidden) - 7(can remove) & 12(can read archives) - 13(can decompress archives) - Permission: 0x30FF, + Permission: 0xFFFF, } if err := op.CreateUser(admin); err != nil { panic(err) @@ -44,25 +67,4 @@ func initUser() { utils.Log.Fatalf("[init user] Failed to get admin user: %v", err) } } - guest, err := op.GetGuest() - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - salt := random.String(16) - guest = &model.User{ - Username: "guest", - PwdHash: model.TwoHashPwd("guest", salt), - Salt: salt, - Role: model.GUEST, - BasePath: "/", - Permission: 0, - Disabled: true, - Authn: "[]", - } - if err := db.CreateUser(guest); err != nil { - utils.Log.Fatalf("[init user] Failed to create guest user: %v", err) - } - } else { - utils.Log.Fatalf("[init user] Failed to get guest user: %v", err) - } - } } diff --git a/internal/bootstrap/frp.go b/internal/bootstrap/frp.go new file mode 100644 index 00000000000..b8417843c30 --- /dev/null +++ b/internal/bootstrap/frp.go @@ -0,0 +1,19 @@ +package bootstrap + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func InitFRP() { + frp.Instance = frp.Init() + if setting.GetBool(conf.FRPEnabled) { + if err := frp.Instance.Start(); err != nil { + utils.Log.Warnf("failed to start frp client: %v", err) + } else { + utils.Log.Info("frp client started") + } + } +} diff --git a/internal/bootstrap/log.go b/internal/bootstrap/log.go index 00411e5e189..b4f4af08f1b 100644 --- a/internal/bootstrap/log.go +++ b/internal/bootstrap/log.go @@ -14,10 +14,14 @@ import ( func init() { formatter := logrus.TextFormatter{ - ForceColors: true, - EnvironmentOverrideColors: true, - TimestampFormat: "2006-01-02 15:04:05", - FullTimestamp: true, + TimestampFormat: "2006-01-02 15:04:05", + FullTimestamp: true, + } + if os.Getenv("NO_COLOR") != "" || os.Getenv("ALIST_NO_COLOR") == "1" { + formatter.DisableColors = true + } else { + formatter.ForceColors = true + formatter.EnvironmentOverrideColors = true } logrus.SetFormatter(&formatter) utils.Log.SetFormatter(&formatter) diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go index b363d12981d..eb679147ec9 100644 --- a/internal/bootstrap/patch/all.go +++ b/internal/bootstrap/patch/all.go @@ -4,6 +4,7 @@ import ( "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_24_0" "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_32_0" "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_41_0" + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_46_0" ) type VersionPatches struct { @@ -32,4 +33,10 @@ var UpgradePatches = []VersionPatches{ v3_41_0.GrantAdminPermissions, }, }, + { + Version: "v3.46.0", + Patches: []func(){ + v3_46_0.ConvertLegacyRoles, + }, + }, } diff --git a/internal/bootstrap/patch/v3_46_0/convert_role.go b/internal/bootstrap/patch/v3_46_0/convert_role.go new file mode 100644 index 00000000000..3aac95b691c --- /dev/null +++ b/internal/bootstrap/patch/v3_46_0/convert_role.go @@ -0,0 +1,186 @@ +package v3_46_0 + +import ( + "database/sql" + "encoding/json" + "errors" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "gorm.io/gorm" +) + +// ConvertLegacyRoles migrates old integer role values to a new role model with permission scopes. +func ConvertLegacyRoles() { + guestRole, err := op.GetRoleByName("guest") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + guestRole = &model.Role{ + ID: uint(model.GUEST), + Name: "guest", + Description: "Guest", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0, + }, + }, + } + if err = op.CreateRole(guestRole); err != nil { + utils.Log.Errorf("[convert roles] failed to create guest role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed to get guest role: %v", err) + return + } + } + + adminRole, err := op.GetRoleByName("admin") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + adminRole = &model.Role{ + ID: uint(model.ADMIN), + Name: "admin", + Description: "Administrator", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0x33FF, + }, + }, + } + if err = op.CreateRole(adminRole); err != nil { + utils.Log.Errorf("[convert roles] failed to create admin role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed to get admin role: %v", err) + return + } + } + + generalRole, err := op.GetRoleByName("general") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + generalRole = &model.Role{ + ID: uint(model.NEWGENERAL), + Name: "general", + Description: "General User", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0, + }, + }, + } + if err = op.CreateRole(generalRole); err != nil { + utils.Log.Errorf("[convert roles] failed create general role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed get general role: %v", err) + return + } + } + + rawDb := db.GetDb() + table := conf.Conf.Database.TablePrefix + "users" + rows, err := rawDb.Table(table).Select("id, username, role").Rows() + if err != nil { + utils.Log.Errorf("[convert roles] failed to get users: %v", err) + return + } + defer rows.Close() + + var updatedCount int + for rows.Next() { + var id uint + var username string + var rawRole []byte + + if err := rows.Scan(&id, &username, &rawRole); err != nil { + utils.Log.Warnf("[convert roles] skip user scan err: %v", err) + continue + } + + utils.Log.Debugf("[convert roles] user: %s raw role: %s", username, string(rawRole)) + + if len(rawRole) == 0 { + continue + } + + var oldRoles []int + wasSingleInt := false + if err := json.Unmarshal(rawRole, &oldRoles); err != nil { + var single int + if err := json.Unmarshal(rawRole, &single); err != nil { + utils.Log.Warnf("[convert roles] user %s has invalid role: %s", username, string(rawRole)) + continue + } + oldRoles = []int{single} + wasSingleInt = true + } + + var newRoles model.Roles + for _, r := range oldRoles { + switch r { + case model.ADMIN: + newRoles = append(newRoles, int(adminRole.ID)) + case model.GUEST: + newRoles = append(newRoles, int(guestRole.ID)) + case model.GENERAL: + newRoles = append(newRoles, int(generalRole.ID)) + default: + newRoles = append(newRoles, r) + } + } + + if wasSingleInt { + err := rawDb.Table(table).Where("id = ?", id).Update("role", newRoles).Error + if err != nil { + utils.Log.Errorf("[convert roles] failed to update user %s: %v", username, err) + } else { + updatedCount++ + utils.Log.Infof("[convert roles] updated user %s: %v → %v", username, oldRoles, newRoles) + } + } + } + + utils.Log.Infof("[convert roles] completed role conversion for %d users", updatedCount) +} + +func IsLegacyRoleDetected() bool { + rawDb := db.GetDb() + table := conf.Conf.Database.TablePrefix + "users" + rows, err := rawDb.Table(table).Select("role").Rows() + if err != nil { + utils.Log.Errorf("[role check] failed to scan user roles: %v", err) + return false + } + defer rows.Close() + + for rows.Next() { + var raw sql.RawBytes + if err := rows.Scan(&raw); err != nil { + continue + } + if len(raw) == 0 { + continue + } + + var roles []int + if err := json.Unmarshal(raw, &roles); err == nil { + continue + } + + var single int + if err := json.Unmarshal(raw, &single); err == nil { + utils.Log.Infof("[role check] detected legacy int role: %d", single) + return true + } + } + return false +} diff --git a/internal/bootstrap/task.go b/internal/bootstrap/task.go index c67e3029b61..8f05b84a347 100644 --- a/internal/bootstrap/task.go +++ b/internal/bootstrap/task.go @@ -37,6 +37,18 @@ func InitTaskManager() { if len(tool.TransferTaskManager.GetAll()) == 0 { //prevent offline downloaded files from being deleted CleanTempDir() } + workers := conf.Conf.Tasks.S3Transition.Workers + if workers < 0 { + workers = 0 + } + fs.S3TransitionTaskManager = tache.NewManager[*fs.S3TransitionTask]( + tache.WithWorks(workers), + tache.WithPersistFunction( + db.GetTaskDataFunc("s3_transition", conf.Conf.Tasks.S3Transition.TaskPersistant), + db.UpdateTaskDataFunc("s3_transition", conf.Conf.Tasks.S3Transition.TaskPersistant), + ), + tache.WithMaxRetry(conf.Conf.Tasks.S3Transition.MaxRetry), + ) fs.ArchiveDownloadTaskManager = tache.NewManager[*fs.ArchiveDownloadTask](tache.WithWorks(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant), db.UpdateTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Decompress.MaxRetry)) op.RegisterSettingChangingCallback(func() { fs.ArchiveDownloadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers))) diff --git a/internal/conf/config.go b/internal/conf/config.go index cdb86fee3ad..383a5a4e52f 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -60,6 +60,7 @@ type TasksConfig struct { Copy TaskConfig `json:"copy" envPrefix:"COPY_"` Decompress TaskConfig `json:"decompress" envPrefix:"DECOMPRESS_"` DecompressUpload TaskConfig `json:"decompress_upload" envPrefix:"DECOMPRESS_UPLOAD_"` + S3Transition TaskConfig `json:"s3_transition" envPrefix:"S3_TRANSITION_"` AllowRetryCanceled bool `json:"allow_retry_canceled" env:"ALLOW_RETRY_CANCELED"` } @@ -93,6 +94,11 @@ type SFTP struct { Listen string `json:"listen" env:"LISTEN"` } +type MCP struct { + Enable bool `json:"enable" env:"ENABLE"` + Port int `json:"port" env:"PORT"` +} + type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` @@ -115,6 +121,7 @@ type Config struct { S3 S3 `json:"s3" envPrefix:"S3_"` FTP FTP `json:"ftp" envPrefix:"FTP_"` SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` + MCP MCP `json:"mcp" envPrefix:"MCP_"` LastLaunchedVersion string `json:"last_launched_version"` } @@ -155,7 +162,7 @@ func DefaultConfig() *Config { }, MaxConnections: 0, MaxConcurrency: 64, - TlsInsecureSkipVerify: true, + TlsInsecureSkipVerify: false, Tasks: TasksConfig{ Download: TaskConfig{ Workers: 5, @@ -184,6 +191,11 @@ func DefaultConfig() *Config { Workers: 5, MaxRetry: 2, }, + S3Transition: TaskConfig{ + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, + }, AllowRetryCanceled: false, }, Cors: Cors{ @@ -212,6 +224,10 @@ func DefaultConfig() *Config { Enable: false, Listen: ":5222", }, + MCP: MCP{ + Enable: false, + Port: 5248, + }, LastLaunchedVersion: "", } } diff --git a/internal/conf/const.go b/internal/conf/const.go index 5cb8d850bf0..79480a4589e 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -10,12 +10,16 @@ const ( const ( // site - VERSION = "version" - SiteTitle = "site_title" - Announcement = "announcement" - AllowIndexed = "allow_indexed" - AllowMounted = "allow_mounted" - RobotsTxt = "robots_txt" + VERSION = "version" + SiteTitle = "site_title" + Announcement = "announcement" + AllowIndexed = "allow_indexed" + AllowMounted = "allow_mounted" + RobotsTxt = "robots_txt" + AllowRegister = "allow_register" + DefaultRole = "default_role" + UseNewui = "use_newui" + FrontendRememberSort = "frontend_remember_sort" Logo = "logo" Favicon = "favicon" @@ -30,6 +34,7 @@ const ( ProxyIgnoreHeaders = "proxy_ignore_headers" AudioAutoplay = "audio_autoplay" VideoAutoplay = "video_autoplay" + ThumbnailSize = "thumbnail_size" PreviewArchivesByDefault = "preview_archives_by_default" ReadMeAutoRender = "readme_autorender" FilterReadMeScripts = "filter_readme_scripts" @@ -45,6 +50,10 @@ const ( ForwardDirectLinkParams = "forward_direct_link_params" IgnoreDirectLinkParams = "ignore_direct_link_params" WebauthnLoginEnabled = "webauthn_login_enabled" + MaxDevices = "max_devices" + DeviceEvictPolicy = "device_evict_policy" + DeviceSessionTTL = "device_session_ttl" + MetaNotFoundCacheExpire = "meta_not_found_cache_expire" // index SearchIndex = "search_index" @@ -69,6 +78,9 @@ const ( // thunder ThunderTempDir = "thunder_temp_dir" + // guangyapan + GuangYaPanTempDir = "guangyapan_temp_dir" + // single Token = "token" IndexProgress = "index_progress" @@ -118,6 +130,21 @@ const ( FTPTLSPrivateKeyPath = "ftp_tls_private_key_path" FTPTLSPublicCertPath = "ftp_tls_public_cert_path" + // frp + FRPEnabled = "frp_enabled" + FRPServerAddr = "frp_server_addr" + FRPServerPort = "frp_server_port" + FRPAuthToken = "frp_auth_token" + FRPProxyName = "frp_proxy_name" + FRPProxyType = "frp_proxy_type" + FRPCustomDomain = "frp_custom_domain" + FRPSubdomain = "frp_subdomain" + FRPRemotePort = "frp_remote_port" + FRPLocalPort = "frp_local_port" + FRPTLSEnable = "frp_tls_enable" + FRPSTCPSecretKey = "frp_stcp_secret_key" + FRPStatus = "frp_status" + // traffic TaskOfflineDownloadThreadsNum = "offline_download_task_threads_num" TaskOfflineDownloadTransferThreadsNum = "offline_download_transfer_task_threads_num" diff --git a/internal/db/db.go b/internal/db/db.go index 2cd18050da9..f33927be0ff 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile), new(model.Session), new(model.Share)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/label.go b/internal/db/label.go new file mode 100644 index 00000000000..fd9842d680c --- /dev/null +++ b/internal/db/label.go @@ -0,0 +1,79 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" + "time" +) + +// GetLabels Get all label from database order by id +func GetLabels(pageIndex, pageSize int) ([]model.Label, int64, error) { + labelDB := db.Model(&model.Label{}) + var count int64 + if err := labelDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get label count") + } + var labels []model.Label + if err := labelDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&labels).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + return labels, count, nil +} + +// GetLabelById Get Label by id, used to update label usually +func GetLabelById(id uint) (*model.Label, error) { + var label model.Label + label.ID = id + if err := db.First(&label).Error; err != nil { + return nil, errors.WithStack(err) + } + return &label, nil +} + +// CreateLabel just insert label to database +func CreateLabel(label model.Label) (uint, error) { + label.CreateTime = time.Now() + err := errors.WithStack(db.Create(&label).Error) + if err != nil { + return label.ID, errors.WithMessage(err, "failed create label in database") + } + return label.ID, nil +} + +// UpdateLabel just update storage in database +func UpdateLabel(label *model.Label) (*model.Label, error) { + label.CreateTime = time.Now() + _, err := GetLabelById(label.ID) + if err != nil { + return nil, errors.WithMessage(err, "failed get old label") + } + err = errors.WithStack(db.Save(label).Error) + if err != nil { + return nil, errors.WithMessage(err, "failed create label in database") + } + return label, nil +} + +// DeleteLabelById just delete label from database by id +func DeleteLabelById(id uint) error { + return errors.WithStack(db.Delete(&model.Label{}, id).Error) +} + +// GetLabelByIds Get label from database order by ids +func GetLabelByIds(ids []uint) ([]model.Label, error) { + labelDB := db.Model(&model.Label{}) + var labels []model.Label + if err := labelDB.Where(ids).Find(&labels).Error; err != nil { + return nil, errors.WithStack(err) + } + return labels, nil +} + +// GetLabelByName Get Label by name +func GetLabelByName(name string) bool { + var label model.Label + result := db.Where("name = ?", name).First(&label) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} diff --git a/internal/db/label_file_binding.go b/internal/db/label_file_binding.go new file mode 100644 index 00000000000..4dda80f2c42 --- /dev/null +++ b/internal/db/label_file_binding.go @@ -0,0 +1,192 @@ +package db + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "time" +) + +// GetLabelIds Get all label_ids from database order by file_name +func GetLabelIds(userId uint, fileName string) ([]uint, error) { + //fmt.Printf(">>> [GetLabelIds] userId: %d, fileName: %s\n", userId, fileName) + labelFileBinDingDB := db.Model(&model.LabelFileBinding{}) + var labelIds []uint + if err := labelFileBinDingDB.Where("file_name = ?", fileName).Where("user_id = ?", userId).Pluck("label_id", &labelIds).Error; err != nil { + return nil, errors.WithStack(err) + } + return labelIds, nil +} + +func CreateLabelFileBinDing(fileName string, labelId, userId uint) error { + var labelFileBinDing model.LabelFileBinding + labelFileBinDing.UserId = userId + labelFileBinDing.LabelId = labelId + labelFileBinDing.FileName = fileName + labelFileBinDing.CreateTime = time.Now() + err := errors.WithStack(db.Create(&labelFileBinDing).Error) + if err != nil { + return errors.WithMessage(err, "failed create label in database") + } + return nil +} + +// GetLabelFileBinDingByLabelIdExists Get Label by label_id, used to del label usually +func GetLabelFileBinDingByLabelIdExists(labelId, userId uint) bool { + var labelFileBinDing model.LabelFileBinding + result := db.Where("label_id = ?", labelId).Where("user_id = ?", userId).First(&labelFileBinDing) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} + +// DelLabelFileBinDingByFileName used to del usually +func DelLabelFileBinDingByFileName(userId uint, fileName string) error { + return errors.WithStack(db.Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinding{}).Error) +} + +// DelLabelFileBinDingById used to del usually +func DelLabelFileBinDingById(labelId, userId uint, fileName string) error { + return errors.WithStack(db.Where("label_id = ?", labelId).Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinding{}).Error) +} + +func GetLabelFileBinDingByLabelId(labelIds []uint, userId uint) (result []model.LabelFileBinding, err error) { + if err := db.Where("label_id in (?)", labelIds).Where("user_id = ?", userId).Find(&result).Error; err != nil { + return nil, errors.WithStack(err) + } + return result, nil +} + +func GetLabelBindingsByFileNamesPublic(fileNames []string) (map[string][]uint, error) { + var binds []model.LabelFileBinding + if err := db.Where("file_name IN ?", fileNames).Find(&binds).Error; err != nil { + return nil, errors.WithStack(err) + } + out := make(map[string][]uint, len(fileNames)) + seen := make(map[string]struct{}, len(binds)) + for _, b := range binds { + key := fmt.Sprintf("%s-%d", b.FileName, b.LabelId) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out[b.FileName] = append(out[b.FileName], b.LabelId) + } + return out, nil +} + +func GetLabelsByFileNamesPublic(fileNames []string) (map[string][]model.Label, error) { + bindMap, err := GetLabelBindingsByFileNamesPublic(fileNames) + if err != nil { + return nil, err + } + + idSet := make(map[uint]struct{}) + for _, ids := range bindMap { + for _, id := range ids { + idSet[id] = struct{}{} + } + } + if len(idSet) == 0 { + return make(map[string][]model.Label, 0), nil + } + allIDs := make([]uint, 0, len(idSet)) + for id := range idSet { + allIDs = append(allIDs, id) + } + labels, err := GetLabelByIds(allIDs) // 你已有的函数 + if err != nil { + return nil, err + } + + labelByID := make(map[uint]model.Label, len(labels)) + for _, l := range labels { + labelByID[l.ID] = l + } + + out := make(map[string][]model.Label, len(bindMap)) + for fname, ids := range bindMap { + for _, id := range ids { + if lab, ok := labelByID[id]; ok { + out[fname] = append(out[fname], lab) + } + } + } + return out, nil +} + +func ListLabelFileBinDing(userId uint, labelIDs []uint, fileName string, page, pageSize int) ([]model.LabelFileBinding, int64, error) { + q := db.Model(&model.LabelFileBinding{}).Where("user_id = ?", userId) + + if len(labelIDs) > 0 { + q = q.Where("label_id IN ?", labelIDs) + } + if fileName != "" { + q = q.Where("file_name LIKE ?", "%"+fileName+"%") + } + + var total int64 + if err := q.Count(&total).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + + var rows []model.LabelFileBinding + if err := q. + Order("id DESC"). + Offset((page - 1) * pageSize). + Limit(pageSize). + Find(&rows).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + return rows, total, nil +} + +func RestoreLabelFileBindings(bindings []model.LabelFileBinding, keepIDs bool, override bool) error { + if len(bindings) == 0 { + return nil + } + tx := db.Begin() + + if override { + type key struct { + uid uint + name string + } + toDel := make(map[key]struct{}, len(bindings)) + for i := range bindings { + k := key{uid: bindings[i].UserId, name: bindings[i].FileName} + toDel[k] = struct{}{} + } + for k := range toDel { + if err := tx.Where("user_id = ? AND file_name = ?", k.uid, k.name). + Delete(&model.LabelFileBinding{}).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } + } + + for i := range bindings { + b := bindings[i] + if !keepIDs { + b.ID = 0 + } + if b.CreateTime.IsZero() { + b.CreateTime = time.Now() + } + if override { + if err := tx.Create(&b).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } else { + if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&b).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } + } + + return errors.WithStack(tx.Commit().Error) +} diff --git a/internal/db/obj_file.go b/internal/db/obj_file.go new file mode 100644 index 00000000000..2bbce9e6dd6 --- /dev/null +++ b/internal/db/obj_file.go @@ -0,0 +1,31 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// GetFileByNameExists Get file by name +func GetFileByNameExists(name string) bool { + var label model.ObjFile + result := db.Where("name = ?", name).First(&label) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} + +// GetFileByName Get file by name +func GetFileByName(name string, userId uint) (objFile model.ObjFile, err error) { + if err = db.Where("name = ?", name).Where("user_id = ?", userId).First(&objFile).Error; err != nil { + return objFile, errors.WithStack(err) + } + return objFile, nil +} + +func CreateObjFile(obj model.ObjFile) error { + err := errors.WithStack(db.Create(&obj).Error) + if err != nil { + return errors.WithMessage(err, "failed create file in database") + } + return nil +} diff --git a/internal/db/role.go b/internal/db/role.go new file mode 100644 index 00000000000..d0b776b391d --- /dev/null +++ b/internal/db/role.go @@ -0,0 +1,106 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "path" + "strings" +) + +func GetRole(id uint) (*model.Role, error) { + var r model.Role + if err := db.First(&r, id).Error; err != nil { + return nil, errors.Wrapf(err, "failed get role") + } + return &r, nil +} + +func GetRoleByName(name string) (*model.Role, error) { + r := model.Role{Name: name} + if err := db.Where(r).First(&r).Error; err != nil { + return nil, errors.Wrapf(err, "failed get role") + } + return &r, nil +} + +func GetRoles(pageIndex, pageSize int) (roles []model.Role, count int64, err error) { + roleDB := db.Model(&model.Role{}) + if err = roleDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get roles count") + } + if err = roleDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&roles).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get find roles") + } + return roles, count, nil +} + +func GetAllRoles() ([]model.Role, error) { + var roles []model.Role + if err := db.Find(&roles).Error; err != nil { + return nil, errors.WithStack(err) + } + return roles, nil +} + +func CreateRole(r *model.Role) error { + if err := db.Create(r).Error; err != nil { + return errors.WithStack(err) + } + if r.Default { + if err := db.Model(&model.Role{}).Where("id <> ?", r.ID).Update("default", false).Error; err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func UpdateRole(r *model.Role) error { + if err := db.Save(r).Error; err != nil { + return errors.WithStack(err) + } + if r.Default { + if err := db.Model(&model.Role{}).Where("id <> ?", r.ID).Update("default", false).Error; err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func DeleteRole(id uint) error { + return errors.WithStack(db.Delete(&model.Role{}, id).Error) +} + +func UpdateRolePermissionsPathPrefix(oldPath, newPath string) ([]uint, error) { + var roles []model.Role + var modifiedRoleIDs []uint + + if err := db.Find(&roles).Error; err != nil { + return nil, errors.WithMessage(err, "failed to load roles") + } + + for _, role := range roles { + if role.Name == "admin" || role.Name == "guest" { + continue + } + updated := false + for i, entry := range role.PermissionScopes { + entryPath := path.Clean(entry.Path) + oldPathClean := path.Clean(oldPath) + + if entryPath == oldPathClean { + role.PermissionScopes[i].Path = newPath + updated = true + } else if strings.HasPrefix(entryPath, oldPathClean+"/") { + role.PermissionScopes[i].Path = newPath + entryPath[len(oldPathClean):] + updated = true + } + } + if updated { + if err := UpdateRole(&role); err != nil { + return nil, errors.WithMessagef(err, "failed to update role ID %d", role.ID) + } + modifiedRoleIDs = append(modifiedRoleIDs, role.ID) + } + } + return modifiedRoleIDs, nil +} diff --git a/internal/db/session.go b/internal/db/session.go new file mode 100644 index 00000000000..35c778c3ac8 --- /dev/null +++ b/internal/db/session.go @@ -0,0 +1,69 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm/clause" +) + +func GetSession(userID uint, deviceKey string) (*model.Session, error) { + s := model.Session{UserID: userID, DeviceKey: deviceKey} + if err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where(&s).First(&s).Error; err != nil { + return nil, errors.Wrap(err, "failed find session") + } + return &s, nil +} + +func CreateSession(s *model.Session) error { + return errors.WithStack(db.Create(s).Error) +} + +func UpsertSession(s *model.Session) error { + return errors.WithStack(db.Clauses(clause.OnConflict{UpdateAll: true}).Create(s).Error) +} + +func DeleteSession(userID uint, deviceKey string) error { + return errors.WithStack(db.Where("user_id = ? AND device_key = ?", userID, deviceKey).Delete(&model.Session{}).Error) +} + +func CountActiveSessionsByUser(userID uint) (int64, error) { + var count int64 + err := db.Model(&model.Session{}). + Where("user_id = ? AND status = ?", userID, model.SessionActive). + Count(&count).Error + return count, errors.WithStack(err) +} + +func DeleteSessionsBefore(ts int64) error { + return errors.WithStack(db.Where("last_active < ?", ts).Delete(&model.Session{}).Error) +} + +// GetOldestActiveSession returns the oldest active session for the specified user. +func GetOldestActiveSession(userID uint) (*model.Session, error) { + var s model.Session + if err := db.Where("user_id = ? AND status = ?", userID, model.SessionActive). + Order("last_active ASC").First(&s).Error; err != nil { + return nil, errors.Wrap(err, "failed get oldest active session") + } + return &s, nil +} + +func UpdateSessionLastActive(userID uint, deviceKey string, lastActive int64) error { + return errors.WithStack(db.Model(&model.Session{}).Where("user_id = ? AND device_key = ?", userID, deviceKey).Update("last_active", lastActive).Error) +} + +func ListSessionsByUser(userID uint) ([]model.Session, error) { + var sessions []model.Session + err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where("user_id = ? AND status = ?", userID, model.SessionActive).Find(&sessions).Error + return sessions, errors.WithStack(err) +} + +func ListSessions() ([]model.Session, error) { + var sessions []model.Session + err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where("status = ?", model.SessionActive).Find(&sessions).Error + return sessions, errors.WithStack(err) +} + +func MarkInactive(sessionID string) error { + return errors.WithStack(db.Model(&model.Session{}).Where("device_key = ?", sessionID).Update("status", model.SessionInactive).Error) +} diff --git a/internal/db/share.go b/internal/db/share.go new file mode 100644 index 00000000000..711c3f278c3 --- /dev/null +++ b/internal/db/share.go @@ -0,0 +1,124 @@ +package db + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func GetShareByShareID(shareID string) (*model.Share, error) { + var share model.Share + if err := db.Where("share_id = ?", shareID).Take(&share).Error; err != nil { + return nil, err + } + return &share, nil +} + +func GetShareByCreatorAndShareID(creatorID uint, shareID string) (*model.Share, error) { + var share model.Share + if err := db.Where("creator_id = ? AND share_id = ?", creatorID, shareID).Take(&share).Error; err != nil { + return nil, err + } + return &share, nil +} + +func ShareIDExists(shareID string) (bool, error) { + var count int64 + if err := db.Model(&model.Share{}).Where("share_id = ?", shareID).Count(&count).Error; err != nil { + return false, err + } + return count > 0, nil +} + +func ShareIDExistsExceptID(shareID string, id uint) (bool, error) { + var count int64 + if err := db.Model(&model.Share{}).Where("share_id = ? AND id <> ?", shareID, id).Count(&count).Error; err != nil { + return false, err + } + return count > 0, nil +} + +func CreateShare(share *model.Share) error { + return db.Create(share).Error +} + +func UpdateShare(share *model.Share) error { + return db.Save(share).Error +} + +func GetSharesByCreator(creatorID uint, pageIndex, pageSize int) (shares []model.Share, count int64, err error) { + tx := db.Model(&model.Share{}).Where("creator_id = ?", creatorID) + err = tx.Count(&count).Error + if err != nil { + return nil, 0, err + } + err = tx.Order("created_at desc").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&shares).Error + return +} + +func DeleteShareByShareID(creatorID uint, shareID string) error { + return db.Where("creator_id = ? AND share_id = ?", creatorID, shareID).Delete(&model.Share{}).Error +} + +func DisableShareByShareID(creatorID uint, shareID string) error { + return db.Model(&model.Share{}). + Where("creator_id = ? AND share_id = ?", creatorID, shareID). + Update("enabled", false).Error +} + +func TouchShareView(shareID string) error { + now := time.Now() + return db.Model(&model.Share{}). + Where("share_id = ?", shareID). + UpdateColumns(map[string]interface{}{ + "last_access_at": now, + "view_count": gorm.Expr("view_count + ?", 1), + }).Error +} + +func TouchShareDownload(shareID string) error { + now := time.Now() + return db.Model(&model.Share{}). + Where("share_id = ?", shareID). + UpdateColumns(map[string]interface{}{ + "last_access_at": now, + "download_count": gorm.Expr("download_count + ?", 1), + }).Error +} + +func RecordShareAccess(shareID string) (*model.Share, error) { + var updated model.Share + err := db.Transaction(func(tx *gorm.DB) error { + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). + Where("share_id = ?", shareID). + Take(&updated).Error; err != nil { + return err + } + + now := time.Now() + updated.AccessCount++ + updated.LastAccessAt = &now + updates := map[string]interface{}{ + "access_count": updated.AccessCount, + "last_access_at": now, + } + + limit := updated.EffectiveAccessLimit() + if limit > 0 && updated.AccessCount >= limit { + updated.Enabled = false + updated.ConsumedAt = &now + updates["enabled"] = false + updates["consumed_at"] = now + } + + return tx.Model(&model.Share{}). + Where("id = ?", updated.ID). + Updates(updates).Error + }) + if err != nil { + return nil, err + } + return &updated, nil +} diff --git a/internal/db/user.go b/internal/db/user.go index 8c9641b2c55..4e5d67ad28e 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -2,19 +2,42 @@ package db import ( "encoding/base64" - + "fmt" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-webauthn/webauthn/webauthn" "github.com/pkg/errors" + "gorm.io/gorm" + "path" + "slices" + "strings" ) func GetUserByRole(role int) (*model.User, error) { - user := model.User{Role: role} - if err := db.Where(user).Take(&user).Error; err != nil { + var users []model.User + if err := db.Find(&users).Error; err != nil { return nil, err } - return &user, nil + for i := range users { + if users[i].Role.Contains(role) { + return &users[i], nil + } + } + return nil, gorm.ErrRecordNotFound +} + +func GetUsersByRole(roleID int) ([]model.User, error) { + var users []model.User + if err := db.Find(&users).Error; err != nil { + return nil, err + } + var result []model.User + for _, u := range users { + if slices.Contains(u.Role, roleID) { + result = append(result, u) + } + } + return result, nil } func GetUserByName(username string) (*model.User, error) { @@ -60,6 +83,14 @@ func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err err return users, count, nil } +func GetAllUsers() ([]model.User, error) { + var users []model.User + if err := db.Find(&users).Error; err != nil { + return nil, errors.WithStack(err) + } + return users, nil +} + func DeleteUserById(id uint) error { return errors.WithStack(db.Delete(&model.User{}, id).Error) } @@ -100,3 +131,50 @@ func RemoveAuthn(u *model.User, id string) error { } return UpdateAuthn(u.ID, string(res)) } + +func UpdateUserBasePathPrefix(oldPath, newPath string, usersOpt ...[]model.User) ([]string, error) { + var users []model.User + var modifiedUsernames []string + + oldPathClean := path.Clean(oldPath) + + if len(usersOpt) > 0 { + users = usersOpt[0] + } else { + if err := db.Find(&users).Error; err != nil { + return nil, errors.WithMessage(err, "failed to load users") + } + } + + for _, user := range users { + basePath := path.Clean(user.BasePath) + updated := false + + if basePath == oldPathClean { + user.BasePath = path.Clean(newPath) + updated = true + } else if strings.HasPrefix(basePath, oldPathClean+"/") { + user.BasePath = path.Clean(newPath + basePath[len(oldPathClean):]) + updated = true + } + + if updated { + if err := UpdateUser(&user); err != nil { + return nil, errors.WithMessagef(err, "failed to update user ID %d", user.ID) + } + modifiedUsernames = append(modifiedUsernames, user.Username) + } + } + + return modifiedUsernames, nil +} + +func CountUsersByRoleAndEnabledExclude(roleID uint, excludeUserID uint) (int64, error) { + var count int64 + jsonValue := fmt.Sprintf("[%d]", roleID) + err := db.Model(&model.User{}). + Where("disabled = ? AND id != ?", false, excludeUserID). + Where("JSON_CONTAINS(role, ?)", jsonValue). + Count(&count).Error + return count, err +} diff --git a/internal/device/session.go b/internal/device/session.go new file mode 100644 index 00000000000..1d9e7ea53cd --- /dev/null +++ b/internal/device/session.go @@ -0,0 +1,138 @@ +package device + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// Handle verifies device sessions for a user and upserts current session. +func Handle(userID uint, deviceKey, ua, ip string) error { + ttl := setting.GetInt(conf.DeviceSessionTTL, 86400) + if ttl > 0 { + _ = db.DeleteSessionsBefore(time.Now().Unix() - int64(ttl)) + } + + ip = utils.MaskIP(ip) + + now := time.Now().Unix() + sess, err := db.GetSession(userID, deviceKey) + if err == nil { + if sess.Status == model.SessionInactive { + return errors.WithStack(errs.SessionInactive) + } + sess.Status = model.SessionActive + sess.LastActive = now + sess.UserAgent = ua + sess.IP = ip + return db.UpsertSession(sess) + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, err := db.CountActiveSessionsByUser(userID) + if err != nil { + return err + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + if oldest, err := db.GetOldestActiveSession(userID); err == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + + s := &model.Session{UserID: userID, DeviceKey: deviceKey, UserAgent: ua, IP: ip, LastActive: now, Status: model.SessionActive} + return db.CreateSession(s) +} + +// EnsureActiveOnLogin is used only in login flow: +// - If session exists (even Inactive): reactivate and refresh fields. +// - If not exists: apply max-devices policy, then create Active session. +func EnsureActiveOnLogin(userID uint, deviceKey, ua, ip string) error { + ip = utils.MaskIP(ip) + now := time.Now().Unix() + + sess, err := db.GetSession(userID, deviceKey) + if err == nil { + if sess.Status == model.SessionInactive { + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, err := db.CountActiveSessionsByUser(userID) + if err != nil { + return err + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + if oldest, gerr := db.GetOldestActiveSession(userID); gerr == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + } + sess.Status = model.SessionActive + sess.LastActive = now + sess.UserAgent = ua + sess.IP = ip + return db.UpsertSession(sess) + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, err := db.CountActiveSessionsByUser(userID) + if err != nil { + return err + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + if oldest, gerr := db.GetOldestActiveSession(userID); gerr == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + + return db.CreateSession(&model.Session{ + UserID: userID, + DeviceKey: deviceKey, + UserAgent: ua, + IP: ip, + LastActive: now, + Status: model.SessionActive, + }) +} + +// Refresh updates last_active for the session. +func Refresh(userID uint, deviceKey string) { + _ = db.UpdateSessionLastActive(userID, deviceKey, time.Now().Unix()) +} diff --git a/internal/driver/proxy.go b/internal/driver/proxy.go new file mode 100644 index 00000000000..f94273c9f92 --- /dev/null +++ b/internal/driver/proxy.go @@ -0,0 +1,7 @@ +package driver + +// ProxyDriver lets a driver override the default "must proxy" download behavior +// on a per-storage basis. +type ProxyDriver interface { + ShouldProxyDownloads() bool +} diff --git a/internal/errs/device.go b/internal/errs/device.go new file mode 100644 index 00000000000..3b79298a672 --- /dev/null +++ b/internal/errs/device.go @@ -0,0 +1,8 @@ +package errs + +import "errors" + +var ( + TooManyDevices = errors.New("too many active devices") + SessionInactive = errors.New("session inactive") +) diff --git a/internal/errs/driver.go b/internal/errs/driver.go index 4b6b5cac48e..7f67c0e2c2d 100644 --- a/internal/errs/driver.go +++ b/internal/errs/driver.go @@ -4,4 +4,5 @@ import "errors" var ( EmptyToken = errors.New("empty token") + LinkIsDir = errors.New("link is dir") ) diff --git a/internal/errs/operate.go b/internal/errs/operate.go index 92fbd6a1a49..d2df47ddb9c 100644 --- a/internal/errs/operate.go +++ b/internal/errs/operate.go @@ -4,4 +4,5 @@ import "errors" var ( PermissionDenied = errors.New("permission denied") + InvalidName = errors.New("invalid file name") ) diff --git a/internal/errs/role.go b/internal/errs/role.go new file mode 100644 index 00000000000..a818ea21264 --- /dev/null +++ b/internal/errs/role.go @@ -0,0 +1,7 @@ +package errs + +import "errors" + +var ( + ErrChangeDefaultRole = errors.New("cannot modify admin role") +) diff --git a/internal/frp/frp.go b/internal/frp/frp.go new file mode 100644 index 00000000000..c244e18b870 --- /dev/null +++ b/internal/frp/frp.go @@ -0,0 +1,335 @@ +package frp + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/cmd/flags" + frpclient "github.com/fatedier/frp/client" + "github.com/fatedier/frp/pkg/config/source" + v1 "github.com/fatedier/frp/pkg/config/v1" + frplog "github.com/fatedier/frp/pkg/util/log" + log "github.com/sirupsen/logrus" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" +) + +// Instance is the global FRP manager. +var Instance *Manager + +// Manager controls the lifecycle of the embedded FRP client. +type Manager struct { + mu sync.Mutex + cancel context.CancelFunc + wg sync.WaitGroup + status string + logs []string + logPath string +} + +const maxLogEntries = 300 +const maxTailBytes = 512 * 1024 + +// RuntimeInfo contains FRP runtime status and recent logs. +type RuntimeInfo struct { + Status string `json:"status"` + Logs []string `json:"logs"` +} + +// Init creates and returns a new Manager. +func Init() *Manager { + m := &Manager{ + status: "stopped", + logPath: filepath.Join(flags.DataDir, "log", "frp.log"), + } + m.logs = append(m.logs, fmt.Sprintf("[%s] initialized", time.Now().Format(time.RFC3339))) + return m +} + +// Start builds the FRP config from settings and starts the client. +func (m *Manager) Start() error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.cancel != nil { + m.appendLogLocked("start skipped: already running") + return nil // already running + } + + cfg, proxyCfgs, err := buildConfig() + if err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("start failed: %s", err.Error()) + return err + } + cfg.Log.To = m.logPath + cfg.Log.Level = "info" + cfg.Log.MaxDays = 7 + if err := os.MkdirAll(filepath.Dir(m.logPath), 0o755); err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("init log dir failed: %s", err.Error()) + return err + } + frplog.InitLogger(cfg.Log.To, cfg.Log.Level, int(cfg.Log.MaxDays), true) + + configSource := source.NewConfigSource() + if err := configSource.ReplaceAll(proxyCfgs, nil); err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("replace config failed: %s", err.Error()) + return err + } + aggregator := source.NewAggregator(configSource) + + svr, err := frpclient.NewService(frpclient.ServiceOptions{ + Common: cfg, + ConfigSourceAggregator: aggregator, + }) + if err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("create service failed: %s", err.Error()) + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + m.cancel = cancel + m.status = "running" + m.appendLogLocked("service started") + m.wg.Add(1) + + go func() { + defer m.wg.Done() + if err := svr.Run(ctx); err != nil && ctx.Err() == nil { + // Context was not cancelled, so this is an unexpected error. + log.Warnf("frp client stopped unexpectedly: %v", err) + m.mu.Lock() + m.status = "error: " + err.Error() + m.appendLogLocked("service stopped unexpectedly: %s", err.Error()) + m.cancel = nil + m.mu.Unlock() + } + }() + + return nil +} + +// Stop gracefully shuts down the FRP client. +func (m *Manager) Stop() { + m.mu.Lock() + cancel := m.cancel + m.cancel = nil + m.mu.Unlock() + + if cancel != nil { + m.appendLog("stopping service") + cancel() + m.wg.Wait() + } + + m.mu.Lock() + m.status = "stopped" + m.appendLogLocked("service stopped") + m.mu.Unlock() +} + +// Restart stops any running client and starts a fresh one with current settings. +func (m *Manager) Restart() error { + m.appendLog("restarting service") + m.Stop() + return m.Start() +} + +// Status returns the current status string: "running", "stopped", or "error: ". +func (m *Manager) Status() string { + m.mu.Lock() + defer m.mu.Unlock() + return m.status +} + +// Runtime returns status and latest logs. +func (m *Manager) Runtime(limit int) RuntimeInfo { + m.mu.Lock() + status := m.status + logPath := m.logPath + m.mu.Unlock() + + logs, err := readLogTail(logPath, limit) + if err != nil { + m.mu.Lock() + logs = m.copyLogsLocked(limit) + m.mu.Unlock() + } + + return RuntimeInfo{ + Status: status, + Logs: logs, + } +} + +func (m *Manager) appendLog(format string, args ...interface{}) { + m.mu.Lock() + defer m.mu.Unlock() + m.appendLogLocked(format, args...) +} + +func (m *Manager) appendLogLocked(format string, args ...interface{}) { + line := fmt.Sprintf(format, args...) + entry := fmt.Sprintf("[%s] %s", time.Now().Format(time.RFC3339), line) + m.logs = append(m.logs, entry) + if len(m.logs) > maxLogEntries { + m.logs = m.logs[len(m.logs)-maxLogEntries:] + } +} + +func (m *Manager) copyLogsLocked(limit int) []string { + if limit <= 0 || limit > maxLogEntries { + limit = maxLogEntries + } + total := len(m.logs) + if total <= limit { + return append([]string(nil), m.logs...) + } + return append([]string(nil), m.logs[total-limit:]...) +} + +func readLogTail(path string, limit int) ([]string, error) { + if limit <= 0 || limit > maxLogEntries { + limit = maxLogEntries + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + size := info.Size() + start := int64(0) + if size > maxTailBytes { + start = size - maxTailBytes + } + if _, err = f.Seek(start, io.SeekStart); err != nil { + return nil, err + } + buf, err := io.ReadAll(f) + if err != nil { + return nil, err + } + + if start > 0 { + if idx := bytes.IndexByte(buf, '\n'); idx >= 0 && idx+1 < len(buf) { + buf = buf[idx+1:] + } + } + text := strings.TrimRight(string(buf), "\n") + if text == "" { + return []string{}, nil + } + lines := strings.Split(text, "\n") + if len(lines) <= limit { + return lines, nil + } + return lines[len(lines)-limit:], nil +} + +func buildConfig() (*v1.ClientCommonConfig, []v1.ProxyConfigurer, error) { + serverAddr := setting.GetStr(conf.FRPServerAddr) + if serverAddr == "" { + return nil, nil, fmt.Errorf("frp server address is required") + } + + serverPort := setting.GetInt(conf.FRPServerPort, 7000) + authToken := setting.GetStr(conf.FRPAuthToken) + proxyName := setting.GetStr(conf.FRPProxyName, "alist") + proxyType := setting.GetStr(conf.FRPProxyType, "http") + customDomain := setting.GetStr(conf.FRPCustomDomain) + subdomain := setting.GetStr(conf.FRPSubdomain) + remotePort := setting.GetInt(conf.FRPRemotePort, 0) + localPort := setting.GetInt(conf.FRPLocalPort, 5244) + tlsEnable := setting.GetBool(conf.FRPTLSEnable) + stcpSecretKey := setting.GetStr(conf.FRPSTCPSecretKey) + + cfg := &v1.ClientCommonConfig{ + ServerAddr: serverAddr, + ServerPort: serverPort, + Auth: v1.AuthClientConfig{ + Method: v1.AuthMethodToken, + Token: authToken, + }, + } + if tlsEnable { + enabled := true + cfg.Transport.TLS.Enable = &enabled + } + + backend := v1.ProxyBackend{ + LocalIP: "127.0.0.1", + LocalPort: localPort, + } + + var proxyCfgs []v1.ProxyConfigurer + + switch proxyType { + case "http": + p := &v1.HTTPProxyConfig{} + p.Name = proxyName + p.Type = "http" + p.ProxyBackend = backend + if customDomain != "" { + p.CustomDomains = []string{customDomain} + } + if subdomain != "" { + p.SubDomain = subdomain + } + proxyCfgs = append(proxyCfgs, p) + + case "https": + p := &v1.HTTPSProxyConfig{} + p.Name = proxyName + p.Type = "https" + p.ProxyBackend = backend + if customDomain != "" { + p.CustomDomains = []string{customDomain} + } + if subdomain != "" { + p.SubDomain = subdomain + } + proxyCfgs = append(proxyCfgs, p) + + case "tcp": + if remotePort <= 0 { + return nil, nil, fmt.Errorf("remote_port is required for tcp proxy type") + } + p := &v1.TCPProxyConfig{} + p.Name = proxyName + p.Type = "tcp" + p.ProxyBackend = backend + p.RemotePort = remotePort + proxyCfgs = append(proxyCfgs, p) + + case "stcp": + p := &v1.STCPProxyConfig{} + p.Name = proxyName + p.Type = "stcp" + p.ProxyBackend = backend + p.Secretkey = stcpSecretKey + p.AllowUsers = []string{"*"} + proxyCfgs = append(proxyCfgs, p) + + default: + return nil, nil, fmt.Errorf("unsupported proxy type: %s", proxyType) + } + + return cfg, proxyCfgs, nil +} diff --git a/internal/fs/list.go b/internal/fs/list.go index d4f59cb829f..927b6ead1a4 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -6,6 +6,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -45,8 +46,13 @@ func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) } func whetherHide(user *model.User, meta *model.Meta, path string) bool { - // if is admin, don't hide - if user == nil || user.CanSeeHides() { + // if user is nil, don't hide + if user == nil { + return false + } + perm := common.MergeRolePermissions(user, path) + // if user has see-hides permission, don't hide + if common.HasPermission(perm, common.PermSeeHides) { return false } // if meta is nil, don't hide diff --git a/internal/fs/other.go b/internal/fs/other.go index 85b7b1d17bf..14f8f63d3ad 100644 --- a/internal/fs/other.go +++ b/internal/fs/other.go @@ -2,10 +2,15 @@ package fs import ( "context" + "encoding/json" + stdpath "path" + "strings" + "github.com/alist-org/alist/v3/drivers/s3" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" "github.com/pkg/errors" ) @@ -53,6 +58,38 @@ func other(ctx context.Context, args model.FsOtherArgs) (interface{}, error) { if err != nil { return nil, errors.WithMessage(err, "failed get storage") } + originalPath := args.Path + + if _, ok := storage.(*s3.S3); ok { + method := strings.ToLower(strings.TrimSpace(args.Method)) + if method == s3.OtherMethodArchive || method == s3.OtherMethodThaw { + if S3TransitionTaskManager == nil { + return nil, errors.New("s3 transition task manager is not initialized") + } + var payload json.RawMessage + if args.Data != nil { + raw, err := json.Marshal(args.Data) + if err != nil { + return nil, errors.WithMessage(err, "failed to encode request payload") + } + payload = raw + } + taskCreator, _ := ctx.Value("user").(*model.User) + tsk := &S3TransitionTask{ + TaskExtension: task.TaskExtension{Creator: taskCreator}, + status: "queued", + StorageMountPath: storage.GetStorage().MountPath, + ObjectPath: actualPath, + DisplayPath: originalPath, + ObjectName: stdpath.Base(actualPath), + Transition: method, + Payload: payload, + } + S3TransitionTaskManager.Add(tsk) + return map[string]string{"task_id": tsk.GetID()}, nil + } + } + args.Path = actualPath return op.Other(ctx, storage, args) } diff --git a/internal/fs/s3_transition.go b/internal/fs/s3_transition.go new file mode 100644 index 00000000000..395f3c5be8c --- /dev/null +++ b/internal/fs/s3_transition.go @@ -0,0 +1,310 @@ +package fs + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/s3" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" + "github.com/pkg/errors" + "github.com/xhofe/tache" +) + +const s3TransitionPollInterval = 15 * time.Second + +// S3TransitionTask represents an asynchronous S3 archive/thaw request that is +// tracked via the task manager so that clients can monitor the progress of the +// operation. +type S3TransitionTask struct { + task.TaskExtension + status string + + StorageMountPath string `json:"storage_mount_path"` + ObjectPath string `json:"object_path"` + DisplayPath string `json:"display_path"` + ObjectName string `json:"object_name"` + Transition string `json:"transition"` + Payload json.RawMessage `json:"payload,omitempty"` + + TargetStorageClass string `json:"target_storage_class,omitempty"` + RequestID string `json:"request_id,omitempty"` + VersionID string `json:"version_id,omitempty"` + + storage driver.Driver `json:"-"` +} + +// S3TransitionTaskManager holds asynchronous S3 archive/thaw tasks. +var S3TransitionTaskManager *tache.Manager[*S3TransitionTask] + +var _ task.TaskExtensionInfo = (*S3TransitionTask)(nil) + +func (t *S3TransitionTask) GetName() string { + action := strings.ToLower(t.Transition) + if action == "" { + action = "transition" + } + display := t.DisplayPath + if display == "" { + display = t.ObjectPath + } + if display == "" { + display = t.ObjectName + } + return fmt.Sprintf("s3 %s %s", action, display) +} + +func (t *S3TransitionTask) GetStatus() string { + return t.status +} + +func (t *S3TransitionTask) Run() error { + t.ReinitCtx() + t.ClearEndTime() + start := time.Now() + t.SetStartTime(start) + defer func() { t.SetEndTime(time.Now()) }() + + if err := t.ensureStorage(); err != nil { + t.status = fmt.Sprintf("locate storage failed: %v", err) + return err + } + + payload, err := t.decodePayload() + if err != nil { + t.status = fmt.Sprintf("decode payload failed: %v", err) + return err + } + + method := strings.ToLower(strings.TrimSpace(t.Transition)) + switch method { + case s3.OtherMethodArchive: + t.status = "submitting archive request" + t.SetProgress(0) + resp, err := op.Other(t.Ctx(), t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodArchive, + Data: payload, + }) + if err != nil { + t.status = fmt.Sprintf("archive request failed: %v", err) + return err + } + archiveResp, ok := toArchiveResponse(resp) + if ok { + if t.TargetStorageClass == "" { + t.TargetStorageClass = archiveResp.StorageClass + } + t.RequestID = archiveResp.RequestID + t.VersionID = archiveResp.VersionID + if archiveResp.StorageClass != "" { + t.status = fmt.Sprintf("archive requested, waiting for %s", archiveResp.StorageClass) + } else { + t.status = "archive requested" + } + } else if sc := t.extractTargetStorageClass(); sc != "" { + t.TargetStorageClass = sc + t.status = fmt.Sprintf("archive requested, waiting for %s", sc) + } else { + t.status = "archive requested" + } + if t.TargetStorageClass != "" { + t.TargetStorageClass = s3.NormalizeStorageClass(t.TargetStorageClass) + } + t.SetProgress(25) + return t.waitForArchive() + case s3.OtherMethodThaw: + t.status = "submitting thaw request" + t.SetProgress(0) + resp, err := op.Other(t.Ctx(), t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodThaw, + Data: payload, + }) + if err != nil { + t.status = fmt.Sprintf("thaw request failed: %v", err) + return err + } + thawResp, ok := toThawResponse(resp) + if ok { + t.RequestID = thawResp.RequestID + if thawResp.Status != nil && !thawResp.Status.Ongoing { + t.SetProgress(100) + t.status = thawCompletionMessage(thawResp.Status) + return nil + } + } + t.status = "thaw requested" + t.SetProgress(25) + return t.waitForThaw() + default: + return errors.Errorf("unsupported transition method: %s", t.Transition) + } +} + +func (t *S3TransitionTask) ensureStorage() error { + if t.storage != nil { + return nil + } + storage, err := op.GetStorageByMountPath(t.StorageMountPath) + if err != nil { + return err + } + t.storage = storage + return nil +} + +func (t *S3TransitionTask) decodePayload() (interface{}, error) { + if len(t.Payload) == 0 { + return nil, nil + } + var payload interface{} + if err := json.Unmarshal(t.Payload, &payload); err != nil { + return nil, err + } + return payload, nil +} + +func (t *S3TransitionTask) extractTargetStorageClass() string { + if len(t.Payload) == 0 { + return "" + } + var req s3.ArchiveRequest + if err := json.Unmarshal(t.Payload, &req); err != nil { + return "" + } + return s3.NormalizeStorageClass(req.StorageClass) +} + +func (t *S3TransitionTask) waitForArchive() error { + ticker := time.NewTicker(s3TransitionPollInterval) + defer ticker.Stop() + + ctx := t.Ctx() + for { + select { + case <-ctx.Done(): + t.status = "archive canceled" + return ctx.Err() + case <-ticker.C: + resp, err := op.Other(ctx, t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodArchiveStatus, + }) + if err != nil { + t.status = fmt.Sprintf("archive status error: %v", err) + return err + } + archiveResp, ok := toArchiveResponse(resp) + if !ok { + t.status = fmt.Sprintf("unexpected archive status response: %T", resp) + return errors.Errorf("unexpected archive status response: %T", resp) + } + currentClass := strings.TrimSpace(archiveResp.StorageClass) + target := strings.TrimSpace(t.TargetStorageClass) + if target == "" { + target = currentClass + t.TargetStorageClass = currentClass + } + if currentClass == "" { + t.status = "waiting for storage class update" + t.SetProgress(50) + continue + } + if strings.EqualFold(currentClass, target) { + t.SetProgress(100) + t.status = fmt.Sprintf("archive complete (%s)", currentClass) + return nil + } + t.status = fmt.Sprintf("storage class %s (target %s)", currentClass, target) + t.SetProgress(75) + } + } +} + +func (t *S3TransitionTask) waitForThaw() error { + ticker := time.NewTicker(s3TransitionPollInterval) + defer ticker.Stop() + + ctx := t.Ctx() + for { + select { + case <-ctx.Done(): + t.status = "thaw canceled" + return ctx.Err() + case <-ticker.C: + resp, err := op.Other(ctx, t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodThawStatus, + }) + if err != nil { + t.status = fmt.Sprintf("thaw status error: %v", err) + return err + } + thawResp, ok := toThawResponse(resp) + if !ok { + t.status = fmt.Sprintf("unexpected thaw status response: %T", resp) + return errors.Errorf("unexpected thaw status response: %T", resp) + } + status := thawResp.Status + if status == nil { + t.status = "waiting for thaw status" + t.SetProgress(50) + continue + } + if status.Ongoing { + t.status = fmt.Sprintf("thaw in progress (%s)", status.Raw) + t.SetProgress(75) + continue + } + t.SetProgress(100) + t.status = thawCompletionMessage(status) + return nil + } + } +} + +func thawCompletionMessage(status *s3.RestoreStatus) string { + if status == nil { + return "thaw complete" + } + if status.Expiry != "" { + return fmt.Sprintf("thaw complete, expires %s", status.Expiry) + } + return "thaw complete" +} + +func toArchiveResponse(v interface{}) (s3.ArchiveResponse, bool) { + switch resp := v.(type) { + case s3.ArchiveResponse: + return resp, true + case *s3.ArchiveResponse: + if resp != nil { + return *resp, true + } + } + return s3.ArchiveResponse{}, false +} + +func toThawResponse(v interface{}) (s3.ThawResponse, bool) { + switch resp := v.(type) { + case s3.ThawResponse: + return resp, true + case *s3.ThawResponse: + if resp != nil { + return *resp, true + } + } + return s3.ThawResponse{}, false +} + +// Ensure compatibility with persistence when tasks are restored. +func (t *S3TransitionTask) OnRestore() { + // The storage handle is not persisted intentionally; it will be lazily + // re-fetched on the next Run invocation. + t.storage = nil +} diff --git a/internal/model/label.go b/internal/model/label.go new file mode 100644 index 00000000000..b397542f5c3 --- /dev/null +++ b/internal/model/label.go @@ -0,0 +1,12 @@ +package model + +import "time" + +type Label struct { + ID uint `json:"id" gorm:"primaryKey"` // unique key + Type int `json:"type"` // use to type + Name string `json:"name"` // use to name + Description string `json:"description"` // use to description + BgColor string `json:"bg_color"` // use to bg_color + CreateTime time.Time `json:"create_time"` +} diff --git a/internal/model/label_file_binding.go b/internal/model/label_file_binding.go new file mode 100644 index 00000000000..af57fed4d88 --- /dev/null +++ b/internal/model/label_file_binding.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type LabelFileBinding struct { + ID uint `json:"id" gorm:"primaryKey"` // unique key + UserId uint `json:"user_id"` // use to user_id + LabelId uint `json:"label_id"` // use to label_id + FileName string `json:"file_name"` // use to file_name + CreateTime time.Time `json:"create_time"` +} diff --git a/internal/model/obj.go b/internal/model/obj.go index f0fce7a133a..ed4e0451ec7 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -20,6 +20,10 @@ type ObjUnwrap interface { Unwrap() Obj } +type StorageClassProvider interface { + StorageClass() string +} + type Obj interface { GetSize() int64 GetName() string @@ -55,6 +59,21 @@ type FileStreamer interface { type UpdateProgress func(percentage float64) +// Reference implementation from OpenListTeam: +// https://github.com/OpenListTeam/OpenList/blob/a703b736c9346c483bae56905a39bc07bf781cff/internal/model/obj.go#L58 +func UpdateProgressWithRange(inner UpdateProgress, start, end float64) UpdateProgress { + return func(p float64) { + if p < 0 { + p = 0 + } + if p > 100 { + p = 100 + } + scaled := start + (end-start)*(p/100.0) + inner(scaled) + } +} + type URL interface { URL() string } @@ -126,6 +145,13 @@ func WrapObjsName(objs []Obj) { } } +func WrapObjStorageClass(obj Obj, storageClass string) Obj { + if storageClass == "" { + return obj + } + return &ObjWrapStorageClass{Obj: obj, storageClass: storageClass} +} + func UnwrapObj(obj Obj) Obj { if unwrap, ok := obj.(ObjUnwrap); ok { obj = unwrap.Unwrap() @@ -153,6 +179,20 @@ func GetUrl(obj Obj) (url string, ok bool) { return url, false } +func GetStorageClass(obj Obj) (string, bool) { + if provider, ok := obj.(StorageClassProvider); ok { + value := provider.StorageClass() + if value == "" { + return "", false + } + return value, true + } + if unwrap, ok := obj.(ObjUnwrap); ok { + return GetStorageClass(unwrap.Unwrap()) + } + return "", false +} + func GetRawObject(obj Obj) *Object { switch v := obj.(type) { case *ObjThumbURL: diff --git a/internal/model/obj_file.go b/internal/model/obj_file.go new file mode 100644 index 00000000000..0fccd6b5cba --- /dev/null +++ b/internal/model/obj_file.go @@ -0,0 +1,18 @@ +package model + +import "time" + +type ObjFile struct { + Id string `json:"id"` + UserId uint `json:"user_id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` +} diff --git a/internal/model/object.go b/internal/model/object.go index c8c10bb9d92..1617662cf9b 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -11,6 +11,11 @@ type ObjWrapName struct { Obj } +type ObjWrapStorageClass struct { + storageClass string + Obj +} + func (o *ObjWrapName) Unwrap() Obj { return o.Obj } @@ -19,6 +24,20 @@ func (o *ObjWrapName) GetName() string { return o.Name } +func (o *ObjWrapStorageClass) Unwrap() Obj { + return o.Obj +} + +func (o *ObjWrapStorageClass) StorageClass() string { + return o.storageClass +} + +func (o *ObjWrapStorageClass) SetPath(path string) { + if setter, ok := o.Obj.(SetPath); ok { + setter.SetPath(path) + } +} + type Object struct { ID string Path string diff --git a/internal/model/paths.go b/internal/model/paths.go new file mode 100644 index 00000000000..8403de8e6a2 --- /dev/null +++ b/internal/model/paths.go @@ -0,0 +1,27 @@ +package model + +import ( + "database/sql/driver" + "encoding/json" + "fmt" +) + +type Paths []string + +func (p Paths) Value() (driver.Value, error) { + return json.Marshal([]string(p)) +} + +func (p *Paths) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, (*[]string)(p)) + case string: + return json.Unmarshal([]byte(v), (*[]string)(p)) + case nil: + *p = nil + return nil + default: + return fmt.Errorf("cannot scan %T", value) + } +} diff --git a/internal/model/role.go b/internal/model/role.go new file mode 100644 index 00000000000..87855551ddb --- /dev/null +++ b/internal/model/role.go @@ -0,0 +1,53 @@ +package model + +import ( + "encoding/json" + + "gorm.io/gorm" +) + +// PermissionEntry defines permission bitmask for a specific path. +type PermissionEntry struct { + Path string `json:"path"` // path prefix, e.g. "/admin" + Permission int32 `json:"permission"` // bitmask permissions +} + +// Role represents a permission template which can be bound to users. +type Role struct { + ID uint `json:"id" gorm:"primaryKey"` + Name string `json:"name" gorm:"unique" binding:"required"` + Description string `json:"description"` + Default bool `json:"default" gorm:"default:false"` + // PermissionScopes stores structured permission list and is ignored by gorm. + PermissionScopes []PermissionEntry `json:"permission_scopes" gorm:"-"` + // RawPermission is the JSON representation of PermissionScopes stored in DB. + RawPermission string `json:"-" gorm:"type:text"` +} + +// BeforeSave GORM hook serializes PermissionScopes into RawPermission. +func (r *Role) BeforeSave(tx *gorm.DB) error { + if len(r.PermissionScopes) == 0 { + r.RawPermission = "" + return nil + } + bs, err := json.Marshal(r.PermissionScopes) + if err != nil { + return err + } + r.RawPermission = string(bs) + return nil +} + +// AfterFind GORM hook deserializes RawPermission into PermissionScopes. +func (r *Role) AfterFind(tx *gorm.DB) error { + if r.RawPermission == "" { + r.PermissionScopes = nil + return nil + } + var scopes []PermissionEntry + if err := json.Unmarshal([]byte(r.RawPermission), &scopes); err != nil { + return err + } + r.PermissionScopes = scopes + return nil +} diff --git a/internal/model/roles.go b/internal/model/roles.go new file mode 100644 index 00000000000..eb626cb93f7 --- /dev/null +++ b/internal/model/roles.go @@ -0,0 +1,36 @@ +package model + +import ( + "database/sql/driver" + "encoding/json" + "fmt" +) + +type Roles []int + +func (r Roles) Value() (driver.Value, error) { + return json.Marshal([]int(r)) +} + +func (r *Roles) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, (*[]int)(r)) + case string: + return json.Unmarshal([]byte(v), (*[]int)(r)) + case nil: + *r = nil + return nil + default: + return fmt.Errorf("cannot scan %T", value) + } +} + +func (r Roles) Contains(role int) bool { + for _, v := range r { + if v == role { + return true + } + } + return false +} diff --git a/internal/model/session.go b/internal/model/session.go new file mode 100644 index 00000000000..3cb6d0dab90 --- /dev/null +++ b/internal/model/session.go @@ -0,0 +1,16 @@ +package model + +// Session represents a device session of a user. +type Session struct { + UserID uint `json:"user_id" gorm:"index"` + DeviceKey string `json:"device_key" gorm:"primaryKey;size:64"` + UserAgent string `json:"user_agent" gorm:"size:255"` + IP string `json:"ip" gorm:"size:64"` + LastActive int64 `json:"last_active"` + Status int `json:"status"` +} + +const ( + SessionActive = iota + SessionInactive +) diff --git a/internal/model/setting.go b/internal/model/setting.go index 93b81fe5941..9e23f9509e6 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -13,6 +13,7 @@ const ( S3 FTP TRAFFIC + FRP ) const ( diff --git a/internal/model/share.go b/internal/model/share.go new file mode 100644 index 00000000000..30fbb8cb7d4 --- /dev/null +++ b/internal/model/share.go @@ -0,0 +1,62 @@ +package model + +import "time" + +type Share struct { + ID uint `json:"id" gorm:"primaryKey"` + ShareID string `json:"share_id" gorm:"uniqueIndex;size:32;not null"` + CreatorID uint `json:"creator_id" gorm:"index;not null"` + Name string `json:"name" gorm:"size:255;not null"` + RootPath string `json:"root_path" gorm:"size:4096;not null"` + IsDir bool `json:"is_dir"` + PasswordHash string `json:"-" gorm:"size:64"` + PasswordSalt string `json:"-" gorm:"size:32"` + BurnAfterRead bool `json:"burn_after_read" gorm:"default:false"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` + AllowPreview bool `json:"allow_preview" gorm:"default:true"` + AllowDownload bool `json:"allow_download" gorm:"default:true"` + Enabled bool `json:"enabled" gorm:"default:true;index"` + ViewCount int64 `json:"view_count"` + DownloadCount int64 `json:"download_count"` + LastAccessAt *time.Time `json:"last_access_at"` + ConsumedAt *time.Time `json:"consumed_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (s Share) HasPassword() bool { + return s.PasswordHash != "" +} + +func (s Share) EffectiveAccessLimit() int64 { + if s.AccessLimit > 0 { + return s.AccessLimit + } + if s.BurnAfterRead { + return 1 + } + return 0 +} + +func (s Share) RemainingAccesses() int64 { + limit := s.EffectiveAccessLimit() + if limit <= 0 { + return 0 + } + remaining := limit - s.AccessCount + if remaining < 0 { + return 0 + } + return remaining +} + +func (s Share) IsConsumed() bool { + limit := s.EffectiveAccessLimit() + return s.ConsumedAt != nil || (limit > 0 && s.AccessCount >= limit) +} + +func (s Share) IsExpired(now time.Time) bool { + return s.ExpiresAt != nil && !s.ExpiresAt.After(now) +} diff --git a/internal/model/storage.go b/internal/model/storage.go index e3c7e1f9731..4d9c062518d 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -28,10 +28,11 @@ type Sort struct { } type Proxy struct { - WebProxy bool `json:"web_proxy"` - WebdavPolicy string `json:"webdav_policy"` - ProxyRange bool `json:"proxy_range"` - DownProxyUrl string `json:"down_proxy_url"` + WebProxy bool `json:"web_proxy"` + WebdavPolicy string `json:"webdav_policy"` + ProxyRange bool `json:"proxy_range"` + DownProxyUrl string `json:"down_proxy_url"` + DownProxySign bool `json:"down_proxy_sign" gorm:"default:true"` } func (s *Storage) GetStorage() *Storage { diff --git a/internal/model/user.go b/internal/model/user.go index eaa0fed9d09..f55b6a5a2a2 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -17,20 +17,22 @@ const ( GENERAL = iota GUEST // only one exists ADMIN + NEWGENERAL ) const StaticHashSalt = "https://github.com/alist-org/alist" type User struct { - ID uint `json:"id" gorm:"primaryKey"` // unique key - Username string `json:"username" gorm:"unique" binding:"required"` // username - PwdHash string `json:"-"` // password hash - PwdTS int64 `json:"-"` // password timestamp - Salt string `json:"-"` // unique salt - Password string `json:"password"` // password - BasePath string `json:"base_path"` // base path - Role int `json:"role"` // user's role - Disabled bool `json:"disabled"` + ID uint `json:"id" gorm:"primaryKey"` // unique key + Username string `json:"username" gorm:"unique" binding:"required"` // username + PwdHash string `json:"-"` // password hash + PwdTS int64 `json:"-"` // password timestamp + Salt string `json:"-"` // unique salt + Password string `json:"password"` // password + BasePath string `json:"base_path"` // base path + Role Roles `json:"role" gorm:"type:text"` // user's roles + RolesDetail []Role `json:"-" gorm:"-"` + Disabled bool `json:"disabled"` // Determine permissions by bit // 0: can see hidden files // 1: can access without password @@ -46,6 +48,9 @@ type User struct { // 11: ftp/sftp write // 12: can read archives // 13: can decompress archives + // 14: check path limit + // 15: mcp read + // 16: mcp write Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -53,11 +58,11 @@ type User struct { } func (u *User) IsGuest() bool { - return u.Role == GUEST + return u.Role.Contains(GUEST) } func (u *User) IsAdmin() bool { - return u.Role == ADMIN + return u.Role.Contains(ADMIN) } func (u *User) ValidateRawPassword(password string) error { @@ -137,8 +142,42 @@ func (u *User) CanDecompress() bool { return (u.Permission>>13)&1 == 1 } +func (u *User) CheckPathLimit() bool { + return (u.Permission>>14)&1 == 1 +} + +func (u *User) CanMCPAccess() bool { + return (u.Permission>>15)&1 == 1 +} + +func (u *User) CanMCPManage() bool { + return (u.Permission>>16)&1 == 1 +} + func (u *User) JoinPath(reqPath string) (string, error) { - return utils.JoinBasePath(u.BasePath, reqPath) + if reqPath == "/" { + return utils.FixAndCleanPath(u.BasePath), nil + } + path, err := utils.JoinBasePath(u.BasePath, reqPath) + if err != nil { + return "", err + } + + if path != "/" && u.CheckPathLimit() { + basePaths := GetAllBasePathsFromRoles(u) + match := false + for _, base := range basePaths { + if utils.IsSubPath(base, path) { + match = true + break + } + } + if !match { + return "", errs.PermissionDenied + } + } + + return path, nil } func StaticHash(password string) string { @@ -177,5 +216,35 @@ func (u *User) WebAuthnCredentials() []webauthn.Credential { } func (u *User) WebAuthnIcon() string { - return "https://alist.nn.ci/logo.svg" + return "https://alistgo.com/logo.svg" +} + +// FetchRole is used to load role details by id. It should be set by the op package +// to avoid an import cycle between model and op. +var FetchRole func(uint) (*Role, error) + +// GetAllBasePathsFromRoles returns all permission paths from user's roles +func GetAllBasePathsFromRoles(u *User) []string { + basePaths := make([]string, 0) + seen := make(map[string]struct{}) + + for _, rid := range u.Role { + if FetchRole == nil { + continue + } + role, err := FetchRole(uint(rid)) + if err != nil || role == nil { + continue + } + for _, entry := range role.PermissionScopes { + if entry.Path == "" { + continue + } + if _, ok := seen[entry.Path]; !ok { + basePaths = append(basePaths, entry.Path) + seen[entry.Path] = struct{}{} + } + } + } + return basePaths } diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index 3d0c7c73a0b..1ba191e47e8 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -3,6 +3,7 @@ package offline_download import ( _ "github.com/alist-org/alist/v3/internal/offline_download/115" _ "github.com/alist-org/alist/v3/internal/offline_download/aria2" + _ "github.com/alist-org/alist/v3/internal/offline_download/guangyapan" _ "github.com/alist-org/alist/v3/internal/offline_download/http" _ "github.com/alist-org/alist/v3/internal/offline_download/pikpak" _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" diff --git a/internal/offline_download/guangyapan/guangyapan.go b/internal/offline_download/guangyapan/guangyapan.go new file mode 100644 index 00000000000..eb58af0bd4a --- /dev/null +++ b/internal/offline_download/guangyapan/guangyapan.go @@ -0,0 +1,131 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" +) + +type GuangYaPan struct { + refreshTaskCache bool +} + +func (g *GuangYaPan) Name() string { + return "GuangYaPan" +} + +func (g *GuangYaPan) Items() []model.SettingItem { + return nil +} + +func (g *GuangYaPan) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (g *GuangYaPan) Init() (string, error) { + g.refreshTaskCache = false + return "ok", nil +} + +func (g *GuangYaPan) IsReady() bool { + tempDir := setting.GetStr(conf.GuangYaPanTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*guangyapandriver.GuangYaPan); !ok { + return false + } + return true +} + +func (g *GuangYaPan) AddURL(args *tool.AddUrlArgs) (string, error) { + g.refreshTaskCache = true + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return "", errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + + ctx := context.Background() + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + task, err := driver.OfflineDownload(ctx, args.Url, parentDir, "") + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + return task.TaskID, nil +} + +func (g *GuangYaPan) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + ctx := context.Background() + if err := driver.DeleteOfflineTasks(ctx, []string{task.GID}, false); err != nil { + return err + } + g.DelTaskCache(driver, task.GID) + return nil +} + +func (g *GuangYaPan) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return nil, errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + tasks, err := g.GetTasks(driver, task.GID) + if err != nil { + return nil, err + } + status := &tool.Status{ + Status: "the task has been deleted", + } + for _, t := range tasks { + if t.TaskID != task.GID { + continue + } + status.Progress = float64(t.Progress) + status.TotalBytes = t.TotalSize + status.Completed = t.Status == offlineStatusCompleted || t.Status == offlineStatusPartiallyCompleted + status.Status = taskStatusText(t) + if t.Status == offlineStatusFailed || t.Status == offlineStatusCanceled { + status.Err = errors.New(status.Status) + } + return status, nil + } + status.Err = errors.New("the task has been deleted") + return status, nil +} + +func init() { + tool.Tools.Add(&GuangYaPan{}) +} diff --git a/internal/offline_download/guangyapan/util.go b/internal/offline_download/guangyapan/util.go new file mode 100644 index 00000000000..cb7bb926ead --- /dev/null +++ b/internal/offline_download/guangyapan/util.go @@ -0,0 +1,77 @@ +package guangyapan + +import ( + "context" + "fmt" + "time" + + "github.com/Xhofe/go-cache" + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/singleflight" +) + +const ( + offlineStatusQueued = 0 + offlineStatusRunning = 1 + offlineStatusCompleted = 2 + offlineStatusFailed = 3 + offlineStatusCanceled = 4 + offlineStatusPartiallyCompleted = 5 +) + +var taskCache = cache.NewMemCache(cache.WithShards[[]guangyapandriver.OfflineTask](16)) +var taskG singleflight.Group[[]guangyapandriver.OfflineTask] + +func (g *GuangYaPan) GetTasks(driver *guangyapandriver.GuangYaPan, taskID string) ([]guangyapandriver.OfflineTask, error) { + key := op.Key(driver, "/cloudcollection/v1/list_task/"+taskID) + if !g.refreshTaskCache { + if tasks, ok := taskCache.Get(key); ok { + return tasks, nil + } + } + g.refreshTaskCache = false + tasks, err, _ := taskG.Do(key, func() ([]guangyapandriver.OfflineTask, error) { + ctx := context.Background() + tasks, err := driver.OfflineList(ctx, []string{taskID}, nil, "", 0) + if err != nil { + return nil, err + } + if len(tasks) > 0 { + taskCache.Set(key, tasks, cache.WithEx[[]guangyapandriver.OfflineTask](time.Second*10)) + } else { + taskCache.Del(key) + } + return tasks, nil + }) + if err != nil { + return nil, err + } + return tasks, nil +} + +func (g *GuangYaPan) DelTaskCache(driver *guangyapandriver.GuangYaPan, taskID string) { + taskCache.Del(op.Key(driver, "/cloudcollection/v1/list_task/"+taskID)) +} + +func taskStatusText(task guangyapandriver.OfflineTask) string { + switch task.Status { + case offlineStatusQueued: + return "queued" + case offlineStatusRunning: + if task.Progress > 0 { + return fmt.Sprintf("running (%d%%)", task.Progress) + } + return "running" + case offlineStatusCompleted: + return "completed" + case offlineStatusFailed: + return "failed" + case offlineStatusCanceled: + return "canceled" + case offlineStatusPartiallyCompleted: + return "partially completed" + default: + return fmt.Sprintf("unknown status %d", task.Status) + } +} diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index d64e43e8615..92e7a3e1dfb 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -7,6 +7,7 @@ import ( "path/filepath" _115 "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/drivers/guangyapan" "github.com/alist-org/alist/v3/drivers/pikpak" "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" @@ -103,6 +104,16 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } else { tempDir = filepath.Join(setting.GetStr(conf.ThunderTempDir), uid) } + case "GuangYaPan": + if _, ok := storage.(*guangyapan.GuangYaPan); ok { + tempDir = args.DstDirPath + } else { + tempBase := setting.GetStr(conf.GuangYaPanTempDir) + if tempBase == "" { + return nil, errors.New("GuangYaPan temp dir is not set") + } + tempDir = filepath.Join(tempBase, uid) + } } taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 42b2dbfb2cb..c6ad09947e1 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -87,6 +87,9 @@ outer: if t.tool.Name() == "Thunder" { return nil } + if t.tool.Name() == "GuangYaPan" { + return nil + } if t.tool.Name() == "115 Cloud" { // hack for 115 <-time.After(time.Second * 1) @@ -159,7 +162,7 @@ func (t *DownloadTask) Update() (bool, error) { func (t *DownloadTask) Transfer() error { toolName := t.tool.Name() - if toolName == "115 Cloud" || toolName == "PikPak" || toolName == "Thunder" { + if toolName == "115 Cloud" || toolName == "PikPak" || toolName == "Thunder" || toolName == "GuangYaPan" { // 如果不是直接下载到目标路径，则进行转存 if t.TempDir != t.DstDirPath { return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) diff --git a/internal/op/driver.go b/internal/op/driver.go index 41b6f6d42c7..4099fbbf5dd 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -117,6 +117,11 @@ func getMainItems(config driver.Config) []driver.Item { Name: "down_proxy_url", Type: conf.TypeText, }) + items = append(items, driver.Item{ + Name: "down_proxy_sign", + Type: conf.TypeBool, + Default: "true", + }) if config.LocalSort { items = append(items, []driver.Item{{ Name: "order_by", diff --git a/internal/op/hook.go b/internal/op/hook.go index 23b8e59af2c..f08966c4ade 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -2,6 +2,7 @@ package op import ( "regexp" + "strconv" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -82,6 +83,18 @@ var settingItemHooks = map[string]SettingItemHook{ conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",") return nil }, + conf.DefaultRole: func(item *model.SettingItem) error { + v := strings.TrimSpace(item.Value) + if v == "" { + return nil + } + id, err := strconv.Atoi(v) + if err != nil { + return errors.WithStack(err) + } + _, err = GetRole(uint(id)) + return err + }, } func RegisterSettingItemHook(key string, hook SettingItemHook) { diff --git a/internal/op/label.go b/internal/op/label.go new file mode 100644 index 00000000000..7e913edfa8d --- /dev/null +++ b/internal/op/label.go @@ -0,0 +1,24 @@ +package op + +import ( + "context" + "github.com/alist-org/alist/v3/internal/db" + "github.com/pkg/errors" +) + +func DeleteLabelById(ctx context.Context, id, userId uint) error { + _, err := db.GetLabelById(id) + if err != nil { + return errors.WithMessage(err, "failed get label") + } + + if db.GetLabelFileBinDingByLabelIdExists(id, userId) { + return errors.New("label have binding relationships") + } + + // delete the label in the database + if err := db.DeleteLabelById(id); err != nil { + return errors.WithMessage(err, "failed delete label in database") + } + return nil +} diff --git a/internal/op/label_file_binding.go b/internal/op/label_file_binding.go new file mode 100644 index 00000000000..2802f0c0b38 --- /dev/null +++ b/internal/op/label_file_binding.go @@ -0,0 +1,195 @@ +package op + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "strconv" + "strings" + "time" +) + +type CreateLabelFileBinDingReq struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + LabelIds string `json:"label_ids"` + LabelIDs []uint64 `json:"labelIdList"` +} + +type ObjLabelResp struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + LabelList []model.Label `json:"label_list"` +} + +func GetLabelByFileName(userId uint, fileName string) ([]model.Label, error) { + labelIds, err := db.GetLabelIds(userId, fileName) + if err != nil { + return nil, errors.WithMessage(err, "failed get label_file_binding") + } + var labels []model.Label + if len(labelIds) > 0 { + if labels, err = db.GetLabelByIds(labelIds); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + } + return labels, nil +} + +func GetLabelsByFileNamesPublic(fileNames []string) (map[string][]model.Label, error) { + return db.GetLabelsByFileNamesPublic(fileNames) +} + +func CreateLabelFileBinDing(req CreateLabelFileBinDingReq, userId uint) error { + if err := db.DelLabelFileBinDingByFileName(userId, req.Name); err != nil { + return errors.WithMessage(err, "failed del label_file_bin_ding in database") + } + + ids, err := collectLabelIDs(req) + if err != nil { + return err + } + if len(ids) == 0 { + return nil + } + + for _, id := range ids { + if err = db.CreateLabelFileBinDing(req.Name, uint(id), userId); err != nil { + return errors.WithMessage(err, "failed labels in database") + } + } + + if !db.GetFileByNameExists(req.Name) { + objFile := model.ObjFile{ + Id: req.Id, + UserId: userId, + Path: req.Path, + Name: req.Name, + Size: req.Size, + IsDir: req.IsDir, + Modified: req.Modified, + Created: req.Created, + Sign: req.Sign, + Thumb: req.Thumb, + Type: req.Type, + HashInfoStr: req.HashInfoStr, + } + if err := db.CreateObjFile(objFile); err != nil { + return errors.WithMessage(err, "failed file in database") + } + } + return nil +} + +func GetFileByLabel(userId uint, labelId string) (result []ObjLabelResp, err error) { + labelMap := strings.Split(labelId, ",") + var labelIds []uint + var labelsFile []model.LabelFileBinding + var labels []model.Label + var labelsFileMap = make(map[string][]model.Label) + var labelsMap = make(map[uint]model.Label) + if labelIds, err = StringSliceToUintSlice(labelMap); err != nil { + return nil, errors.WithMessage(err, "failed string to uint err") + } + //查询标签信息 + if labels, err = db.GetLabelByIds(labelIds); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + for _, val := range labels { + labelsMap[val.ID] = val + } + //查询标签对应文件名列表 + if labelsFile, err = db.GetLabelFileBinDingByLabelId(labelIds, userId); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + for _, value := range labelsFile { + var labelTemp model.Label + labelTemp = labelsMap[value.LabelId] + labelsFileMap[value.FileName] = append(labelsFileMap[value.FileName], labelTemp) + } + for index, v := range labelsFileMap { + objFile, err := db.GetFileByName(index, userId) + if err != nil { + return nil, errors.WithMessage(err, "failed GetFileByName in database") + } + objLabel := ObjLabelResp{ + Id: objFile.Id, + Path: objFile.Path, + Name: objFile.Name, + Size: objFile.Size, + IsDir: objFile.IsDir, + Modified: objFile.Modified, + Created: objFile.Created, + Sign: objFile.Sign, + Thumb: objFile.Thumb, + Type: objFile.Type, + HashInfoStr: objFile.HashInfoStr, + LabelList: v, + } + result = append(result, objLabel) + } + return result, nil +} + +func StringSliceToUintSlice(strSlice []string) ([]uint, error) { + uintSlice := make([]uint, len(strSlice)) + for i, str := range strSlice { + // 使用strconv.ParseUint将字符串转换为uint64 + uint64Value, err := strconv.ParseUint(str, 10, 64) + if err != nil { + return nil, err // 如果转换失败，返回错误 + } + // 将uint64值转换为uint（注意：这里可能存在精度损失，如果uint64值超出了uint的范围） + uintSlice[i] = uint(uint64Value) + } + return uintSlice, nil +} + +func RestoreLabelFileBindings(bindings []model.LabelFileBinding, keepIDs bool, override bool) error { + return db.RestoreLabelFileBindings(bindings, keepIDs, override) +} + +func collectLabelIDs(req CreateLabelFileBinDingReq) ([]uint64, error) { + if len(req.LabelIDs) > 0 { + return req.LabelIDs, nil + } + s := strings.TrimSpace(req.LabelIds) + if s == "" { + return nil, nil + } + replacer := strings.NewReplacer("，", ",", "、", ",", "；", ",", ";", ",") + s = replacer.Replace(s) + parts := strings.Split(s, ",") + ids := make([]uint64, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + id, err := strconv.ParseUint(p, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid label ID '%s': %v", p, err) + } + ids = append(ids, id) + } + return ids, nil +} diff --git a/internal/op/meta.go b/internal/op/meta.go index 930f49634c3..29146fcc72c 100644 --- a/internal/op/meta.go +++ b/internal/op/meta.go @@ -2,9 +2,11 @@ package op import ( stdpath "path" + "strconv" "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -19,6 +21,17 @@ var metaCache = cache.NewMemCache(cache.WithShards[*model.Meta](2)) // metaG maybe not needed var metaG singleflight.Group[*model.Meta] +const ( + metaCacheExpiration = time.Hour + defaultMetaNotFoundCacheSec = 60 +) + +func init() { + RegisterSettingChangingCallback(func() { + metaCache.Clear() + }) +} + func GetNearestMeta(path string) (*model.Meta, error) { return getNearestMeta(utils.FixAndCleanPath(path)) } @@ -51,17 +64,34 @@ func getMetaByPath(path string) (*model.Meta, error) { _meta, err := db.GetMetaByPath(path) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { - metaCache.Set(path, nil) + if ex := metaNotFoundCacheExpiration(); ex > 0 { + metaCache.Set(path, nil, cache.WithEx[*model.Meta](ex)) + } return nil, errs.MetaNotFound } return nil, err } - metaCache.Set(path, _meta, cache.WithEx[*model.Meta](time.Hour)) + metaCache.Set(path, _meta, cache.WithEx[*model.Meta](metaCacheExpiration)) return _meta, nil }) return meta, err } +func metaNotFoundCacheExpiration() time.Duration { + item, err := GetSettingItemByKey(conf.MetaNotFoundCacheExpire) + if err != nil || item == nil { + return time.Second * defaultMetaNotFoundCacheSec + } + seconds, err := strconv.Atoi(item.Value) + if err != nil { + return time.Second * defaultMetaNotFoundCacheSec + } + if seconds <= 0 { + return 0 + } + return time.Second * time.Duration(seconds) +} + func DeleteMetaById(id uint) error { old, err := db.GetMetaById(id) if err != nil { @@ -78,6 +108,7 @@ func UpdateMeta(u *model.Meta) error { return err } metaCache.Del(old.Path) + metaCache.Del(u.Path) return db.UpdateMeta(u) } diff --git a/internal/op/role.go b/internal/op/role.go new file mode 100644 index 00000000000..bd874eeed19 --- /dev/null +++ b/internal/op/role.go @@ -0,0 +1,225 @@ +package op + +import ( + "fmt" + "strconv" + "time" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/singleflight" + "github.com/alist-org/alist/v3/pkg/utils" +) + +var roleCache = cache.NewMemCache[*model.Role](cache.WithShards[*model.Role](2)) +var roleG singleflight.Group[*model.Role] + +func init() { + model.FetchRole = GetRole +} + +func enforceAdminRoleDefaults(r *model.Role) error { + if r == nil || r.Name != "admin" { + return nil + } + if len(r.PermissionScopes) == 1 { + scopePath := utils.FixAndCleanPath(r.PermissionScopes[0].Path) + if scopePath == "/" && r.PermissionScopes[0].Permission == 0xFFFF { + r.PermissionScopes[0].Path = "/" + return nil + } + } + + r.PermissionScopes = []model.PermissionEntry{ + {Path: "/", Permission: 0xFFFF}, + } + return db.UpdateRole(r) +} + +func GetRole(id uint) (*model.Role, error) { + key := fmt.Sprint(id) + if r, ok := roleCache.Get(key); ok { + return r, nil + } + r, err, _ := roleG.Do(key, func() (*model.Role, error) { + _r, err := db.GetRole(id) + if err != nil { + return nil, err + } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } + roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(_r.Name, _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + return r, err +} + +func GetRoleByName(name string) (*model.Role, error) { + if r, ok := roleCache.Get(name); ok { + return r, nil + } + r, err, _ := roleG.Do(name, func() (*model.Role, error) { + _r, err := db.GetRoleByName(name) + if err != nil { + return nil, err + } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } + roleCache.Set(name, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(fmt.Sprint(_r.ID), _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + return r, err +} + +func GetDefaultRoleID() int { + item, err := GetSettingItemByKey(conf.DefaultRole) + if err == nil && item != nil && item.Value != "" { + if id, err := strconv.Atoi(item.Value); err == nil && id != 0 { + return id + } + if r, err := db.GetRoleByName(item.Value); err == nil { + return int(r.ID) + } + } + var r model.Role + if err := db.GetDb().Where("`default` = ?", true).First(&r).Error; err == nil { + return int(r.ID) + } + return int(model.GUEST) +} + +func GetRolesByUserID(userID uint) ([]model.Role, error) { + user, err := GetUserById(userID) + if err != nil { + return nil, err + } + + var roles []model.Role + for _, roleID := range user.Role { + key := fmt.Sprint(roleID) + + if r, ok := roleCache.Get(key); ok { + roles = append(roles, *r) + continue + } + + r, err, _ := roleG.Do(key, func() (*model.Role, error) { + _r, err := db.GetRole(uint(roleID)) + if err != nil { + return nil, err + } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } + roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(_r.Name, _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + if err != nil { + return nil, err + } + roles = append(roles, *r) + } + + return roles, nil +} + +func GetRoles(pageIndex, pageSize int) ([]model.Role, int64, error) { + return db.GetRoles(pageIndex, pageSize) +} + +func CreateRole(r *model.Role) error { + for i := range r.PermissionScopes { + r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) + } + roleCache.Del(fmt.Sprint(r.ID)) + roleCache.Del(r.Name) + if err := db.CreateRole(r); err != nil { + return err + } + if r.Default { + roleCache.Clear() + item, err := GetSettingItemByKey(conf.DefaultRole) + if err != nil { + return err + } + item.Value = strconv.Itoa(int(r.ID)) + if err := SaveSettingItem(item); err != nil { + return err + } + } + return nil +} + +func UpdateRole(r *model.Role) error { + old, err := db.GetRole(r.ID) + if err != nil { + return err + } + switch old.Name { + case "admin": + return errs.ErrChangeDefaultRole + case "guest": + r.Name = "guest" + } + for i := range r.PermissionScopes { + r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) + } + //if len(old.PermissionScopes) > 0 && len(r.PermissionScopes) > 0 && + // old.PermissionScopes[0].Path != r.PermissionScopes[0].Path { + // + // oldPath := old.PermissionScopes[0].Path + // newPath := r.PermissionScopes[0].Path + // + // users, err := db.GetUsersByRole(int(r.ID)) + // if err != nil { + // return errors.WithMessage(err, "failed to get users by role") + // } + // + // modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath, users) + // if err != nil { + // return errors.WithMessage(err, "failed to update user base path when role updated") + // } + // + // for _, name := range modifiedUsernames { + // userCache.Del(name) + // } + //} + roleCache.Del(fmt.Sprint(r.ID)) + roleCache.Del(r.Name) + if err := db.UpdateRole(r); err != nil { + return err + } + if r.Default { + roleCache.Clear() + item, err := GetSettingItemByKey(conf.DefaultRole) + if err != nil { + return err + } + item.Value = strconv.Itoa(int(r.ID)) + if err := SaveSettingItem(item); err != nil { + return err + } + } + return nil +} + +func DeleteRole(id uint) error { + old, err := db.GetRole(id) + if err != nil { + return err + } + if old.Name == "admin" || old.Name == "guest" { + return errs.ErrChangeDefaultRole + } + roleCache.Del(fmt.Sprint(id)) + roleCache.Del(old.Name) + return db.DeleteRole(id) +} diff --git a/internal/op/storage.go b/internal/op/storage.go index f957f95b596..3961e32ed9a 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -41,11 +41,28 @@ func GetStorageByMountPath(mountPath string) (driver.Driver, error) { return storageDriver, nil } +func firstPathSegment(p string) string { + p = utils.FixAndCleanPath(p) + p = strings.TrimPrefix(p, "/") + if p == "" { + return "" + } + if i := strings.Index(p, "/"); i >= 0 { + return p[:i] + } + return p +} + // CreateStorage Save the storage to database so storage can get an id // then instantiate corresponding driver and save it in memory func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + + //if storage.MountPath == "/" { + // return 0, errors.New("Mount path cannot be '/'") + //} + var err error // check driver first driverName := storage.Driver @@ -205,17 +222,46 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { } storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + //if storage.MountPath == "/" { + // return errors.New("Mount path cannot be '/'") + //} err = db.UpdateStorage(&storage) if err != nil { return errors.WithMessage(err, "failed update storage in database") } + storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) + if err == nil { + ClearCache(storageDriver, "/") + } if storage.Disabled { return nil } - storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) + modifiedRoleIDs, err := db.UpdateRolePermissionsPathPrefix(oldStorage.MountPath, storage.MountPath) + if err != nil { + return errors.WithMessage(err, "failed to update role permissions") + } + for _, id := range modifiedRoleIDs { + roleCache.Del(fmt.Sprint(id)) + } + + //modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldStorage.MountPath, storage.MountPath) + //if err != nil { + // return errors.WithMessage(err, "failed to update user base path") + //} + for _, id := range modifiedRoleIDs { + roleCache.Del(fmt.Sprint(id)) + + users, err := db.GetUsersByRole(int(id)) + if err != nil { + return errors.WithMessage(err, "failed to get users by role") + } + for _, user := range users { + userCache.Del(user.Username) + } + } } if err != nil { return errors.WithMessage(err, "failed get storage driver") @@ -236,6 +282,34 @@ func DeleteStorageById(ctx context.Context, id uint) error { if err != nil { return errors.WithMessage(err, "failed get storage") } + firstMount := firstPathSegment(storage.MountPath) + if firstMount != "" { + roles, err := db.GetAllRoles() + if err != nil { + return errors.WithMessage(err, "failed to load roles") + } + users, err := db.GetAllUsers() + if err != nil { + return errors.WithMessage(err, "failed to load users") + } + var usedBy []string + for _, r := range roles { + for _, entry := range r.PermissionScopes { + if firstPathSegment(entry.Path) == firstMount { + usedBy = append(usedBy, "role:"+r.Name) + break + } + } + } + for _, u := range users { + if firstPathSegment(u.BasePath) == firstMount { + usedBy = append(usedBy, "user:"+u.Username) + } + } + if len(usedBy) > 0 { + return errors.Errorf("storage is used by %s, please cancel usage first", strings.Join(usedBy, ", ")) + } + } if !storage.Disabled { storageDriver, err := GetStorageByMountPath(storage.MountPath) if err != nil { diff --git a/internal/op/user.go b/internal/op/user.go index 79e73db86ce..b58a87ed338 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -16,20 +16,52 @@ var userG singleflight.Group[*model.User] var guestUser *model.User var adminUser *model.User +func enforceAdminUserDefaults(u *model.User) error { + if u == nil || !u.IsAdmin() { + return nil + } + changed := false + if utils.FixAndCleanPath(u.BasePath) != "/" { + u.BasePath = "/" + changed = true + } + if u.Permission != 0xFFFF { + u.Permission = 0xFFFF + changed = true + } + if !changed { + return nil + } + return db.UpdateUser(u) +} + func GetAdmin() (*model.User, error) { if adminUser == nil { - user, err := db.GetUserByRole(model.ADMIN) + role, err := GetRoleByName("admin") if err != nil { return nil, err } + user, err := db.GetUserByRole(int(role.ID)) + if err != nil { + return nil, err + } + if err := enforceAdminUserDefaults(user); err != nil { + return nil, err + } adminUser = user + } else if err := enforceAdminUserDefaults(adminUser); err != nil { + return nil, err } return adminUser, nil } func GetGuest() (*model.User, error) { if guestUser == nil { - user, err := db.GetUserByRole(model.GUEST) + role, err := GetRoleByName("guest") + if err != nil { + return nil, err + } + user, err := db.GetUserByRole(int(role.ID)) if err != nil { return nil, err } @@ -42,11 +74,18 @@ func GetUserByRole(role int) (*model.User, error) { return db.GetUserByRole(role) } +func GetUsersByRole(role int) ([]model.User, error) { + return db.GetUsersByRole(role) +} + func GetUserByName(username string) (*model.User, error) { if username == "" { return nil, errs.EmptyUsername } if user, ok := userCache.Get(username); ok { + if err := enforceAdminUserDefaults(user); err != nil { + return nil, err + } return user, nil } user, err, _ := userG.Do(username, func() (*model.User, error) { @@ -54,6 +93,9 @@ func GetUserByName(username string) (*model.User, error) { if err != nil { return nil, err } + if err := enforceAdminUserDefaults(_user); err != nil { + return nil, err + } userCache.Set(username, _user, cache.WithEx[*model.User](time.Hour)) return _user, nil }) @@ -70,7 +112,25 @@ func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err err func CreateUser(u *model.User) error { u.BasePath = utils.FixAndCleanPath(u.BasePath) - return db.CreateUser(u) + + err := db.CreateUser(u) + if err != nil { + return err + } + + roles, err := GetRolesByUserID(u.ID) + if err == nil { + for _, role := range roles { + if len(role.PermissionScopes) > 0 { + u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) + break + } + } + _ = db.UpdateUser(u) + userCache.Del(u.Username) + } + + return nil } func DeleteUserById(id uint) error { @@ -98,6 +158,17 @@ func UpdateUser(u *model.User) error { } userCache.Del(old.Username) u.BasePath = utils.FixAndCleanPath(u.BasePath) + //if len(u.Role) > 0 { + // roles, err := GetRolesByUserID(u.ID) + // if err == nil { + // for _, role := range roles { + // if len(role.PermissionScopes) > 0 { + // u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) + // break + // } + // } + // } + //} return db.UpdateUser(u) } @@ -128,3 +199,11 @@ func DelUserCache(username string) error { userCache.Del(username) return nil } + +func CountEnabledAdminsExcluding(userID uint) (int64, error) { + adminRole, err := GetRoleByName("admin") + if err != nil { + return 0, err + } + return db.CountUsersByRoleAndEnabledExclude(adminRole.ID, userID) +} diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 00000000000..47d1b70125c --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,8 @@ +package session + +import "github.com/alist-org/alist/v3/internal/db" + +// MarkInactive marks the session with the given ID as inactive. +func MarkInactive(sessionID string) error { + return db.MarkInactive(sessionID) +} diff --git a/internal/share/access.go b/internal/share/access.go new file mode 100644 index 00000000000..7baa34e3ef9 --- /dev/null +++ b/internal/share/access.go @@ -0,0 +1,31 @@ +package share + +import ( + "fmt" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + signPkg "github.com/alist-org/alist/v3/pkg/sign" +) + +func tokenPayload(share *model.Share) string { + updatedAt := int64(0) + if !share.UpdatedAt.IsZero() { + updatedAt = share.UpdatedAt.Unix() + } + return fmt.Sprintf("%s:%s:%d", share.ShareID, share.PasswordHash, updatedAt) +} + +func signer() signPkg.Sign { + return signPkg.NewHMACSign([]byte(setting.GetStr(conf.Token) + "-share-access")) +} + +func SignAccess(share *model.Share, d time.Duration) string { + return signer().Sign(tokenPayload(share), time.Now().Add(d).Unix()) +} + +func VerifyAccess(share *model.Share, token string) error { + return signer().Verify(tokenPayload(share), token) +} diff --git a/pkg/utils/mask.go b/pkg/utils/mask.go new file mode 100644 index 00000000000..1513ad40368 --- /dev/null +++ b/pkg/utils/mask.go @@ -0,0 +1,30 @@ +package utils + +import "strings" + +// MaskIP anonymizes middle segments of an IP address. +func MaskIP(ip string) string { + if ip == "" { + return "" + } + if strings.Contains(ip, ":") { + parts := strings.Split(ip, ":") + if len(parts) > 2 { + for i := 1; i < len(parts)-1; i++ { + if parts[i] != "" { + parts[i] = "*" + } + } + return strings.Join(parts, ":") + } + return ip + } + parts := strings.Split(ip, ".") + if len(parts) == 4 { + for i := 1; i < len(parts)-1; i++ { + parts[i] = "*" + } + return strings.Join(parts, ".") + } + return ip +} diff --git a/pkg/utils/path.go b/pkg/utils/path.go index 135f8e4ebca..fe4ff2fd96a 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -88,9 +88,51 @@ func JoinBasePath(basePath, reqPath string) (string, error) { strings.Contains(reqPath, "/../") { return "", errs.RelativePath } + + reqPath = FixAndCleanPath(reqPath) + + if strings.HasPrefix(reqPath, "/") { + return reqPath, nil + } + return stdpath.Join(FixAndCleanPath(basePath), FixAndCleanPath(reqPath)), nil } func GetFullPath(mountPath, path string) string { return stdpath.Join(GetActualMountPath(mountPath), path) } + +// ValidateNameComponent validates a single path component. +// It rejects empty names, dot segments, separators, ".." sequences, and NUL bytes. +func ValidateNameComponent(name string) error { + if name == "" { + return errs.InvalidName + } + if name == "." || name == ".." { + return errs.InvalidName + } + if strings.Contains(name, "/") || strings.Contains(name, "\\") { + return errs.InvalidName + } + if strings.Contains(name, "..") { + return errs.InvalidName + } + if strings.ContainsRune(name, 0) { + return errs.InvalidName + } + return nil +} + +// JoinUnderBase safely joins baseDir with a single name component and ensures the +// result stays under baseDir after normalization. +func JoinUnderBase(baseDir, name string) (string, error) { + if err := ValidateNameComponent(name); err != nil { + return "", err + } + base := FixAndCleanPath(baseDir) + joined := FixAndCleanPath(stdpath.Join(base, name)) + if !IsSubPath(base, joined) { + return "", errs.InvalidName + } + return joined, nil +} diff --git a/pkg/utils/path_test.go b/pkg/utils/path_test.go index f42f2f8bb5d..def286b8038 100644 --- a/pkg/utils/path_test.go +++ b/pkg/utils/path_test.go @@ -20,3 +20,49 @@ func TestFixAndCleanPath(t *testing.T) { } } } + +func TestValidateNameComponent(t *testing.T) { + validNames := []string{ + "file.txt", + "abc", + "file_name-1", + } + for _, name := range validNames { + if err := ValidateNameComponent(name); err != nil { + t.Fatalf("expected valid name %q, got error: %v", name, err) + } + } + + invalidNames := []string{ + "", + ".", + "..", + "a/b", + `a\b`, + "a..b", + string([]byte{'a', 0, 'b'}), + } + for _, name := range invalidNames { + if err := ValidateNameComponent(name); err == nil { + t.Fatalf("expected invalid name %q to be rejected", name) + } + } +} + +func TestJoinUnderBase(t *testing.T) { + base := "/lanzou-y/shared/test1" + out, err := JoinUnderBase(base, "file.txt") + if err != nil { + t.Fatalf("expected join success, got error: %v", err) + } + if out != "/lanzou-y/shared/test1/file.txt" { + t.Fatalf("unexpected join result: %s", out) + } + + if _, err := JoinUnderBase(base, "../admin/screts.txt"); err == nil { + t.Fatalf("expected traversal to be rejected") + } + if _, err := JoinUnderBase(base, "sub/child"); err == nil { + t.Fatalf("expected nested path to be rejected") + } +} diff --git a/server/common/check.go b/server/common/check.go index 78051f4ee1e..010d3131745 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -1,15 +1,11 @@ package common import ( - "path" - "strings" - "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" - "github.com/dlclark/regexp2" ) func IsStorageSignEnabled(rawPath string) bool { @@ -32,30 +28,11 @@ func IsApply(metaPath, reqPath string, applySub bool) bool { } func CanAccess(user *model.User, meta *model.Meta, reqPath string, password string) bool { - // if the reqPath is in hide (only can check the nearest meta) and user can't see hides, can't access - if meta != nil && !user.CanSeeHides() && meta.Hide != "" && - IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path - for _, hide := range strings.Split(meta.Hide, "\n") { - re := regexp2.MustCompile(hide, regexp2.None) - if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { - return false - } - } - } - // if is not guest and can access without password - if user.CanAccessWithoutPassword() { - return true - } - // if meta is nil or password is empty, can access - if meta == nil || meta.Password == "" { - return true - } - // if meta doesn't apply to sub_folder, can access - if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { - return true - } - // validate password - return meta.Password == password + // Deprecated: CanAccess is kept for backward compatibility. + // The logic has been moved to CanAccessWithRoles which performs the + // necessary checks based on role permissions. This wrapper ensures + // older calls still work without relying on user permission bits. + return CanAccessWithRoles(user, meta, reqPath, password) } // ShouldProxy TODO need optimize @@ -64,7 +41,11 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri // 2. storage.WebProxy // 3. proxy_types func ShouldProxy(storage driver.Driver, filename string) bool { - if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + if proxyDriver, ok := storage.(driver.ProxyDriver); ok { + if proxyDriver.ShouldProxyDownloads() || storage.GetStorage().WebProxy { + return true + } + } else if storage.Config().MustProxy() || storage.GetStorage().WebProxy { return true } if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { diff --git a/server/common/proxy.go b/server/common/proxy.go index ca7f6325d7d..dae97c9bf1c 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -13,6 +13,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" + "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" @@ -79,6 +80,9 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. defer res.Body.Close() maps.Copy(w.Header(), res.Header) + if r.URL.Query().Get("type") == "preview" { + w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.PathEscape(file.GetName()))) + } w.WriteHeader(res.StatusCode) if r.Method == http.MethodHead { return nil @@ -129,6 +133,14 @@ func ProxyRange(link *model.Link, size int64) { } } +func BuildDownProxyURL(downProxyURL, path string, useSign bool) string { + base := strings.Split(downProxyURL, "\n")[0] + if useSign { + return fmt.Sprintf("%s%s?sign=%s", base, utils.EncodePath(path, true), sign.Sign(path)) + } + return fmt.Sprintf("%s%s", base, utils.EncodePath(path, true)) +} + type InterceptResponseWriter struct { http.ResponseWriter io.Writer diff --git a/server/common/role_perm.go b/server/common/role_perm.go new file mode 100644 index 00000000000..ec82d4d91a0 --- /dev/null +++ b/server/common/role_perm.go @@ -0,0 +1,139 @@ +package common + +import ( + "path" + "strings" + + "github.com/dlclark/regexp2" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + PermSeeHides = iota + PermAccessWithoutPassword + PermAddOfflineDownload + PermWrite + PermRename + PermMove + PermCopy + PermRemove + PermWebdavRead + PermWebdavManage + PermFTPAccess + PermFTPManage + PermReadArchives + PermDecompress + PermPathLimit + PermMCPAccess + PermMCPManage +) + +func HasPermission(perm int32, bit uint) bool { + return (perm>>bit)&1 == 1 +} + +func MergeRolePermissions(u *model.User, reqPath string) int32 { + if u == nil { + return 0 + } + var perm int32 + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + if reqPath == "/" || utils.PathEqual(reqPath, u.BasePath) { + for _, entry := range role.PermissionScopes { + perm |= entry.Permission + } + } else { + for _, entry := range role.PermissionScopes { + if utils.IsSubPath(entry.Path, reqPath) { + perm |= entry.Permission + } + } + } + } + return perm +} + +func CanAccessWithRoles(u *model.User, meta *model.Meta, reqPath, password string) bool { + if !CanReadPathByRole(u, reqPath) { + return false + } + perm := MergeRolePermissions(u, reqPath) + if meta != nil && !HasPermission(perm, PermSeeHides) && meta.Hide != "" && + IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { + for _, hide := range strings.Split(meta.Hide, "\n") { + re := regexp2.MustCompile(hide, regexp2.None) + if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { + return false + } + } + } + if HasPermission(perm, PermAccessWithoutPassword) { + return true + } + if meta == nil || meta.Password == "" { + return true + } + if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { + return true + } + return meta.Password == password +} + +func CanReadPathByRole(u *model.User, reqPath string) bool { + if u == nil { + return false + } + if reqPath == "/" || utils.PathEqual(reqPath, u.BasePath) { + return len(u.Role) > 0 + } + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + for _, entry := range role.PermissionScopes { + if utils.PathEqual(entry.Path, reqPath) || utils.IsSubPath(entry.Path, reqPath) || utils.IsSubPath(reqPath, entry.Path) { + return true + } + } + } + return false +} + +// HasChildPermission checks whether any child path under reqPath grants the +// specified permission bit. +func HasChildPermission(u *model.User, reqPath string, bit uint) bool { + if u == nil { + return false + } + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + for _, entry := range role.PermissionScopes { + if utils.IsSubPath(reqPath, entry.Path) && HasPermission(entry.Permission, bit) { + return true + } + } + } + return false +} + +// CheckPathLimitWithRoles checks whether the path is allowed when the user has +// the `PermPathLimit` permission for the target path. When the user does not +// have this permission, the check passes by default. +func CheckPathLimitWithRoles(u *model.User, reqPath string) bool { + perm := MergeRolePermissions(u, reqPath) + if HasPermission(perm, PermPathLimit) { + return CanReadPathByRole(u, reqPath) + } + return true +} diff --git a/server/ftp.go b/server/ftp.go index 4d507b684b4..d41063731bf 100644 --- a/server/ftp.go +++ b/server/ftp.go @@ -11,6 +11,7 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/ftp" "math/rand" "net" @@ -130,7 +131,8 @@ func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) return nil, err } } - if userObj.Disabled || !userObj.CanFTPAccess() { + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via FTP") } diff --git a/server/ftp/fsmanage.go b/server/ftp/fsmanage.go index fb03c1b95cb..83e7bae1733 100644 --- a/server/ftp/fsmanage.go +++ b/server/ftp/fsmanage.go @@ -18,7 +18,8 @@ func Mkdir(ctx context.Context, path string) error { if err != nil { return err } - if !user.CanWrite() || !user.CanFTPManage() { + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) || !common.HasPermission(perm, common.PermFTPManage) { meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -34,7 +35,8 @@ func Mkdir(ctx context.Context, path string) error { func Remove(ctx context.Context, path string) error { user := ctx.Value("user").(*model.User) - if !user.CanRemove() || !user.CanFTPManage() { + perm := common.MergeRolePermissions(user, path) + if !common.HasPermission(perm, common.PermRemove) || !common.HasPermission(perm, common.PermFTPManage) { return errs.PermissionDenied } reqPath, err := user.JoinPath(path) @@ -56,13 +58,14 @@ func Rename(ctx context.Context, oldPath, newPath string) error { } srcDir, srcBase := stdpath.Split(srcPath) dstDir, dstBase := stdpath.Split(dstPath) + permSrc := common.MergeRolePermissions(user, srcPath) if srcDir == dstDir { - if !user.CanRename() || !user.CanFTPManage() { + if !common.HasPermission(permSrc, common.PermRename) || !common.HasPermission(permSrc, common.PermFTPManage) { return errs.PermissionDenied } return fs.Rename(ctx, srcPath, dstBase) } else { - if !user.CanFTPManage() || !user.CanMove() || (srcBase != dstBase && !user.CanRename()) { + if !common.HasPermission(permSrc, common.PermFTPManage) || !common.HasPermission(permSrc, common.PermMove) || (srcBase != dstBase && !common.HasPermission(permSrc, common.PermRename)) { return errs.PermissionDenied } if err = fs.Move(ctx, srcPath, dstDir); err != nil { diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index c051a19db21..2ba8cb82abc 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -30,7 +30,7 @@ func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownl } } ctx = context.WithValue(ctx, "meta", meta) - if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { return nil, errs.PermissionDenied } @@ -125,7 +125,7 @@ func Stat(ctx context.Context, path string) (os.FileInfo, error) { } } ctx = context.WithValue(ctx, "meta", meta) - if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { return nil, errs.PermissionDenied } obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) @@ -148,7 +148,7 @@ func List(ctx context.Context, path string) ([]os.FileInfo, error) { } } ctx = context.WithValue(ctx, "meta", meta) - if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { return nil, errs.PermissionDenied } objs, err := fs.List(ctx, reqPath, &fs.ListArgs{}) diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index ee38b1bfb07..9610eea7588 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -35,8 +35,10 @@ func uploadAuth(ctx context.Context, path string) error { return err } } - if !(common.CanAccess(user, meta, path, ctx.Value("meta_pass").(string)) && - ((user.CanFTPManage() && user.CanWrite()) || common.CanWrite(meta, stdpath.Dir(path)))) { + perm := common.MergeRolePermissions(user, path) + if !(common.CanAccessWithRoles(user, meta, path, ctx.Value("meta_pass").(string)) && + ((common.HasPermission(perm, common.PermFTPManage) && common.HasPermission(perm, common.PermWrite)) || + common.CanWrite(meta, stdpath.Dir(path)))) { return errs.PermissionDenied } return nil diff --git a/server/handles/archive.go b/server/handles/archive.go index 550bc3cec9c..844947408be 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -44,17 +44,19 @@ type ArchiveContentResp struct { } func toObjsRespWithoutSignAndThumb(obj model.Obj) ObjResp { + storageClass, _ := model.GetStorageClass(obj) return ObjResp{ - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: "", - Thumb: "", - Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: "", + Thumb: "", + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + StorageClass: storageClass, } } @@ -78,15 +80,20 @@ func FsArchiveMeta(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanReadArchives() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermReadArchives) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -156,15 +163,20 @@ func FsArchiveList(c *gin.Context) { } req.Validate() user := c.MustGet("user").(*model.User) - if !user.CanReadArchives() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermReadArchives) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -242,15 +254,20 @@ func FsArchiveDecompress(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanDecompress() { - common.ErrorResp(c, errs.PermissionDenied, 403) + srcDir, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) return } srcPaths := make([]string, 0, len(req.Name)) for _, name := range req.Name { - srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) + srcPath, err := utils.JoinUnderBase(srcDir, name) if err != nil { - common.ErrorResp(c, err, 403) + common.ErrorResp(c, err, 400) + return + } + if !common.CheckPathLimitWithRoles(user, srcPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) return } srcPaths = append(srcPaths, srcPath) @@ -260,8 +277,17 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } tasks := make([]task.TaskExtensionInfo, 0, len(srcPaths)) for _, srcPath := range srcPaths { + perm := common.MergeRolePermissions(user, srcPath) + if !common.HasPermission(perm, common.PermDecompress) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } t, e := fs.ArchiveDecompress(c, srcPath, dstDir, model.ArchiveDecompressArgs{ ArchiveInnerArgs: model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ diff --git a/server/handles/auth.go b/server/handles/auth.go index 7a2c0fb5376..f59b4fb1ed7 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -3,12 +3,22 @@ package handles import ( "bytes" "encoding/base64" + "errors" + "fmt" "image/png" + "path" + "strings" "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/session" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" @@ -16,8 +26,9 @@ import ( var loginCache = cache.NewMemCache[int]() var ( - defaultDuration = time.Minute * 5 - defaultTimes = 5 + defaultDuration = time.Minute * 5 + defaultTimes = 5 + invalidLoginCredentialsMsg = "username or password is incorrect" ) type LoginReq struct { @@ -59,13 +70,13 @@ func loginHash(c *gin.Context, req *LoginReq) { // check username user, err := op.GetUserByName(req.Username) if err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, invalidLoginCredentialsMsg, 400) loginCache.Set(ip, count+1) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, invalidLoginCredentialsMsg, 400) loginCache.Set(ip, count+1) return } @@ -77,25 +88,74 @@ func loginHash(c *gin.Context, req *LoginReq) { return } } + + clientID := c.GetHeader("Client-Id") + if clientID == "" { + clientID = c.Query("client_id") + } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", + user.ID, clientID)) + + if err := device.EnsureActiveOnLogin(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + if errors.Is(err, errs.TooManyDevices) { + common.ErrorResp(c, err, 403) + } else { + common.ErrorResp(c, err, 400, true) + } + return + } + // generate token token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return } - common.SuccessResp(c, gin.H{"token": token}) + common.SuccessResp(c, gin.H{"token": token, "device_key": key}) loginCache.Del(ip) } +type RegisterReq struct { + Username string `json:"username" binding:"required"` + Password string `json:"password" binding:"required"` +} + +// Register a new user +func Register(c *gin.Context) { + if !setting.GetBool(conf.AllowRegister) { + common.ErrorStrResp(c, "registration is disabled", 403) + return + } + var req RegisterReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := &model.User{ + Username: req.Username, + Role: model.Roles{op.GetDefaultRoleID()}, + Authn: "[]", + } + user.SetPassword(req.Password) + if err := op.CreateUser(user); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + type UserResp struct { model.User - Otp bool `json:"otp"` + Otp bool `json:"otp"` + RoleNames []string `json:"role_names"` + Permissions []model.PermissionEntry `json:"permissions"` } // CurrentUser get current user by token // if token is empty, return guest user func CurrentUser(c *gin.Context) { user := c.MustGet("user").(*model.User) + userResp := UserResp{ User: *user, } @@ -103,6 +163,30 @@ func CurrentUser(c *gin.Context) { if userResp.OtpSecret != "" { userResp.Otp = true } + + var roleNames []string + permMap := map[string]int32{} + paths := make([]string, 0) + + for _, role := range user.RolesDetail { + roleNames = append(roleNames, role.Name) + for _, entry := range role.PermissionScopes { + cleanPath := path.Clean("/" + strings.TrimPrefix(entry.Path, "/")) + if _, ok := permMap[cleanPath]; !ok { + paths = append(paths, cleanPath) + } + permMap[cleanPath] |= entry.Permission + } + } + userResp.RoleNames = roleNames + + for _, fullPath := range paths { + userResp.Permissions = append(userResp.Permissions, model.PermissionEntry{ + Path: fullPath, + Permission: permMap[fullPath], + }) + } + common.SuccessResp(c, userResp) } @@ -187,6 +271,13 @@ func Verify2FA(c *gin.Context) { } func LogOut(c *gin.Context) { + if keyVal, ok := c.Get("device_key"); ok { + if err := session.MarkInactive(keyVal.(string)); err != nil { + common.ErrorResp(c, err, 500) + return + } + c.Set("session_inactive", true) + } err := common.InvalidateToken(c.GetHeader("Authorization")) if err != nil { common.ErrorResp(c, err, 500) diff --git a/server/handles/down.go b/server/handles/down.go index 2c5c2fafc51..680c33f54b1 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -6,20 +6,19 @@ import ( "io" stdpath "path" "strconv" - "strings" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" ) func Down(c *gin.Context) { @@ -57,15 +56,26 @@ func Proxy(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if c.Query("type") == "preview" && storage.GetStorage().Driver == "DoubaoNew" { + // Force proxy for DoubaoNew preview so headers are preserved. + link, file, err := fs.Link(c, rawPath, model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + localProxy(c, link, file, storage.GetStorage().ProxyRange) + return + } if canProxy(storage, filename) { downProxyUrl := storage.GetStorage().DownProxyUrl if downProxyUrl != "" { _, ok := c.GetQuery("d") if !ok { - URL := fmt.Sprintf("%s%s?sign=%s", - strings.Split(downProxyUrl, "\n")[0], - utils.EncodePath(rawPath, true), - sign.Sign(rawPath)) + URL := common.BuildDownProxyURL(downProxyUrl, rawPath, storage.GetStorage().DownProxySign) c.Redirect(302, URL) return } @@ -142,7 +152,8 @@ func localProxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange boo } var html bytes.Buffer - if err = goldmark.Convert(buf.Bytes(), &html); err != nil { + md := goldmark.New(goldmark.WithExtensions(extension.GFM)) + if err = md.Convert(buf.Bytes(), &html); err != nil { err = fmt.Errorf("markdown conversion failed: %w", err) } else { buf.Reset() @@ -178,6 +189,9 @@ func canProxy(storage driver.Driver, filename string) bool { if storage.Config().MustProxy() || storage.GetStorage().WebProxy || storage.GetStorage().WebdavProxy() { return true } + if storage.GetStorage().Driver == "Quark" && utils.GetFileType(filename) == conf.VIDEO { + return true + } if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { return true } diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 3841bff5a34..bccbee72d29 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -10,6 +10,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/generic" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" @@ -29,20 +30,29 @@ func FsRecursiveMove(c *gin.Context) { } user := c.MustGet("user").(*model.User) - if !user.CanMove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermMove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(srcDir) if err != nil { @@ -149,16 +159,20 @@ func FsBatchRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } - reqPath, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(reqPath) if err != nil { @@ -172,7 +186,18 @@ func FsBatchRename(c *gin.Context) { if renameObject.SrcName == "" || renameObject.NewName == "" { continue } - filePath := fmt.Sprintf("%s/%s", reqPath, renameObject.SrcName) + if err := utils.ValidateNameComponent(renameObject.NewName); err != nil { + common.ErrorResp(c, err, 400) + return + } + filePath, err := utils.JoinUnderBase(reqPath, renameObject.SrcName) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if !canRenamePath(c, filePath) { + return + } if err := fs.Rename(c, filePath, renameObject.NewName); err != nil { common.ErrorResp(c, err, 500) return @@ -194,14 +219,19 @@ func FsRegexRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { + reqPath, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CheckPathLimitWithRoles(user, reqPath) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - reqPath, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) return } @@ -229,8 +259,19 @@ func FsRegexRename(c *gin.Context) { for _, file := range files { if srcRegexp.MatchString(file.GetName()) { - filePath := fmt.Sprintf("%s/%s", reqPath, file.GetName()) + filePath, err := utils.JoinUnderBase(reqPath, file.GetName()) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if !canRenamePath(c, filePath) { + return + } newFileName := srcRegexp.ReplaceAllString(file.GetName(), req.NewNameRegex) + if err := utils.ValidateNameComponent(newFileName); err != nil { + common.ErrorResp(c, err, 400) + return + } if err := fs.Rename(c, filePath, newFileName); err != nil { common.ErrorResp(c, err, 500) return diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index c527464e2e4..31976edd9ad 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -2,10 +2,11 @@ package handles import ( "fmt" - "github.com/alist-org/alist/v3/internal/task" "io" stdpath "path" + "github.com/alist-org/alist/v3/internal/task" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" @@ -35,7 +36,12 @@ func FsMkdir(c *gin.Context) { common.ErrorResp(c, err, 403) return } - if !user.CanWrite() { + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) { meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -73,30 +79,54 @@ func FsMove(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanMove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + permMove := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(permMove, common.PermMove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } if !req.Overwrite { for _, name := range req.Names { - if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { + dstPath, err := utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } } } for i, name := range req.Names { - err := fs.Move(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, err = utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + err = fs.Move(c, srcPath, dstDir, len(req.Names) > i+1) if err != nil { common.ErrorResp(c, err, 500) return @@ -116,23 +146,37 @@ func FsCopy(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanCopy() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermCopy) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } if !req.Overwrite { for _, name := range req.Names { - if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { + dstPath, err := utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } @@ -140,7 +184,17 @@ func FsCopy(c *gin.Context) { } var addedTasks []task.TaskExtensionInfo for i, name := range req.Names { - t, err := fs.Copy(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, err = utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + t, err := fs.Copy(c, srcPath, dstDir, len(req.Names) > i+1) if t != nil { addedTasks = append(addedTasks, t) } @@ -160,6 +214,22 @@ type RenameReq struct { Overwrite bool `json:"overwrite"` } +func canRenamePath(c *gin.Context, reqPath string) bool { + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return false + } + return true + } + if meta != nil && meta.Password != "" && common.IsApply(meta.Path, reqPath, meta.PSub) { + common.ErrorStrResp(c, "Path is password-protected and cannot be renamed.", 403) + return false + } + return true +} + func FsRename(c *gin.Context) { var req RenameReq if err := c.ShouldBind(&req); err != nil { @@ -167,17 +237,33 @@ func FsRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + if !canRenamePath(c, reqPath) { + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + if err := utils.ValidateNameComponent(req.Name); err != nil { + common.ErrorResp(c, err, 400) + return + } if !req.Overwrite { - dstPath := stdpath.Join(stdpath.Dir(reqPath), req.Name) + dstPath, err := utils.JoinUnderBase(stdpath.Dir(reqPath), req.Name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } if dstPath != reqPath { if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", req.Name), 403) @@ -208,17 +294,27 @@ func FsRemove(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRemove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqDir, err := user.JoinPath(req.Dir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqDir) + if !common.HasPermission(perm, common.PermRemove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } for _, name := range req.Names { - err := fs.Remove(c, stdpath.Join(reqDir, name)) + removePath, err := utils.JoinUnderBase(reqDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + err = fs.Remove(c, removePath) if err != nil { common.ErrorResp(c, err, 500) return @@ -240,15 +336,20 @@ func FsRemoveEmptyDirectory(c *gin.Context) { } user := c.MustGet("user").(*model.User) - if !user.CanRemove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermRemove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(srcDir) if err != nil { diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 73bde23b6de..1d85dcae935 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -33,36 +33,67 @@ type DirReq struct { } type ObjResp struct { - Id string `json:"id"` - Path string `json:"path"` - Name string `json:"name"` - Size int64 `json:"size"` - IsDir bool `json:"is_dir"` - Modified time.Time `json:"modified"` - Created time.Time `json:"created"` - Sign string `json:"sign"` - Thumb string `json:"thumb"` - Type int `json:"type"` - HashInfoStr string `json:"hashinfo"` - HashInfo map[*utils.HashType]string `json:"hash_info"` + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + StorageClass string `json:"storage_class,omitempty"` } type FsListResp struct { - Content []ObjResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` + Content []ObjLabelResp `json:"content"` + Total int64 `json:"total"` + FilteredTotal int64 `json:"filtered_total"` + Page int `json:"page"` + PerPage int `json:"per_page"` + HasMore bool `json:"has_more"` + PagesTotal int `json:"pages_total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + Provider string `json:"provider"` } +type ObjLabelResp struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + LabelList []model.Label `json:"label_list"` + StorageClass string `json:"storage_class,omitempty"` +} + +const ( + DefaultPerPage = 200 + MaxPerPage = 500 + AllPerPage = -1 +) + func FsList(c *gin.Context) { var req ListReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } - req.Validate() + effPage, effPerPage := normalizeListPage(req.Page, req.PerPage) + req.Page = effPage + req.PerPage = effPerPage user := c.MustGet("user").(*model.User) reqPath, err := user.JoinPath(req.Path) if err != nil { @@ -77,32 +108,49 @@ func FsList(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } - if !user.CanWrite() && !common.CanWrite(meta, reqPath) && req.Refresh { + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) && !common.CanWrite(meta, reqPath) && req.Refresh { common.ErrorStrResp(c, "Refresh without permission", 403) return } + provider := "unknown" + storage, storageErr := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if storageErr == nil { + provider = storage.GetStorage().Driver + } objs, err := fs.List(c, reqPath, &fs.ListArgs{Refresh: req.Refresh}) if err != nil { common.ErrorResp(c, err, 500) return } - total, objs := pagination(objs, &req.PageReq) - provider := "unknown" - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) - if err == nil { - provider = storage.GetStorage().Driver + filtered := make([]model.Obj, 0, len(objs)) + for _, obj := range objs { + childPath := stdpath.Join(reqPath, obj.GetName()) + if common.CanReadPathByRole(user, childPath) { + filtered = append(filtered, obj) + } } + total, pageObjs := pagination(filtered, &req.PageReq) + respContent := toObjsResp(pageObjs, reqPath, isEncrypt(meta, reqPath)) + pagesTotal := calcPagesTotal(total, req.PerPage) + hasMore := req.PerPage != AllPerPage && req.Page*req.PerPage < total + common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), - Total: int64(total), - Readme: getReadme(meta, reqPath), - Header: getHeader(meta, reqPath), - Write: user.CanWrite() || common.CanWrite(meta, reqPath), - Provider: provider, + Content: respContent, + Total: int64(total), + FilteredTotal: int64(total), + Page: req.Page, + PerPage: req.PerPage, + HasMore: hasMore, + PagesTotal: pagesTotal, + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Write: common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, reqPath), + Provider: provider, }) } @@ -135,7 +183,7 @@ func FsDirs(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } @@ -144,7 +192,14 @@ func FsDirs(c *gin.Context) { common.ErrorResp(c, err, 500) return } - dirs := filterDirs(objs) + visible := make([]model.Obj, 0, len(objs)) + for _, obj := range objs { + childPath := stdpath.Join(reqPath, obj.GetName()) + if common.CanReadPathByRole(user, childPath) { + visible = append(visible, obj) + } + } + dirs := filterDirs(visible) common.SuccessResp(c, dirs) } @@ -193,9 +248,43 @@ func isEncrypt(meta *model.Meta, path string) bool { return true } +func normalizeListPage(page, perPage int) (int, int) { + effPage := page + if effPage <= 0 { + effPage = 1 + } + effPerPage := perPage + if effPerPage < 0 { + return effPage, AllPerPage + } + if effPerPage == 0 { + effPerPage = DefaultPerPage + } + if effPerPage > MaxPerPage { + effPerPage = MaxPerPage + } + return effPage, effPerPage +} + +func calcPagesTotal(total, perPage int) int { + if perPage == AllPerPage { + if total > 0 { + return 1 + } + return 0 + } + if total <= 0 || perPage <= 0 { + return 0 + } + return (total + perPage - 1) / perPage +} + func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { pageIndex, pageSize := req.Page, req.PerPage total := len(objs) + if pageSize == AllPerPage { + return total, objs + } start := (pageIndex - 1) * pageSize if start > total { return total, []model.Obj{} @@ -207,23 +296,40 @@ func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { return total, objs[start:end] } -func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { - var resp []ObjResp +func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjLabelResp { + var resp []ObjLabelResp + + names := make([]string, 0, len(objs)) + for _, obj := range objs { + if !obj.IsDir() { + names = append(names, obj.GetName()) + } + } + + labelsByName, _ := op.GetLabelsByFileNamesPublic(names) + for _, obj := range objs { + var labels []model.Label + if !obj.IsDir() { + labels = labelsByName[obj.GetName()] + } thumb, _ := model.GetThumb(obj) - resp = append(resp, ObjResp{ - Id: obj.GetID(), - Path: obj.GetPath(), - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: common.Sign(obj, parent, encrypt), - Thumb: thumb, - Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + storageClass, _ := model.GetStorageClass(obj) + resp = append(resp, ObjLabelResp{ + Id: obj.GetID(), + Path: obj.GetPath(), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: common.Sign(obj, parent, encrypt), + Thumb: thumb, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + LabelList: labels, + StorageClass: storageClass, }) } return resp @@ -236,11 +342,12 @@ type FsGetReq struct { type FsGetResp struct { ObjResp - RawURL string `json:"raw_url"` - Readme string `json:"readme"` - Header string `json:"header"` - Provider string `json:"provider"` - Related []ObjResp `json:"related"` + RawURL string `json:"raw_url"` + Readme string `json:"readme"` + Header string `json:"header"` + Provider string `json:"provider"` + WebProxy bool `json:"web_proxy"` + Related []ObjLabelResp `json:"related"` } func FsGet(c *gin.Context) { @@ -263,7 +370,7 @@ func FsGet(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } @@ -274,26 +381,38 @@ func FsGet(c *gin.Context) { } var rawURL string - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + storage, storageErr := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) provider := "unknown" - if err == nil { + if storageErr == nil { provider = storage.Config().Name } if !obj.IsDir() { - if err != nil { - common.ErrorResp(c, err, 500) + if storageErr != nil { + common.ErrorResp(c, storageErr, 500) return } - if storage.Config().MustProxy() || storage.GetStorage().WebProxy { - query := "" - if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { - query = "?sign=" + sign.Sign(reqPath) - } + query := "" + if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { + query = "?sign=" + sign.Sign(reqPath) + } + forceRedirectRawURL := storage.GetStorage().Driver == "BaiduYouth" + forcePreviewRawURL := storage.GetStorage().Driver == "Lark" && isLarkCloudDocName(obj.GetName()) + forceProxyRawURL := storage.GetStorage().Driver == "Quark" && utils.GetFileType(obj.GetName()) == conf.VIDEO + if forceRedirectRawURL { + // Baidu Youth direct links are minted per request and are not stable enough + // to expose as fs/get raw_url. Return the local /d endpoint so the frontend + // obtains a fresh link on each download click. + rawURL = fmt.Sprintf("%s/d%s%s", + common.GetApiUrl(c.Request), + utils.EncodePath(reqPath, true), + query) + } else if !forcePreviewRawURL && (storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL) { if storage.GetStorage().DownProxyUrl != "" { - rawURL = fmt.Sprintf("%s%s?sign=%s", - strings.Split(storage.GetStorage().DownProxyUrl, "\n")[0], - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)) + rawURL = common.BuildDownProxyURL( + storage.GetStorage().DownProxyUrl, + reqPath, + storage.GetStorage().DownProxySign, + ) } else { rawURL = fmt.Sprintf("%s/p%s%s", common.GetApiUrl(c.Request), @@ -328,25 +447,28 @@ func FsGet(c *gin.Context) { } parentMeta, _ := op.GetNearestMeta(parentPath) thumb, _ := model.GetThumb(obj) + storageClass, _ := model.GetStorageClass(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ - Id: obj.GetID(), - Path: obj.GetPath(), - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), - Type: utils.GetFileType(obj.GetName()), - Thumb: thumb, + Id: obj.GetID(), + Path: obj.GetPath(), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), + Type: utils.GetFileType(obj.GetName()), + Thumb: thumb, + StorageClass: storageClass, }, RawURL: rawURL, Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), Provider: provider, + WebProxy: storageErr == nil && storage.GetStorage().WebProxy, Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)), }) } @@ -365,6 +487,22 @@ func filterRelated(objs []model.Obj, obj model.Obj) []model.Obj { return related } +func isLarkCloudDocName(name string) bool { + for _, suffix := range []string{ + ".lark-doc", + ".lark-docx", + ".lark-sheet", + ".lark-bitable", + ".lark-mindnote", + ".lark-slides", + } { + if strings.HasSuffix(name, suffix) { + return true + } + } + return false +} + type FsOtherReq struct { model.FsOtherArgs Password string `json:"password" form:"password"` @@ -391,7 +529,7 @@ func FsOther(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, req.Path, req.Password) { + if !common.CanAccessWithRoles(user, meta, req.Path, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } diff --git a/server/handles/label.go b/server/handles/label.go new file mode 100644 index 00000000000..4631124ecbf --- /dev/null +++ b/server/handles/label.go @@ -0,0 +1,99 @@ +package handles + +import ( + "errors" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + "strconv" +) + +func ListLabel(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + log.Debugf("%+v", req) + labels, total, err := db.GetLabels(req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, common.PageResp{ + Content: labels, + Total: total, + }) +} + +func GetLabel(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + label, err := db.GetLabelById(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, label) +} + +func CreateLabel(c *gin.Context) { + var req model.Label + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if db.GetLabelByName(req.Name) { + common.ErrorResp(c, errors.New("label name is exists"), 401) + return + } + if id, err := db.CreateLabel(req); err != nil { + common.ErrorWithDataResp(c, err, 500, gin.H{ + "id": id, + }, true) + } else { + common.SuccessResp(c, gin.H{ + "id": id, + }) + } +} + +func UpdateLabel(c *gin.Context) { + var req model.Label + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if label, err := db.UpdateLabel(&req); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c, label) + } +} + +func DeleteLabel(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + if err = op.DeleteLabelById(c, uint(id), userObj.ID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/label_file_binding.go b/server/handles/label_file_binding.go new file mode 100644 index 00000000000..04f0c105fc2 --- /dev/null +++ b/server/handles/label_file_binding.go @@ -0,0 +1,250 @@ +package handles + +import ( + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "net/url" + "strconv" + "strings" +) + +type DelLabelFileBinDingReq struct { + FileName string `json:"file_name"` + LabelId string `json:"label_id"` +} + +type pageResp[T any] struct { + Content []T `json:"content"` + Total int64 `json:"total"` +} + +type restoreLabelBindingsReq struct { + KeepIDs bool `json:"keep_ids"` + Override bool `json:"override"` + Bindings []model.LabelFileBinding `json:"bindings"` +} + +func GetLabelByFileName(c *gin.Context) { + fileName := c.Query("file_name") + if fileName == "" { + common.ErrorResp(c, errors.New("file_name must not empty"), 400) + return + } + decodedFileName, err := url.QueryUnescape(fileName) + if err != nil { + common.ErrorResp(c, errors.New("invalid file_name"), 400) + return + } + fmt.Println(">>> 原始 fileName:", fileName) + fmt.Println(">>> 解码后 fileName:", decodedFileName) + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + labels, err := op.GetLabelByFileName(userObj.ID, decodedFileName) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, labels) +} + +func CreateLabelFileBinDing(c *gin.Context) { + var req op.CreateLabelFileBinDingReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.IsDir == true { + common.ErrorStrResp(c, "Unable to bind folder", 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + if err := op.CreateLabelFileBinDing(req, userObj.ID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } else { + common.SuccessResp(c, gin.H{ + "msg": "添加成功！", + }) + } +} + +func DelLabelByFileName(c *gin.Context) { + var req DelLabelFileBinDingReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + labelId, err := strconv.ParseUint(req.LabelId, 10, 64) + if err != nil { + common.ErrorResp(c, fmt.Errorf("invalid label ID '%s': %v", req.LabelId, err), 500, true) + return + } + if err = db.DelLabelFileBinDingById(uint(labelId), userObj.ID, req.FileName); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func GetFileByLabel(c *gin.Context) { + labelId := c.Query("label_id") + if labelId == "" { + common.ErrorResp(c, errors.New("file_name must not empty"), 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + fileList, err := op.GetFileByLabel(userObj.ID, labelId) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, fileList) +} + +func ListLabelFileBinding(c *gin.Context) { + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + + pageStr := c.DefaultQuery("page", "1") + sizeStr := c.DefaultQuery("page_size", "50") + page, err := strconv.Atoi(pageStr) + if err != nil || page <= 0 { + page = 1 + } + pageSize, err := strconv.Atoi(sizeStr) + if err != nil || pageSize <= 0 || pageSize > 200 { + pageSize = 50 + } + + fileName := c.Query("file_name") + labelIDStr := c.Query("label_id") + var labelIDs []uint + if labelIDStr != "" { + parts := strings.Split(labelIDStr, ",") + for _, p := range parts { + if p == "" { + continue + } + id64, err := strconv.ParseUint(strings.TrimSpace(p), 10, 64) + if err != nil { + common.ErrorResp(c, fmt.Errorf("invalid label_id '%s': %v", p, err), 400) + return + } + labelIDs = append(labelIDs, uint(id64)) + } + } + + list, total, err := db.ListLabelFileBinDing(userObj.ID, labelIDs, fileName, page, pageSize) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, pageResp[model.LabelFileBinding]{ + Content: list, + Total: total, + }) +} + +func RestoreLabelFileBinding(c *gin.Context) { + var req restoreLabelBindingsReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if len(req.Bindings) == 0 { + common.ErrorStrResp(c, "empty bindings", 400) + return + } + + if u, ok := c.Value("user").(*model.User); ok { + for i := range req.Bindings { + if req.Bindings[i].UserId == 0 { + req.Bindings[i].UserId = u.ID + } + } + } + + for i := range req.Bindings { + b := req.Bindings[i] + if b.UserId == 0 || b.LabelId == 0 || strings.TrimSpace(b.FileName) == "" { + common.ErrorStrResp(c, "invalid binding: user_id/label_id/file_name required", 400) + return + } + } + + if err := op.RestoreLabelFileBindings(req.Bindings, req.KeepIDs, req.Override); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, gin.H{ + "msg": fmt.Sprintf("restored %d rows", len(req.Bindings)), + }) +} + +func CreateLabelFileBinDingBatch(c *gin.Context) { + var req struct { + Items []op.CreateLabelFileBinDingReq `json:"items" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil || len(req.Items) == 0 { + common.ErrorResp(c, err, 400) + return + } + + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + + type perResult struct { + Name string `json:"name"` + Ok bool `json:"ok"` + ErrMsg string `json:"errMsg,omitempty"` + } + results := make([]perResult, 0, len(req.Items)) + succeed := 0 + + for _, item := range req.Items { + if item.IsDir { + results = append(results, perResult{Name: item.Name, Ok: false, ErrMsg: "Unable to bind folder"}) + continue + } + if err := op.CreateLabelFileBinDing(item, userObj.ID); err != nil { + results = append(results, perResult{Name: item.Name, Ok: false, ErrMsg: err.Error()}) + continue + } + succeed++ + results = append(results, perResult{Name: item.Name, Ok: true}) + } + + common.SuccessResp(c, gin.H{ + "total": len(req.Items), + "succeed": succeed, + "failed": len(req.Items) - succeed, + "results": results, + }) +} diff --git a/server/handles/lark.go b/server/handles/lark.go new file mode 100644 index 00000000000..b8f6f7f9d60 --- /dev/null +++ b/server/handles/lark.go @@ -0,0 +1,84 @@ +package handles + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type larkExportDownloader interface { + DownloadExportFile(ctx context.Context, fileToken string) (io.Reader, string, error) +} + +func LarkExportDownload(c *gin.Context) { + rawPath := c.Query("path") + fileToken := c.Query("file_token") + password := c.Query("password") + filename := strings.TrimSpace(c.Query("filename")) + if rawPath == "" || fileToken == "" { + common.ErrorStrResp(c, "path and file_token are required", 400) + return + } + + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(rawPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500) + return + } + } + if !common.CanAccessWithRoles(user, meta, reqPath, password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + downloader, ok := storage.(larkExportDownloader) + if !ok || storage.GetStorage().Driver != "Lark" { + common.ErrorStrResp(c, "lark export download is not supported for this storage", 400) + return + } + + reader, respFilename, err := downloader.DownloadExportFile(c, fileToken) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if filename == "" { + filename = respFilename + } + if filename == "" { + filename = stdpath.Base(reqPath) + } + + c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, filename, url.PathEscape(filename))) + c.Header("Content-Type", utils.GetMimeType(filename)) + c.Status(http.StatusOK) + if _, err = io.Copy(c.Writer, reader); err != nil { + common.ErrorResp(c, err, 500) + return + } +} diff --git a/server/handles/ldap_login.go b/server/handles/ldap_login.go index cf3148291b1..fb8417b68b5 100644 --- a/server/handles/ldap_login.go +++ b/server/handles/ldap_login.go @@ -131,7 +131,7 @@ func ladpRegister(username string) (*model.User, error) { Password: random.String(16), Permission: int32(setting.GetInt(conf.LdapDefaultPermission, 0)), BasePath: setting.GetStr(conf.LdapDefaultDir), - Role: 0, + Role: nil, Disabled: false, } if err := db.CreateUser(user); err != nil { @@ -150,7 +150,7 @@ func dial(ldapServer string) (*ldap.Conn, error) { } if tlsEnabled { - return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: true}) + return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) } else { return ldap.Dial("tcp", ldapServer) } diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 24ff7a05369..68a922efda7 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -2,9 +2,11 @@ package handles import ( _115 "github.com/alist-org/alist/v3/drivers/115" + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" "github.com/alist-org/alist/v3/drivers/pikpak" "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/op" @@ -239,6 +241,50 @@ func SetThunder(c *gin.Context) { common.SuccessResp(c, "ok") } +type SetGuangYaPanReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetGuangYaPan(c *gin.Context) { + var req SetGuangYaPanReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*guangyapandriver.GuangYaPan); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only GuangYaPan is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.GuangYaPanTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("GuangYaPan") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + func OfflineDownloadTools(c *gin.Context) { tools := tool.Tools.Names() common.SuccessResp(c, tools) @@ -253,10 +299,6 @@ type AddOfflineDownloadReq struct { func AddOfflineDownload(c *gin.Context) { user := c.MustGet("user").(*model.User) - if !user.CanAddOfflineDownloadTasks() { - common.ErrorStrResp(c, "permission denied", 403) - return - } var req AddOfflineDownloadReq if err := c.ShouldBind(&req); err != nil { @@ -268,6 +310,15 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermAddOfflineDownload) { + common.ErrorStrResp(c, "permission denied", 403) + return + } var tasks []task.TaskExtensionInfo for _, url := range req.Urls { t, err := tool.AddURL(c, &tool.AddURLArgs{ diff --git a/server/handles/role.go b/server/handles/role.go new file mode 100644 index 00000000000..17271a530de --- /dev/null +++ b/server/handles/role.go @@ -0,0 +1,117 @@ +package handles + +import ( + "strconv" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" +) + +func ListRoles(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + log.Debugf("%+v", req) + roles, total, err := op.GetRoles(req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, common.PageResp{Content: roles, Total: total}) +} + +func GetRole(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, role) +} + +func CreateRole(c *gin.Context) { + var req model.Role + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := op.CreateRole(&req); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c) + } +} + +func UpdateRole(c *gin.Context) { + var req struct { + ID uint `json:"id"` + Name string `json:"name" binding:"required"` + Description string `json:"description"` + PermissionScopes []model.PermissionEntry `json:"permission_scopes"` + Default *bool `json:"default"` + } + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(req.ID) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + switch role.Name { + case "admin": + common.ErrorResp(c, errs.ErrChangeDefaultRole, 403) + return + + case "guest": + req.Name = "guest" + } + role.Name = req.Name + role.Description = req.Description + role.PermissionScopes = req.PermissionScopes + if req.Default != nil { + role.Default = *req.Default + } + if err := op.UpdateRole(role); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c) + } +} + +func DeleteRole(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + if role.Name == "admin" || role.Name == "guest" { + common.ErrorResp(c, errs.ErrChangeDefaultRole, 403) + return + } + if err := op.DeleteRole(uint(id)); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/search.go b/server/handles/search.go index 8881731bd60..832fc94fb07 100644 --- a/server/handles/search.go +++ b/server/handles/search.go @@ -43,28 +43,39 @@ func Search(c *gin.Context) { common.ErrorResp(c, err, 400) return } - nodes, total, err := search.Search(c, req.SearchReq) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - var filteredNodes []model.SearchNode - for _, node := range nodes { - if !strings.HasPrefix(node.Parent, user.BasePath) { - continue + var ( + filteredNodes []model.SearchNode + ) + for len(filteredNodes) < req.PerPage { + nodes, _, err := search.Search(c, req.SearchReq) + if err != nil { + common.ErrorResp(c, err, 500) + return } - meta, err := op.GetNearestMeta(node.Parent) - if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { - continue + if len(nodes) == 0 { + break } - if !common.CanAccess(user, meta, path.Join(node.Parent, node.Name), req.Password) { - continue + for _, node := range nodes { + if !strings.HasPrefix(node.Parent, user.BasePath) { + continue + } + meta, err := op.GetNearestMeta(node.Parent) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + continue + } + if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), req.Password) { + continue + } + filteredNodes = append(filteredNodes, node) + if len(filteredNodes) >= req.PerPage { + break + } } - filteredNodes = append(filteredNodes, node) + req.Page++ } common.SuccessResp(c, common.PageResp{ Content: utils.MustSliceConvert(filteredNodes, nodeToSearchResp), - Total: total, + Total: int64(len(filteredNodes)), }) } diff --git a/server/handles/session.go b/server/handles/session.go new file mode 100644 index 00000000000..886be66ab01 --- /dev/null +++ b/server/handles/session.go @@ -0,0 +1,92 @@ +package handles + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +type SessionResp struct { + SessionID string `json:"session_id"` + UserID uint `json:"user_id,omitempty"` + LastActive int64 `json:"last_active"` + Status int `json:"status"` + UA string `json:"ua"` + IP string `json:"ip"` +} + +func ListMySessions(c *gin.Context) { + user := c.MustGet("user").(*model.User) + sessions, err := db.ListSessionsByUser(user.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + resp := make([]SessionResp, len(sessions)) + for i, s := range sessions { + resp[i] = SessionResp{ + SessionID: s.DeviceKey, + LastActive: s.LastActive, + Status: s.Status, + UA: s.UserAgent, + IP: s.IP, + } + } + common.SuccessResp(c, resp) +} + +type EvictSessionReq struct { + SessionID string `json:"session_id"` +} + +func EvictMySession(c *gin.Context) { + var req EvictSessionReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if _, err := db.GetSession(user.ID, req.SessionID); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := db.MarkInactive(req.SessionID); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +func ListSessions(c *gin.Context) { + sessions, err := db.ListSessions() + if err != nil { + common.ErrorResp(c, err, 500) + return + } + resp := make([]SessionResp, len(sessions)) + for i, s := range sessions { + resp[i] = SessionResp{ + SessionID: s.DeviceKey, + UserID: s.UserID, + LastActive: s.LastActive, + Status: s.Status, + UA: s.UserAgent, + IP: s.IP, + } + } + common.SuccessResp(c, resp) +} + +func EvictSession(c *gin.Context) { + var req EvictSessionReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := db.MarkInactive(req.SessionID); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/setting.go b/server/handles/setting.go index f778b1803c5..81f7dc61c24 100644 --- a/server/handles/setting.go +++ b/server/handles/setting.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/sign" @@ -14,9 +15,28 @@ import ( "github.com/gin-gonic/gin" ) +func getRoleOptions() string { + roles, _, err := op.GetRoles(1, model.MaxInt) + if err != nil { + return "" + } + names := make([]string, 0, len(roles)) + for _, r := range roles { + if r.Name == "admin" || r.Name == "guest" { + continue + } + names = append(names, r.Name) + } + return strings.Join(names, ",") +} + +type SetTokenReq struct { + Token string `json:"token" form:"token" binding:"required"` +} + func ResetToken(c *gin.Context) { token := random.Token() - item := model.SettingItem{Key: "token", Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} + item := model.SettingItem{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} if err := op.SaveSettingItem(&item); err != nil { common.ErrorResp(c, err, 500) return @@ -25,6 +45,21 @@ func ResetToken(c *gin.Context) { common.SuccessResp(c, token) } +func SetToken(c *gin.Context) { + var req SetTokenReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + item := model.SettingItem{Key: conf.Token, Value: req.Token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} + if err := op.SaveSettingItem(&item); err != nil { + common.ErrorResp(c, err, 500) + return + } + sign.Instance() + common.SuccessResp(c, req.Token) +} + func GetSetting(c *gin.Context) { key := c.Query("key") keys := c.Query("keys") @@ -34,6 +69,17 @@ func GetSetting(c *gin.Context) { common.ErrorResp(c, err, 400) return } + if item.Key == conf.DefaultRole { + copy := *item + copy.Options = getRoleOptions() + if id, err := strconv.Atoi(copy.Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + copy.Value = r.Name + } + } + common.SuccessResp(c, copy) + return + } common.SuccessResp(c, item) } else { items, err := op.GetSettingItemInKeys(strings.Split(keys, ",")) @@ -41,6 +87,17 @@ func GetSetting(c *gin.Context) { common.ErrorResp(c, err, 400) return } + for i := range items { + if items[i].Key == conf.DefaultRole { + if id, err := strconv.Atoi(items[i].Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + items[i].Value = r.Name + } + } + items[i].Options = getRoleOptions() + break + } + } common.SuccessResp(c, items) } } @@ -51,6 +108,22 @@ func SaveSettings(c *gin.Context) { common.ErrorResp(c, err, 400) return } + + for i := range req { + if req[i].Key == conf.DefaultRole { + role, err := op.GetRoleByName(req[i].Value) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if role.Name == "admin" || role.Name == "guest" { + common.ErrorStrResp(c, "cannot set admin or guest as default role", 400) + return + } + req[i].Value = strconv.Itoa(int(role.ID)) + } + } + if err := op.SaveSettingItems(req); err != nil { common.ErrorResp(c, err, 500) } else { @@ -88,6 +161,17 @@ func ListSettings(c *gin.Context) { common.ErrorResp(c, err, 400) return } + for i := range settings { + if settings[i].Key == conf.DefaultRole { + if id, err := strconv.Atoi(settings[i].Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + settings[i].Value = r.Name + } + } + settings[i].Options = getRoleOptions() + break + } + } common.SuccessResp(c, settings) } @@ -100,6 +184,42 @@ func DeleteSetting(c *gin.Context) { common.SuccessResp(c) } +// SetFRP saves FRP settings and restarts the FRP client. +// Returns the current FRP connection status. +func SetFRP(c *gin.Context) { + var req []model.SettingItem + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := op.SaveSettingItems(req); err != nil { + common.ErrorResp(c, err, 500) + return + } + if err := frp.Instance.Restart(); err != nil { + common.SuccessResp(c, frp.Instance.Status()) + return + } + common.SuccessResp(c, frp.Instance.Status()) +} + +// StopFRP stops the FRP client and returns current status. +func StopFRP(c *gin.Context) { + frp.Instance.Stop() + common.SuccessResp(c, frp.Instance.Status()) +} + +// GetFRPRuntime returns current FRP status and recent runtime logs. +func GetFRPRuntime(c *gin.Context) { + limit := 200 + if limitStr := c.Query("limit"); limitStr != "" { + if parsed, err := strconv.Atoi(limitStr); err == nil { + limit = parsed + } + } + common.SuccessResp(c, frp.Instance.Runtime(limit)) +} + func PublicSettings(c *gin.Context) { common.SuccessResp(c, op.GetPublicSettingsMap()) } diff --git a/server/handles/share.go b/server/handles/share.go new file mode 100644 index 00000000000..4de7da63672 --- /dev/null +++ b/server/handles/share.go @@ -0,0 +1,635 @@ +package handles + +import ( + "crypto/subtle" + "errors" + "fmt" + "net/http" + "net/url" + stdpath "path" + "regexp" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/db" + shareauth "github.com/alist-org/alist/v3/internal/share" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +const shareAccessTokenLifetime = 24 * time.Hour + +var shareIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,32}$`) + +var ( + errShareIDInvalid = errors.New("share_id must be 1-32 characters of letters, numbers, underscore or hyphen") + errShareIDExists = errors.New("share link already exists") +) + +type CreateShareReq struct { + Path string `json:"path" binding:"required"` + ShareID string `json:"share_id"` + Name string `json:"name"` + Password string `json:"password"` + ExpireAt string `json:"expire_at"` + ExpireHours int64 `json:"expire_hours"` + AccessLimit int64 `json:"access_limit"` + BurnAfterRead *bool `json:"burn_after_read"` + AllowPreview *bool `json:"allow_preview"` + AllowDownload *bool `json:"allow_download"` +} + +type UpdateShareReq struct { + ShareID string `json:"share_id" binding:"required"` + NewShareID string `json:"new_share_id"` + Name string `json:"name"` + Password string `json:"password"` + ExpireAt *string `json:"expire_at"` + AccessLimit *int64 `json:"access_limit"` + AllowPreview *bool `json:"allow_preview"` + AllowDownload *bool `json:"allow_download"` +} + +type ShareDeleteReq struct { + ShareID string `json:"share_id" binding:"required"` +} + +type ShareAuthReq struct { + ShareID string `json:"share_id" binding:"required"` + Password string `json:"password"` +} + +type PublicShareReq struct { + ShareID string `json:"share_id" binding:"required"` + Path string `json:"path"` + Token string `json:"token"` +} + +type PublicShareListReq struct { + model.PageReq + ShareID string `json:"share_id" binding:"required"` + Path string `json:"path"` + Token string `json:"token"` +} + +type ShareResp struct { + ID uint `json:"id"` + ShareID string `json:"share_id"` + Name string `json:"name"` + RootPath string `json:"root_path"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` + RemainingAccesses int64 `json:"remaining_accesses"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Enabled bool `json:"enabled"` + ViewCount int64 `json:"view_count"` + DownloadCount int64 `json:"download_count"` + LastAccessAt *time.Time `json:"last_access_at"` + ConsumedAt *time.Time `json:"consumed_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + URL string `json:"url"` +} + +type PublicShareInfoResp struct { + ShareID string `json:"share_id"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` + RemainingAccesses int64 `json:"remaining_accesses"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Authed bool `json:"authed"` + ConsumedAt *time.Time `json:"consumed_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` +} + +type PublicShareObjResp struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Thumb string `json:"thumb"` + Type int `json:"type"` + Path string `json:"path"` + StorageClass string `json:"storage_class,omitempty"` + DownloadURL string `json:"download_url,omitempty"` + PreviewURL string `json:"preview_url,omitempty"` +} + +type PublicShareListResp struct { + Content []PublicShareObjResp `json:"content"` + Total int64 `json:"total"` + Page int `json:"page"` + PerPage int `json:"per_page"` + HasMore bool `json:"has_more"` + PagesTotal int `json:"pages_total"` +} + +type PublicShareGetResp struct { + Item PublicShareObjResp `json:"item"` + Provider string `json:"provider"` +} + +func shareURL(c *gin.Context, shareID string) string { + return fmt.Sprintf("%s/s/%s", common.GetApiUrl(c.Request), shareID) +} + +func toShareResp(c *gin.Context, share *model.Share) ShareResp { + accessLimit := share.EffectiveAccessLimit() + return ShareResp{ + ID: share.ID, + ShareID: share.ShareID, + Name: share.Name, + RootPath: share.RootPath, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + BurnAfterRead: accessLimit == 1, + AccessLimit: accessLimit, + AccessCount: share.AccessCount, + RemainingAccesses: share.RemainingAccesses(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Enabled: share.Enabled, + ViewCount: share.ViewCount, + DownloadCount: share.DownloadCount, + LastAccessAt: share.LastAccessAt, + ConsumedAt: share.ConsumedAt, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, + UpdatedAt: share.UpdatedAt, + URL: shareURL(c, share.ShareID), + } +} + +func normalizeShareName(obj model.Obj, name string) string { + return normalizeOptionalShareName(name, obj.GetName()) +} + +func normalizeOptionalShareName(name, fallback string) string { + if strings.TrimSpace(name) != "" { + return strings.TrimSpace(name) + } + return fallback +} + +func generateShareID() (string, error) { + for range 10 { + shareID := random.String(8) + exists, err := db.ShareIDExists(shareID) + if err != nil { + return "", err + } + if !exists { + return shareID, nil + } + } + return "", fmt.Errorf("failed to generate unique share id") +} + +func sharePasswordHash(password, salt string) string { + return model.HashPwd(model.StaticHash(password), salt) +} + +func validateCustomShareID(shareID string) error { + if shareID == "" { + return nil + } + if !shareIDPattern.MatchString(shareID) { + return errShareIDInvalid + } + return nil +} + +func resolveRequestedShareID(rawShareID, fallback string, excludeID uint) (string, error) { + shareID := strings.TrimSpace(rawShareID) + if shareID == "" { + if fallback != "" { + return fallback, nil + } + return generateShareID() + } + if err := validateCustomShareID(shareID); err != nil { + return "", err + } + if excludeID == 0 { + exists, err := db.ShareIDExists(shareID) + if err != nil { + return "", fmt.Errorf("check share id availability: %w", err) + } + if exists { + return "", errShareIDExists + } + return shareID, nil + } + exists, err := db.ShareIDExistsExceptID(shareID, excludeID) + if err != nil { + return "", fmt.Errorf("check share id availability: %w", err) + } + if exists { + return "", errShareIDExists + } + return shareID, nil +} + +func normalizeShareAccessLimit(accessLimit int64, burnAfterRead *bool) (int64, bool, error) { + if accessLimit < 0 { + return 0, false, fmt.Errorf("access_limit must be 0 or greater") + } + if accessLimit == 0 && burnAfterRead != nil && *burnAfterRead { + accessLimit = 1 + } + return accessLimit, accessLimit == 1, nil +} + +func parseShareExpireAt(raw string) (*time.Time, error) { + value := strings.TrimSpace(raw) + if value == "" { + return nil, nil + } + if parsed, err := time.Parse(time.RFC3339, value); err == nil { + return &parsed, nil + } + layouts := []string{ + "2006-01-02T15:04:05", + "2006-01-02T15:04", + "2006-01-02 15:04:05", + } + for _, layout := range layouts { + if parsed, err := time.ParseInLocation(layout, value, time.Local); err == nil { + return &parsed, nil + } + } + return nil, fmt.Errorf("invalid expire_at") +} + +func resolveShareExpireAt(expireAt string, expireHours int64) (*time.Time, error) { + if strings.TrimSpace(expireAt) != "" { + return parseShareExpireAt(expireAt) + } + if expireHours < 0 { + return nil, fmt.Errorf("expire_hours must be 0 or greater") + } + if expireHours == 0 { + return nil, nil + } + expires := time.Now().Add(time.Duration(expireHours) * time.Hour) + return &expires, nil +} + +func sharePasswordMatched(share *model.Share, password string) bool { + if !share.HasPassword() { + return true + } + hash := sharePasswordHash(password, share.PasswordSalt) + return subtle.ConstantTimeCompare([]byte(hash), []byte(share.PasswordHash)) == 1 +} + +func getShareAccessToken(c *gin.Context, fallback string) string { + if fallback != "" { + return fallback + } + if token := c.Query("auth"); token != "" { + return token + } + return c.GetHeader("X-Share-Token") +} + +func ensureShareAvailable(c *gin.Context, share *model.Share) bool { + now := time.Now() + if share.IsConsumed() { + common.ErrorStrResp(c, "share has been consumed", 410) + return false + } + if !share.Enabled { + common.ErrorStrResp(c, "share is disabled", 404) + return false + } + if share.IsExpired(now) { + common.ErrorStrResp(c, "share is expired", 410) + return false + } + return true +} + +func recordShareAccess(share *model.Share) error { + updated, err := db.RecordShareAccess(share.ShareID) + if err != nil { + return err + } + if updated != nil { + *share = *updated + } + return nil +} + +func ensureShareAccess(c *gin.Context, share *model.Share, token string) bool { + if !share.HasPassword() { + return true + } + if token == "" { + common.ErrorStrResp(c, "share password required", 401) + return false + } + if err := shareauth.VerifyAccess(share, token); err != nil { + common.ErrorResp(c, err, 401) + return false + } + return true +} + +func shouldTrackShareContentAccess(c *gin.Context) bool { + return c.Request.Method != http.MethodHead +} + +func resolveShareTarget(share *model.Share, rawRelPath string) (string, string, error) { + cleanRelPath := utils.FixAndCleanPath(rawRelPath) + if !share.IsDir && cleanRelPath != "/" { + return "", "", fmt.Errorf("file share does not support nested path") + } + if cleanRelPath == "/" { + return share.RootPath, "/", nil + } + target := utils.FixAndCleanPath(stdpath.Join(share.RootPath, cleanRelPath)) + if !utils.IsSubPath(share.RootPath, target) { + return "", "", fmt.Errorf("share path out of range") + } + return target, cleanRelPath, nil +} + +func resolveShareWildcardTarget(share *model.Share, rawPath string) (string, string, error) { + path, err := url.PathUnescape(rawPath) + if err != nil { + return "", "", err + } + return resolveShareTarget(share, strings.TrimPrefix(path, "/")) +} + +func buildPublicShareAssetURL(c *gin.Context, prefix, shareID, relPath, token string, preview bool) string { + base := common.GetApiUrl(c.Request) + prefix + shareID + cleanPath := utils.FixAndCleanPath(relPath) + if cleanPath != "/" { + base += utils.EncodePath(cleanPath, true) + } + query := url.Values{} + if token != "" { + query.Set("auth", token) + } + if preview { + query.Set("type", "preview") + } + if encoded := query.Encode(); encoded != "" { + base += "?" + encoded + } + return base +} + +func buildPublicSharePreviewURL(c *gin.Context, obj model.Obj, targetPath, shareID, relPath, token string) string { + prefix := "/sd/" + storage, err := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if err == nil && canProxy(storage, obj.GetName()) { + prefix = "/sp/" + } + return buildPublicShareAssetURL(c, prefix, shareID, relPath, token, true) +} + +func toPublicShareObjResp(c *gin.Context, share *model.Share, obj model.Obj, targetPath, relPath, token string) PublicShareObjResp { + thumb, _ := model.GetThumb(obj) + storageClass, _ := model.GetStorageClass(obj) + resp := PublicShareObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + Thumb: thumb, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + Path: relPath, + StorageClass: storageClass, + } + if !obj.IsDir() && share.AllowDownload { + resp.DownloadURL = buildPublicShareAssetURL(c, "/sd/", share.ShareID, relPath, token, false) + } + if !obj.IsDir() && share.AllowPreview { + resp.PreviewURL = buildPublicSharePreviewURL(c, obj, targetPath, share.ShareID, relPath, token) + } + return resp +} + +func CreateShare(c *gin.Context) { + var req CreateShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CanReadPathByRole(user, reqPath) { + common.ErrorStrResp(c, "you have no permission", 403) + return + } + obj, err := fs.Get(c, reqPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + shareID, err := resolveRequestedShareID(req.ShareID, "", 0) + if err != nil { + if errors.Is(err, errShareIDInvalid) || errors.Is(err, errShareIDExists) { + common.ErrorResp(c, err, 400) + return + } + common.ErrorResp(c, err, 500, true) + return + } + allowPreview := true + if req.AllowPreview != nil { + allowPreview = *req.AllowPreview + } + allowDownload := true + if req.AllowDownload != nil { + allowDownload = *req.AllowDownload + } + accessLimit, burnAfterRead, err := normalizeShareAccessLimit(req.AccessLimit, req.BurnAfterRead) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + expiresAt, err := resolveShareExpireAt(req.ExpireAt, req.ExpireHours) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + share := &model.Share{ + ShareID: shareID, + CreatorID: user.ID, + Name: normalizeShareName(obj, req.Name), + RootPath: reqPath, + IsDir: obj.IsDir(), + BurnAfterRead: burnAfterRead, + AccessLimit: accessLimit, + AllowPreview: allowPreview, + AllowDownload: allowDownload, + Enabled: true, + ExpiresAt: expiresAt, + } + if req.Password != "" { + share.PasswordSalt = random.String(16) + share.PasswordHash = sharePasswordHash(req.Password, share.PasswordSalt) + } + if err := db.CreateShare(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, toShareResp(c, share)) +} + +func UpdateShare(c *gin.Context) { + var req UpdateShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + share, err := db.GetShareByCreatorAndShareID(user.ID, req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + + shareID, err := resolveRequestedShareID(req.NewShareID, share.ShareID, share.ID) + if err != nil { + if errors.Is(err, errShareIDInvalid) || errors.Is(err, errShareIDExists) { + common.ErrorResp(c, err, 400) + return + } + common.ErrorResp(c, err, 500, true) + return + } + + allowPreview := share.AllowPreview + if req.AllowPreview != nil { + allowPreview = *req.AllowPreview + } + allowDownload := share.AllowDownload + if req.AllowDownload != nil { + allowDownload = *req.AllowDownload + } + + accessLimit := share.EffectiveAccessLimit() + burnAfterRead := accessLimit == 1 + if req.AccessLimit != nil { + accessLimit, burnAfterRead, err = normalizeShareAccessLimit(*req.AccessLimit, nil) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + } + + expiresAt := share.ExpiresAt + if req.ExpireAt != nil { + expiresAt, err = parseShareExpireAt(*req.ExpireAt) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + } + + share.ShareID = shareID + share.Name = normalizeOptionalShareName(req.Name, share.Name) + share.BurnAfterRead = burnAfterRead + share.AccessLimit = accessLimit + share.AllowPreview = allowPreview + share.AllowDownload = allowDownload + share.ExpiresAt = expiresAt + if req.Password != "" { + share.PasswordSalt = random.String(16) + share.PasswordHash = sharePasswordHash(req.Password, share.PasswordSalt) + } + if share.Enabled && accessLimit > 0 && share.AccessCount >= accessLimit { + now := time.Now() + share.Enabled = false + if share.ConsumedAt == nil { + share.ConsumedAt = &now + } + } + if err := db.UpdateShare(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, toShareResp(c, share)) +} + +func ListShares(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + user := c.MustGet("user").(*model.User) + shares, total, err := db.GetSharesByCreator(user.ID, req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + resp := make([]ShareResp, 0, len(shares)) + for i := range shares { + resp = append(resp, toShareResp(c, &shares[i])) + } + common.SuccessResp(c, common.PageResp{ + Content: resp, + Total: total, + }) +} + +func DisableShare(c *gin.Context) { + var req ShareDeleteReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if _, err := db.GetShareByCreatorAndShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 404) + return + } + if err := db.DisableShareByShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func DeleteShare(c *gin.Context) { + var req ShareDeleteReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if err := db.DeleteShareByShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/share_page.go b/server/handles/share_page.go new file mode 100644 index 00000000000..98cc62d8daf --- /dev/null +++ b/server/handles/share_page.go @@ -0,0 +1,16 @@ +package handles + +import ( + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/gin-gonic/gin" +) + +func GetSharePage(c *gin.Context) { + c.Header("Content-Type", "text/html") + c.Status(200) + _, _ = c.Writer.WriteString(strings.Replace(conf.IndexHtml, "", "", 1)) + c.Writer.Flush() + c.Writer.WriteHeaderNow() +} diff --git a/server/handles/share_public.go b/server/handles/share_public.go new file mode 100644 index 00000000000..f88d5b338c5 --- /dev/null +++ b/server/handles/share_public.go @@ -0,0 +1,275 @@ +package handles + +import ( + stdpath "path" + "time" + + "github.com/alist-org/alist/v3/internal/db" + shareauth "github.com/alist-org/alist/v3/internal/share" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +func GetPublicShareInfo(c *gin.Context) { + shareID := c.Query("share_id") + if shareID == "" { + common.ErrorStrResp(c, "share_id is required", 400) + return + } + share, err := db.GetShareByShareID(shareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, "") + authed := !share.HasPassword() + if share.HasPassword() && token != "" { + authed = shareauth.VerifyAccess(share, token) == nil + } + if authed { + _ = db.TouchShareView(share.ShareID) + } + common.SuccessResp(c, PublicShareInfoResp{ + ShareID: share.ShareID, + Name: share.Name, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + BurnAfterRead: share.EffectiveAccessLimit() == 1, + AccessLimit: share.EffectiveAccessLimit(), + AccessCount: share.AccessCount, + RemainingAccesses: share.RemainingAccesses(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Authed: authed, + ConsumedAt: share.ConsumedAt, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, + }) +} + +func AuthPublicShare(c *gin.Context) { + var req ShareAuthReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !sharePasswordMatched(share, req.Password) { + common.ErrorStrResp(c, "password is incorrect", 403) + return + } + token := "" + if share.HasPassword() { + ttl := shareAccessTokenLifetime + if share.ExpiresAt != nil { + remaining := time.Until(*share.ExpiresAt) + if remaining <= 0 { + common.ErrorStrResp(c, "share is expired", 410) + return + } + if remaining < ttl { + ttl = remaining + } + } + token = shareauth.SignAccess(share, ttl) + } + _ = db.TouchShareView(share.ShareID) + common.SuccessResp(c, gin.H{"token": token}) +} + +func ListPublicShare(c *gin.Context) { + var req PublicShareListReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Page, req.PerPage = normalizeListPage(req.Page, req.PerPage) + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, req.Token) + if !ensureShareAccess(c, share, token) { + return + } + targetPath, relPath, err := resolveShareTarget(share, req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !obj.IsDir() { + common.ErrorStrResp(c, "path is not a directory", 400) + return + } + objs, err := fs.List(c, targetPath, &fs.ListArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + total, pageObjs := pagination(objs, &req.PageReq) + content := make([]PublicShareObjResp, 0, len(pageObjs)) + for _, item := range pageObjs { + itemRelPath := relPath + if itemRelPath == "/" { + itemRelPath = stdpath.Join("/", item.GetName()) + } else { + itemRelPath = stdpath.Join(relPath, item.GetName()) + } + itemTargetPath, _, err := resolveShareTarget(share, itemRelPath) + if err != nil { + continue + } + content = append(content, toPublicShareObjResp(c, share, item, itemTargetPath, itemRelPath, token)) + } + common.SuccessResp(c, PublicShareListResp{ + Content: content, + Total: int64(total), + Page: req.Page, + PerPage: req.PerPage, + HasMore: req.PerPage != AllPerPage && req.Page*req.PerPage < total, + PagesTotal: calcPagesTotal(total, req.PerPage), + }) +} + +func GetPublicShare(c *gin.Context) { + var req PublicShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, req.Token) + if !ensureShareAccess(c, share, token) { + return + } + targetPath, relPath, err := resolveShareTarget(share, req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + provider := "unknown" + storage, storageErr := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if storageErr == nil { + provider = storage.GetStorage().Driver + } + common.SuccessResp(c, PublicShareGetResp{ + Item: toPublicShareObjResp(c, share, obj, targetPath, relPath, token), + Provider: provider, + }) +} + +func ShareDown(c *gin.Context) { + share, err := db.GetShareByShareID(c.Param("share_id")) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !share.AllowDownload { + common.ErrorStrResp(c, "download is not allowed", 403) + return + } + token := getShareAccessToken(c, "") + if !ensureShareAccess(c, share, token) { + return + } + targetPath, _, err := resolveShareWildcardTarget(share, c.Param("path")) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if obj.IsDir() { + common.ErrorStrResp(c, "directory download is not supported", 400) + return + } + if shouldTrackShareContentAccess(c) { + _ = db.TouchShareDownload(share.ShareID) + if err := recordShareAccess(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("path", targetPath) + Down(c) +} + +func ShareProxy(c *gin.Context) { + share, err := db.GetShareByShareID(c.Param("share_id")) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !share.AllowPreview { + common.ErrorStrResp(c, "preview is not allowed", 403) + return + } + token := getShareAccessToken(c, "") + if !ensureShareAccess(c, share, token) { + return + } + targetPath, _, err := resolveShareWildcardTarget(share, c.Param("path")) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if obj.IsDir() { + common.ErrorStrResp(c, "directory preview is not supported", 400) + return + } + if shouldTrackShareContentAccess(c) { + if err := recordShareAccess(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("path", targetPath) + Proxy(c) +} diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index 62bd4aaa2bf..779cc13239b 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "errors" "fmt" + "github.com/alist-org/alist/v3/internal/op" "net/http" "net/url" "path" @@ -154,7 +155,7 @@ func autoRegister(username, userID string, err error) (*model.User, error) { Password: random.String(16), Permission: int32(setting.GetInt(conf.SSODefaultPermission, 0)), BasePath: setting.GetStr(conf.SSODefaultDir), - Role: 0, + Role: model.Roles{op.GetDefaultRoleID()}, Disabled: false, SsoID: userID, } @@ -256,6 +257,7 @@ func OIDCLoginCallback(c *gin.Context) { user, err = autoRegister(userID, userID, err) if err != nil { common.ErrorResp(c, err, 400) + return } } token, err := common.GenerateToken(user) diff --git a/server/handles/task.go b/server/handles/task.go index af7974a9c29..a4dbce0f1fa 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -18,7 +18,7 @@ type TaskInfo struct { ID string `json:"id"` Name string `json:"name"` Creator string `json:"creator"` - CreatorRole int `json:"creator_role"` + CreatorRole model.Roles `json:"creator_role"` State tache.State `json:"state"` Status string `json:"status"` Progress float64 `json:"progress"` @@ -39,7 +39,7 @@ func getTaskInfo[T task.TaskExtensionInfo](task T) TaskInfo { progress = 100 } creatorName := "" - creatorRole := -1 + var creatorRole model.Roles if task.GetCreator() != nil { creatorName = task.GetCreator().Username creatorRole = task.GetCreator().Role @@ -220,6 +220,7 @@ func SetupTaskRoute(g *gin.RouterGroup) { taskRoute(g.Group("/copy"), fs.CopyTaskManager) taskRoute(g.Group("/offline_download"), tool.DownloadTaskManager) taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager) + taskRoute(g.Group("/s3_transition"), fs.S3TransitionTaskManager) taskRoute(g.Group("/decompress"), fs.ArchiveDownloadTaskManager) taskRoute(g.Group("/decompress_upload"), fs.ArchiveContentUploadTaskManager) } diff --git a/server/handles/user.go b/server/handles/user.go index 4d404a4c652..ac3a06e8180 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -3,6 +3,8 @@ package handles import ( "strconv" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/server/common" @@ -35,6 +37,9 @@ func CreateUser(c *gin.Context) { common.ErrorResp(c, err, 400) return } + if len(req.Role) == 0 { + req.Role = model.Roles{op.GetDefaultRoleID()} + } if req.IsAdmin() || req.IsGuest() { common.ErrorStrResp(c, "admin or guest user can not be created", 400, true) return @@ -60,10 +65,18 @@ func UpdateUser(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if user.Role != req.Role { - common.ErrorStrResp(c, "role can not be changed", 400) - return + + if user.Username == "admin" { + if !utils.SliceEqual(user.Role, req.Role) { + common.ErrorStrResp(c, "cannot change role of admin user", 403) + return + } + //if user.Username != req.Username { + // common.ErrorStrResp(c, "cannot change username of admin user", 403) + // return + //} } + if req.Password == "" { req.PwdHash = user.PwdHash req.Salt = user.Salt @@ -74,10 +87,25 @@ func UpdateUser(c *gin.Context) { if req.OtpSecret == "" { req.OtpSecret = user.OtpSecret } - if req.Disabled && req.IsAdmin() { - common.ErrorStrResp(c, "admin user can not be disabled", 400) - return + if req.Disabled && user.IsAdmin() { + count, err := op.CountEnabledAdminsExcluding(user.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if count == 0 { + common.ErrorStrResp(c, "at least one enabled admin must be kept", 400) + return + } + } + + if !utils.SliceEqual(user.Role, req.Role) { + if req.IsAdmin() || req.IsGuest() { + common.ErrorStrResp(c, "cannot assign admin or guest role to user", 400, true) + return + } } + if err := op.UpdateUser(&req); err != nil { common.ErrorResp(c, err, 500) } else { diff --git a/server/mcp/auth.go b/server/mcp/auth.go new file mode 100644 index 00000000000..6d1e670d14a --- /dev/null +++ b/server/mcp/auth.go @@ -0,0 +1,182 @@ +package mcp + +import ( + "context" + "crypto/subtle" + "fmt" + "net/http" + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/server/common" + log "github.com/sirupsen/logrus" +) + +type ctxKey string + +const userKey ctxKey = "user" + +// HTTPContextFunc extracts JWT/admin token from HTTP request and injects user into context. +// Used as WithHTTPContextFunc callback for Streamable HTTP transport. +func HTTPContextFunc(ctx context.Context, r *http.Request) context.Context { + token := r.Header.Get("Authorization") + if token == "" { + token = r.URL.Query().Get("token") + } + + user, err := authenticateToken(token) + if err != nil { + log.Debugf("MCP auth failed: %v", err) + return ctx + } + + return context.WithValue(ctx, userKey, user) +} + +func authenticateToken(token string) (*model.User, error) { + // Check admin static token + if token != "" && subtle.ConstantTimeCompare([]byte(token), []byte(setting.GetStr(conf.Token))) == 1 { + admin, err := op.GetAdmin() + if err != nil { + return nil, fmt.Errorf("failed to get admin: %w", err) + } + if err := loadRoles(admin); err != nil { + return nil, err + } + return admin, nil + } + + // No token: guest + if token == "" { + guest, err := op.GetGuest() + if err != nil { + return nil, fmt.Errorf("failed to get guest: %w", err) + } + if guest.Disabled { + return nil, fmt.Errorf("guest user is disabled") + } + if err := loadRoles(guest); err != nil { + return nil, err + } + return guest, nil + } + + // JWT token + claims, err := common.ParseToken(token) + if err != nil { + return nil, fmt.Errorf("invalid token: %w", err) + } + + user, err := op.GetUserByName(claims.Username) + if err != nil { + return nil, fmt.Errorf("user not found: %w", err) + } + + if claims.PwdTS != user.PwdTS { + return nil, fmt.Errorf("password has been changed") + } + if user.Disabled { + return nil, fmt.Errorf("user is disabled") + } + + if err := loadRoles(user); err != nil { + return nil, err + } + return user, nil +} + +func loadRoles(user *model.User) error { + if len(user.Role) > 0 { + roles, err := op.GetRolesByUserID(user.ID) + if err != nil { + return fmt.Errorf("failed to load roles: %w", err) + } + user.RolesDetail = roles + } + return nil +} + +// resolveUser extracts the authenticated user from context. +func resolveUser(ctx context.Context) (*model.User, error) { + user, ok := ctx.Value(userKey).(*model.User) + if !ok || user == nil { + return nil, fmt.Errorf("authentication required") + } + return user, nil +} + +// buildFsContext resolves path and sets meta in context for fs operations. +func buildFsContext(ctx context.Context, user *model.User, path string) (context.Context, string, error) { + reqPath, err := user.JoinPath(path) + if err != nil { + return ctx, "", err + } + meta, _ := op.GetNearestMeta(reqPath) + ctx = context.WithValue(ctx, "meta", meta) + ctx = context.WithValue(ctx, "user", user) + return ctx, reqPath, nil +} + +// checkAccess checks if user can access the path (read). +func checkAccess(user *model.User, reqPath string) error { + meta, _ := op.GetNearestMeta(reqPath) + if !common.CanAccessWithRoles(user, meta, reqPath, "") { + return fmt.Errorf("permission denied") + } + perm := common.MergeRolePermissions(user, reqPath) + if !user.IsAdmin() && !common.HasPermission(perm, common.PermMCPAccess) { + return fmt.Errorf("MCP access not permitted") + } + return nil +} + +// checkManage checks if user can perform write operations via MCP. +func checkManage(user *model.User, reqPath string, permBit uint) error { + if err := checkAccess(user, reqPath); err != nil { + return err + } + perm := common.MergeRolePermissions(user, reqPath) + if !user.IsAdmin() && !common.HasPermission(perm, common.PermMCPManage) { + return fmt.Errorf("MCP manage not permitted") + } + if !user.IsAdmin() && !common.HasPermission(perm, permBit) { + return fmt.Errorf("permission denied for this operation") + } + return nil +} + +// UserContextFunc returns an HTTPContextFunc that injects a specific user (for STDIO mode). +func userContextMiddleware(user *model.User) func(ctx context.Context) context.Context { + return func(ctx context.Context) context.Context { + return context.WithValue(ctx, userKey, user) + } +} + +// resolveUserForStdio resolves a user by username for STDIO mode. +func resolveUserForStdio(username string) (*model.User, error) { + username = strings.TrimSpace(username) + if username == "" || username == "admin" { + admin, err := op.GetAdmin() + if err != nil { + return nil, fmt.Errorf("failed to get admin user: %w", err) + } + if err := loadRoles(admin); err != nil { + return nil, err + } + return admin, nil + } + user, err := op.GetUserByName(username) + if err != nil { + return nil, fmt.Errorf("user %q not found: %w", username, err) + } + if user.Disabled { + return nil, fmt.Errorf("user %q is disabled", username) + } + if err := loadRoles(user); err != nil { + return nil, err + } + return user, nil +} diff --git a/server/mcp/convert.go b/server/mcp/convert.go new file mode 100644 index 00000000000..7eede8cbc87 --- /dev/null +++ b/server/mcp/convert.go @@ -0,0 +1,56 @@ +package mcp + +import ( + "encoding/json" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/mark3labs/mcp-go/mcp" +) + +type objJSON struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + HashInfo map[string]string `json:"hash_info,omitempty"` +} + +func objToJSON(obj model.Obj) objJSON { + j := objJSON{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + } + hi := obj.GetHash() + if hm := hashInfoToMap(hi); len(hm) > 0 { + j.HashInfo = hm + } + return j +} + +func hashInfoToMap(hi utils.HashInfo) map[string]string { + m := make(map[string]string) + for ht, v := range hi.All() { + if v != "" { + m[ht.Name] = v + } + } + return m +} + +func jsonResult(v interface{}) (*mcp.CallToolResult, error) { + data, err := json.Marshal(v) + if err != nil { + return nil, err + } + return mcp.NewToolResultText(string(data)), nil +} + +func textResult(msg string) (*mcp.CallToolResult, error) { + return mcp.NewToolResultText(msg), nil +} diff --git a/server/mcp/errors.go b/server/mcp/errors.go new file mode 100644 index 00000000000..c107633739f --- /dev/null +++ b/server/mcp/errors.go @@ -0,0 +1,40 @@ +package mcp + +import ( + "fmt" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/mark3labs/mcp-go/mcp" + pkgerr "github.com/pkg/errors" +) + +func toolError(msg string) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(msg), nil +} + +func toolErrorf(format string, args ...interface{}) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(fmt.Sprintf(format, args...)), nil +} + +func wrapError(err error) (*mcp.CallToolResult, error) { + if err == nil { + return nil, nil + } + cause := pkgerr.Cause(err) + switch { + case errs.IsObjectNotFound(err) || errs.IsNotFoundError(err): + return toolErrorf("not found: %s", err.Error()) + case cause == errs.PermissionDenied: + return toolError("permission denied") + case cause == errs.NotImplement: + return toolError("not supported by storage driver") + case cause == errs.NotSupport: + return toolError("operation not supported") + case cause == errs.UploadNotSupported: + return toolError("upload not supported by storage") + case cause == errs.MoveBetweenTwoStorages: + return toolError("can't move between two storages, use copy instead") + default: + return toolErrorf("error: %s", err.Error()) + } +} diff --git a/server/mcp/server.go b/server/mcp/server.go new file mode 100644 index 00000000000..b45a0113eb3 --- /dev/null +++ b/server/mcp/server.go @@ -0,0 +1,64 @@ +package mcp + +import ( + "context" + "net/http" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +// NewServer creates an MCP server with all alist tools registered. +func NewServer() *mcpserver.MCPServer { + s := mcpserver.NewMCPServer( + "alist", + conf.Version, + mcpserver.WithToolCapabilities(false), + mcpserver.WithRecovery(), + ) + registerReadTools(s) + registerManageTools(s) + registerUploadTools(s) + return s +} + +// NewHTTPHandler creates a Streamable HTTP handler for the MCP server. +func NewHTTPHandler() http.Handler { + s := NewServer() + return mcpserver.NewStreamableHTTPServer(s, + mcpserver.WithHTTPContextFunc(HTTPContextFunc), + ) +} + +// NewStdioServer creates an MCP server configured for STDIO mode with a fixed user. +func NewStdioServer(username string) (*mcpserver.MCPServer, *model.User, error) { + user, err := resolveUserForStdio(username) + if err != nil { + return nil, nil, err + } + s := NewServer() + return s, user, nil +} + +// ServeStdio starts the MCP server in STDIO mode. +func ServeStdio(username string) error { + s, user, err := NewStdioServer(username) + if err != nil { + return err + } + ctxFunc := userContextMiddleware(user) + return mcpserver.ServeStdio(s, mcpserver.WithStdioContextFunc(ctxFunc)) +} + +// toolHandlerWithAuth wraps a tool handler to require authentication. +func toolHandlerWithAuth(fn func(ctx context.Context, user *model.User, request mcp.CallToolRequest) (*mcp.CallToolResult, error)) mcpserver.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + user, err := resolveUser(ctx) + if err != nil { + return toolError("authentication required") + } + return fn(ctx, user, request) + } +} diff --git a/server/mcp/tools_manage.go b/server/mcp/tools_manage.go new file mode 100644 index 00000000000..d287bb6bed0 --- /dev/null +++ b/server/mcp/tools_manage.go @@ -0,0 +1,228 @@ +package mcp + +import ( + "context" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +func registerManageTools(s *mcpserver.MCPServer) { + // fs_mkdir + s.AddTool(mcp.NewTool("fs_mkdir", + mcp.WithDescription("Create a new directory"), + mcp.WithString("path", mcp.Required(), mcp.Description("Full path of directory to create")), + ), toolHandlerWithAuth(handleFsMkdir)) + + // fs_rename + s.AddTool(mcp.NewTool("fs_rename", + mcp.WithDescription("Rename a file or directory"), + mcp.WithString("path", mcp.Required(), mcp.Description("Current path of the file/directory")), + mcp.WithString("name", mcp.Required(), mcp.Description("New name (filename only, not a path)")), + ), toolHandlerWithAuth(handleFsRename)) + + // fs_move + s.AddTool(mcp.NewTool("fs_move", + mcp.WithDescription("Move files/directories to another location"), + mcp.WithString("src_dir", mcp.Required(), mcp.Description("Source directory")), + mcp.WithString("dst_dir", mcp.Required(), mcp.Description("Destination directory")), + mcp.WithArray("names", mcp.Description("Names of files/directories to move")), + ), toolHandlerWithAuth(handleFsMove)) + + // fs_copy + s.AddTool(mcp.NewTool("fs_copy", + mcp.WithDescription("Copy files/directories to another location"), + mcp.WithString("src_dir", mcp.Required(), mcp.Description("Source directory")), + mcp.WithString("dst_dir", mcp.Required(), mcp.Description("Destination directory")), + mcp.WithArray("names", mcp.Description("Names of files/directories to copy")), + ), toolHandlerWithAuth(handleFsCopy)) + + // fs_remove + s.AddTool(mcp.NewTool("fs_remove", + mcp.WithDescription("Delete files/directories"), + mcp.WithString("dir", mcp.Required(), mcp.Description("Directory containing items to remove")), + mcp.WithArray("names", mcp.Description("Names of files/directories to remove")), + ), toolHandlerWithAuth(handleFsRemove)) +} + +func handleFsMkdir(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqPath, common.PermWrite); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.MakeDir(ctx, reqPath); err != nil { + return wrapError(err) + } + return textResult("directory created successfully") +} + +func handleFsRename(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + name, err := req.RequireString("name") + if err != nil { + return toolError("name is required") + } + if err := utils.ValidateNameComponent(name); err != nil { + return toolErrorf("invalid name: %s", err.Error()) + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqPath, common.PermRename); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.Rename(ctx, reqPath, name); err != nil { + return wrapError(err) + } + return textResult("renamed successfully") +} + +func handleFsMove(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + srcDirStr, err := req.RequireString("src_dir") + if err != nil { + return toolError("src_dir is required") + } + dstDirStr, err := req.RequireString("dst_dir") + if err != nil { + return toolError("dst_dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + srcDir, err := user.JoinPath(srcDirStr) + if err != nil { + return wrapError(err) + } + dstDir, err := user.JoinPath(dstDirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, srcDir, common.PermMove); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for i, name := range names { + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if err := fs.Move(ctx, srcPath, dstDir, len(names) > i+1); err != nil { + return wrapError(err) + } + } + return textResult("moved successfully") +} + +func handleFsCopy(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + srcDirStr, err := req.RequireString("src_dir") + if err != nil { + return toolError("src_dir is required") + } + dstDirStr, err := req.RequireString("dst_dir") + if err != nil { + return toolError("dst_dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + srcDir, err := user.JoinPath(srcDirStr) + if err != nil { + return wrapError(err) + } + dstDir, err := user.JoinPath(dstDirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, srcDir, common.PermCopy); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for i, name := range names { + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if _, err := fs.Copy(ctx, srcPath, dstDir, len(names) > i+1); err != nil { + return wrapError(err) + } + } + return textResult("copied successfully") +} + +func handleFsRemove(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + dirStr, err := req.RequireString("dir") + if err != nil { + return toolError("dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + reqDir, err := user.JoinPath(dirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqDir, common.PermRemove); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for _, name := range names { + removePath, err := utils.JoinUnderBase(reqDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if err := fs.Remove(ctx, removePath); err != nil { + return wrapError(err) + } + } + return textResult("removed successfully") +} + +// getStringArray extracts a string array from tool request arguments. +func getStringArray(req mcp.CallToolRequest, name string) []string { + args := req.GetArguments() + val, ok := args[name] + if !ok { + return nil + } + arr, ok := val.([]interface{}) + if !ok { + return nil + } + result := make([]string, 0, len(arr)) + for _, v := range arr { + if s, ok := v.(string); ok { + result = append(result, s) + } + } + return result +} diff --git a/server/mcp/tools_read.go b/server/mcp/tools_read.go new file mode 100644 index 00000000000..d8af570b233 --- /dev/null +++ b/server/mcp/tools_read.go @@ -0,0 +1,207 @@ +package mcp + +import ( + "context" + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/search" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" + "github.com/pkg/errors" +) + +func registerReadTools(s *mcpserver.MCPServer) { + // fs_list + s.AddTool(mcp.NewTool("fs_list", + mcp.WithDescription("List files and directories at the given path"), + mcp.WithString("path", mcp.Required(), mcp.Description("Directory path to list")), + mcp.WithNumber("page", mcp.Description("Page number (default: 1)")), + mcp.WithNumber("per_page", mcp.Description("Items per page (default: 30, max: 500)")), + mcp.WithBoolean("refresh", mcp.Description("Force refresh from storage (default: false)")), + ), toolHandlerWithAuth(handleFsList)) + + // fs_get + s.AddTool(mcp.NewTool("fs_get", + mcp.WithDescription("Get file or directory metadata"), + mcp.WithString("path", mcp.Required(), mcp.Description("Path to file or directory")), + ), toolHandlerWithAuth(handleFsGet)) + + // fs_search + s.AddTool(mcp.NewTool("fs_search", + mcp.WithDescription("Search for files by keywords"), + mcp.WithString("path", mcp.Required(), mcp.Description("Parent directory to search within")), + mcp.WithString("keywords", mcp.Required(), mcp.Description("Search keywords")), + mcp.WithNumber("scope", mcp.Description("0=all, 1=dir only, 2=file only (default: 0)")), + mcp.WithNumber("page", mcp.Description("Page number (default: 1)")), + mcp.WithNumber("per_page", mcp.Description("Items per page (default: 20)")), + ), toolHandlerWithAuth(handleFsSearch)) + + // fs_download_url + s.AddTool(mcp.NewTool("fs_download_url", + mcp.WithDescription("Get download URL for a file"), + mcp.WithString("path", mcp.Required(), mcp.Description("Path to the file")), + ), toolHandlerWithAuth(handleFsDownloadURL)) +} + +func handleFsList(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + page := intParam(req, "page", 1) + perPage := intParam(req, "per_page", 30) + if perPage > 500 { + perPage = 500 + } + refresh := req.GetBool("refresh", false) + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + objs, err := fs.List(ctx, reqPath, &fs.ListArgs{Refresh: refresh}) + if err != nil { + return wrapError(err) + } + + // Paginate + total := len(objs) + start := (page - 1) * perPage + if start > total { + start = total + } + end := start + perPage + if end > total { + end = total + } + pageObjs := objs[start:end] + + items := make([]objJSON, 0, len(pageObjs)) + for _, obj := range pageObjs { + items = append(items, objToJSON(obj)) + } + + return jsonResult(map[string]interface{}{ + "content": items, + "total": total, + "page": page, + "per_page": perPage, + }) +} + +func handleFsGet(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + if err != nil { + return wrapError(err) + } + + return jsonResult(objToJSON(obj)) +} + +func handleFsSearch(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + keywords, err := req.RequireString("keywords") + if err != nil { + return toolError("keywords is required") + } + scope := intParam(req, "scope", 0) + page := intParam(req, "page", 1) + perPage := intParam(req, "per_page", 20) + + parent, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + + searchReq := model.SearchReq{ + Parent: parent, + Keywords: keywords, + Scope: scope, + PageReq: model.PageReq{Page: page, PerPage: perPage}, + } + if err := searchReq.Validate(); err != nil { + return toolErrorf("invalid search request: %s", err.Error()) + } + + nodes, total, err := search.Search(ctx, searchReq) + if err != nil { + return wrapError(err) + } + + // Filter by permission + filtered := make([]model.SearchNode, 0, len(nodes)) + for _, node := range nodes { + if !strings.HasPrefix(node.Parent, user.BasePath) { + continue + } + meta, err := op.GetNearestMeta(node.Parent) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + continue + } + if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), "") { + continue + } + filtered = append(filtered, node) + } + + return jsonResult(map[string]interface{}{ + "content": filtered, + "total": total, + }) +} + +func handleFsDownloadURL(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{}) + if err != nil { + return wrapError(err) + } + + return jsonResult(map[string]interface{}{ + "raw_url": link.URL, + }) +} + +// intParam extracts an integer parameter with a default value. +func intParam(req mcp.CallToolRequest, name string, defaultVal int) int { + v := req.GetFloat(name, float64(defaultVal)) + return int(v) +} diff --git a/server/mcp/tools_upload.go b/server/mcp/tools_upload.go new file mode 100644 index 00000000000..c84eb9623a9 --- /dev/null +++ b/server/mcp/tools_upload.go @@ -0,0 +1,131 @@ +package mcp + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + stdpath "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +func registerUploadTools(s *mcpserver.MCPServer) { + s.AddTool(mcp.NewTool("fs_upload", + mcp.WithDescription("Upload a local file to alist. Automatically uses direct internal upload (local deployment) or HTTP API upload (remote deployment)."), + mcp.WithString("path", mcp.Required(), mcp.Description("Destination path in alist including filename")), + mcp.WithString("local_path", mcp.Required(), mcp.Description("Absolute local file path to upload")), + ), toolHandlerWithAuth(handleFsUpload)) +} + +func handleFsUpload(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + localPath, err := req.RequireString("local_path") + if err != nil { + return toolError("local_path is required") + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + dir := stdpath.Dir(reqPath) + + if err := checkManage(user, dir, common.PermWrite); err != nil { + return toolError(err.Error()) + } + + if strings.Contains(conf.Conf.SiteURL, "://") { + return uploadViaHTTP(reqPath, localPath) + } + return uploadDirectly(ctx, user, reqPath, localPath) +} + +func uploadDirectly(ctx context.Context, user *model.User, reqPath, localPath string) (*mcp.CallToolResult, error) { + file, err := os.Open(localPath) + if err != nil { + return toolErrorf("failed to open local file: %s", err.Error()) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return toolErrorf("failed to stat local file: %s", err.Error()) + } + + dir := stdpath.Dir(reqPath) + name := stdpath.Base(reqPath) + + fileStream := &stream.FileStream{ + Ctx: ctx, + Obj: &model.Object{ + Name: name, + Size: info.Size(), + Modified: time.Now(), + IsFolder: false, + }, + Reader: io.NopCloser(file), + Closers: utils.EmptyClosers(), + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.PutDirectly(ctx, dir, fileStream); err != nil { + return wrapError(err) + } + return textResult("uploaded successfully") +} + +func uploadViaHTTP(reqPath, localPath string) (*mcp.CallToolResult, error) { + file, err := os.Open(localPath) + if err != nil { + return toolErrorf("failed to open local file: %s", err.Error()) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return toolErrorf("failed to stat local file: %s", err.Error()) + } + + name := stdpath.Base(reqPath) + apiURL := fmt.Sprintf("%s/api/fs/put", conf.Conf.SiteURL) + + httpReq, err := http.NewRequest(http.MethodPut, apiURL, file) + if err != nil { + return toolErrorf("failed to create request: %s", err.Error()) + } + + httpReq.Header.Set("File-Path", url.PathEscape(reqPath)) + httpReq.Header.Set("Content-Length", strconv.FormatInt(info.Size(), 10)) + httpReq.Header.Set("Content-Type", utils.GetMimeType(name)) + httpReq.Header.Set("Authorization", setting.GetStr(conf.Token)) + httpReq.ContentLength = info.Size() + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return toolErrorf("upload request failed: %s", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return toolErrorf("upload failed (HTTP %d): %s", resp.StatusCode, string(body)) + } + return textResult("uploaded successfully") +} diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index d65d1ad648a..204b4b7205e 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -2,11 +2,16 @@ package middlewares import ( "crypto/subtle" + "errors" + "fmt" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -23,7 +28,9 @@ func Auth(c *gin.Context) { c.Abort() return } - c.Set("user", admin) + if !HandleSession(c, admin) { + return + } log.Debugf("use admin token: %+v", admin) c.Next() return @@ -40,7 +47,18 @@ func Auth(c *gin.Context) { c.Abort() return } - c.Set("user", guest) + if len(guest.Role) > 0 { + roles, err := op.GetRolesByUserID(guest.ID) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("Fail to load guest roles: %v", err), 500) + c.Abort() + return + } + guest.RolesDetail = roles + } + if !HandleSession(c, guest) { + return + } log.Debugf("use empty token: %+v", guest) c.Next() return @@ -68,11 +86,45 @@ func Auth(c *gin.Context) { c.Abort() return } - c.Set("user", user) + if len(user.Role) > 0 { + roles, err := op.GetRolesByUserID(user.ID) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("Fail to load roles: %v", err), 500) + c.Abort() + return + } + user.RolesDetail = roles + } + if !HandleSession(c, user) { + return + } log.Debugf("use login token: %+v", user) c.Next() } +// HandleSession verifies device sessions and stores context values. +func HandleSession(c *gin.Context, user *model.User) bool { + clientID := c.GetHeader("Client-Id") + if clientID == "" { + clientID = c.Query("client_id") + } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", user.ID, clientID)) + if err := device.Handle(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + token := c.GetHeader("Authorization") + if errors.Is(err, errs.SessionInactive) { + _ = common.InvalidateToken(token) + common.ErrorResp(c, err, 401) + } else { + common.ErrorResp(c, err, 403) + } + c.Abort() + return false + } + c.Set("device_key", key) + c.Set("user", user) + return true +} + func Authn(c *gin.Context) { token := c.GetHeader("Authorization") if subtle.ConstantTimeCompare([]byte(token), []byte(setting.GetStr(conf.Token))) == 1 { @@ -122,6 +174,19 @@ func Authn(c *gin.Context) { c.Abort() return } + if len(user.Role) > 0 { + var roles []model.Role + for _, roleID := range user.Role { + role, err := op.GetRole(uint(roleID)) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("load role %d failed", roleID), 500) + c.Abort() + return + } + roles = append(roles, *role) + } + user.RolesDetail = roles + } c.Set("user", user) log.Debugf("use login token: %+v", user) c.Next() diff --git a/server/middlewares/down.go b/server/middlewares/down.go index d015672de80..7e67663b74b 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -1,6 +1,7 @@ package middlewares import ( + "net/url" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -41,9 +42,8 @@ func Down(verifyFunc func(string, string) error) func(c *gin.Context) { } } -// TODO: implement -// path maybe contains # ? etc. func parsePath(path string) string { + path, _ = url.PathUnescape(path) return utils.FixAndCleanPath(path) } diff --git a/server/middlewares/fsup.go b/server/middlewares/fsup.go index 2aa7fca6d03..243c22e4131 100644 --- a/server/middlewares/fsup.go +++ b/server/middlewares/fsup.go @@ -35,7 +35,9 @@ func FsUp(c *gin.Context) { return } } - if !(common.CanAccess(user, meta, path, password) && (user.CanWrite() || common.CanWrite(meta, stdpath.Dir(path)))) { + perm := common.MergeRolePermissions(user, path) + if !(common.CanAccessWithRoles(user, meta, path, password) && + (common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, stdpath.Dir(path)))) { common.ErrorResp(c, errs.PermissionDenied, 403) c.Abort() return diff --git a/server/middlewares/session_refresh.go b/server/middlewares/session_refresh.go new file mode 100644 index 00000000000..2073020ce79 --- /dev/null +++ b/server/middlewares/session_refresh.go @@ -0,0 +1,26 @@ +package middlewares + +import ( + "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/model" + "github.com/gin-gonic/gin" +) + +// SessionRefresh updates session's last_active after successful requests. +func SessionRefresh(c *gin.Context) { + c.Next() + if c.Writer.Status() >= 400 { + return + } + if inactive, ok := c.Get("session_inactive"); ok { + if b, ok := inactive.(bool); ok && b { + return + } + } + userVal, uok := c.Get("user") + keyVal, kok := c.Get("device_key") + if uok && kok { + user := userVal.(*model.User) + device.Refresh(user.ID, keyVal.(string)) + } +} diff --git a/server/router.go b/server/router.go index 09a0bb44faf..e42d09472d1 100644 --- a/server/router.go +++ b/server/router.go @@ -22,6 +22,7 @@ func Init(e *gin.Engine) { }) } Cors(e) + e.Use(middlewares.SessionRefresh) g := e.Group(conf.URL.Path) if conf.Conf.Scheme.HttpPort != -1 && conf.Conf.Scheme.HttpsPort != -1 && conf.Conf.Scheme.ForceHttps { e.Use(middlewares.ForceHttps) @@ -46,6 +47,16 @@ func Init(e *gin.Engine) { g.GET("/p/*path", signCheck, downloadLimiter, handles.Proxy) g.HEAD("/d/*path", signCheck, handles.Down) g.HEAD("/p/*path", signCheck, handles.Proxy) + g.GET("/s/:share_id", handles.GetSharePage) + g.GET("/s/:share_id/*path", handles.GetSharePage) + g.GET("/sd/:share_id", downloadLimiter, handles.ShareDown) + g.GET("/sd/:share_id/*path", downloadLimiter, handles.ShareDown) + g.HEAD("/sd/:share_id", handles.ShareDown) + g.HEAD("/sd/:share_id/*path", handles.ShareDown) + g.GET("/sp/:share_id", downloadLimiter, handles.ShareProxy) + g.GET("/sp/:share_id/*path", downloadLimiter, handles.ShareProxy) + g.HEAD("/sp/:share_id", handles.ShareProxy) + g.HEAD("/sp/:share_id/*path", handles.ShareProxy) archiveSignCheck := middlewares.Down(sign.VerifyArchive) g.GET("/ad/*path", archiveSignCheck, downloadLimiter, handles.ArchiveDown) g.GET("/ap/*path", archiveSignCheck, downloadLimiter, handles.ArchiveProxy) @@ -61,6 +72,7 @@ func Init(e *gin.Engine) { api.POST("/auth/login", handles.Login) api.POST("/auth/login/hash", handles.LoginHash) api.POST("/auth/login/ldap", handles.LoginLdap) + api.POST("/auth/register", handles.Register) auth.GET("/me", handles.CurrentUser) auth.POST("/me/update", handles.UpdateCurrent) auth.GET("/me/sshkey/list", handles.ListMyPublicKey) @@ -69,6 +81,8 @@ func Init(e *gin.Engine) { auth.POST("/auth/2fa/generate", handles.Generate2FA) auth.POST("/auth/2fa/verify", handles.Verify2FA) auth.GET("/auth/logout", handles.LogOut) + auth.GET("/me/sessions", handles.ListMySessions) + auth.POST("/me/sessions/evict", handles.EvictMySession) // auth api.GET("/auth/sso", handles.SSOLoginRedirect) @@ -89,9 +103,21 @@ func Init(e *gin.Engine) { public.Any("/settings", handles.PublicSettings) public.Any("/offline_download_tools", handles.OfflineDownloadTools) public.Any("/archive_extensions", handles.ArchiveExtensions) + public.GET("/share/info", handles.GetPublicShareInfo) + public.POST("/share/auth", handles.AuthPublicShare) + public.POST("/share/list", handles.ListPublicShare) + public.POST("/share/get", handles.GetPublicShare) _fs(auth.Group("/fs")) + share := auth.Group("/share", middlewares.AuthNotGuest) + share.POST("/create", handles.CreateShare) + share.POST("/update", handles.UpdateShare) + share.POST("/disable", handles.DisableShare) + share.GET("/list", handles.ListShares) + share.POST("/delete", handles.DeleteShare) _task(auth.Group("/task", middlewares.AuthNotGuest)) + _label(auth.Group("/label")) + _labelFileBinding(auth.Group("/label_file_binding")) admin(auth.Group("/admin", middlewares.AuthAdmin)) if flags.Debug || flags.Dev { debug(g.Group("/debug")) @@ -120,6 +146,13 @@ func admin(g *gin.RouterGroup) { user.GET("/sshkey/list", handles.ListPublicKeys) user.POST("/sshkey/delete", handles.DeletePublicKey) + role := g.Group("/role") + role.GET("/list", handles.ListRoles) + role.GET("/get", handles.GetRole) + role.POST("/create", handles.CreateRole) + role.POST("/update", handles.UpdateRole) + role.POST("/delete", handles.DeleteRole) + storage := g.Group("/storage") storage.GET("/list", handles.ListStorages) storage.GET("/get", handles.GetStorage) @@ -141,12 +174,17 @@ func admin(g *gin.RouterGroup) { setting.POST("/save", handles.SaveSettings) setting.POST("/delete", handles.DeleteSetting) setting.POST("/reset_token", handles.ResetToken) + setting.POST("/set_token", handles.SetToken) setting.POST("/set_aria2", handles.SetAria2) setting.POST("/set_qbit", handles.SetQbittorrent) setting.POST("/set_transmission", handles.SetTransmission) setting.POST("/set_115", handles.Set115) setting.POST("/set_pikpak", handles.SetPikPak) setting.POST("/set_thunder", handles.SetThunder) + setting.POST("/set_guangyapan", handles.SetGuangYaPan) + setting.POST("/set_frp", handles.SetFRP) + setting.POST("/stop_frp", handles.StopFRP) + setting.GET("/frp_runtime", handles.GetFRPRuntime) // retain /admin/task API to ensure compatibility with legacy automation scripts _task(g.Group("/task")) @@ -161,6 +199,23 @@ func admin(g *gin.RouterGroup) { index.POST("/stop", middlewares.SearchIndex, handles.StopIndex) index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex) index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) + + label := g.Group("/label") + label.POST("/create", handles.CreateLabel) + label.POST("/update", handles.UpdateLabel) + label.POST("/delete", handles.DeleteLabel) + + labelFileBinding := g.Group("/label_file_binding") + labelFileBinding.GET("/list", handles.ListLabelFileBinding) + labelFileBinding.POST("/create", handles.CreateLabelFileBinDing) + labelFileBinding.POST("/create_batch", handles.CreateLabelFileBinDingBatch) + labelFileBinding.POST("/delete", handles.DelLabelByFileName) + labelFileBinding.POST("/restore", handles.RestoreLabelFileBinding) + + session := g.Group("/session") + session.GET("/list", handles.ListSessions) + session.POST("/evict", handles.EvictSession) + } func _fs(g *gin.RouterGroup) { @@ -168,6 +223,7 @@ func _fs(g *gin.RouterGroup) { g.Any("/search", middlewares.SearchIndex, handles.Search) g.Any("/get", handles.FsGet) g.Any("/other", handles.FsOther) + g.GET("/lark/export/download", handles.LarkExportDownload) g.Any("/dirs", handles.FsDirs) g.POST("/mkdir", handles.FsMkdir) g.POST("/rename", handles.FsRename) @@ -196,6 +252,16 @@ func _task(g *gin.RouterGroup) { handles.SetupTaskRoute(g) } +func _label(g *gin.RouterGroup) { + g.GET("/list", handles.ListLabel) + g.GET("/get", handles.GetLabel) +} + +func _labelFileBinding(g *gin.RouterGroup) { + g.GET("/get", handles.GetLabelByFileName) + g.GET("/get_file_by_label", handles.GetFileByLabel) +} + func Cors(r *gin.Engine) { config := cors.DefaultConfig() // config.AllowAllOrigins = true diff --git a/server/sftp.go b/server/sftp.go index 42c676e8c17..7d8c7212e9a 100644 --- a/server/sftp.go +++ b/server/sftp.go @@ -8,6 +8,7 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/ftp" "github.com/alist-org/alist/v3/server/sftp" "github.com/pkg/errors" @@ -78,7 +79,8 @@ func (d *SftpDriver) NoClientAuth(conn ssh.ConnMetadata) (*ssh.Permissions, erro if err != nil { return nil, err } - if guest.Disabled || !guest.CanFTPAccess() { + permGuest := common.MergeRolePermissions(guest, guest.BasePath) + if guest.Disabled || !common.HasPermission(permGuest, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via SFTP") } return nil, nil @@ -89,7 +91,8 @@ func (d *SftpDriver) PasswordAuth(conn ssh.ConnMetadata, password []byte) (*ssh. if err != nil { return nil, err } - if userObj.Disabled || !userObj.CanFTPAccess() { + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via SFTP") } passHash := model.StaticHash(string(password)) @@ -104,7 +107,8 @@ func (d *SftpDriver) PublicKeyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*s if err != nil { return nil, err } - if userObj.Disabled || !userObj.CanFTPAccess() { + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via SFTP") } keys, _, err := op.GetSSHPublicKeyByUserId(userObj.ID, 1, -1) diff --git a/server/webdav.go b/server/webdav.go index a735e285527..d188cb8010d 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -3,16 +3,23 @@ package server import ( "context" "crypto/subtle" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/server/middlewares" + "fmt" "net/http" + "net/url" "path" "strings" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/server/middlewares" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/webdav" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -25,6 +32,12 @@ func WebDav(dav *gin.RouterGroup) { Prefix: path.Join(conf.URL.Path, "/dav"), LockSystem: webdav.NewMemLS(), Logger: func(request *http.Request, err error) { + // Skip logging for NotFoundError as it's not a program error + // but a normal case when a file doesn't exist + if errs.IsNotFoundError(err) { + log.Debugf("%s %s %v", request.Method, request.URL.Path, err) + return + } log.Errorf("%s %s %+v", request.Method, request.URL.Path, err) }, } @@ -66,6 +79,13 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", admin.ID, c.ClientIP())) + if err := device.Handle(admin.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + c.Status(http.StatusForbidden) + c.Abort() + return + } + c.Set("device_key", key) c.Set("user", admin) c.Next() return @@ -92,7 +112,23 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - if user.Disabled || !user.CanWebdavRead() { + if roles, err := op.GetRolesByUserID(user.ID); err == nil { + user.RolesDetail = roles + } + reqPath := c.Param("path") + if reqPath == "" { + reqPath = "/" + } + reqPath, _ = url.PathUnescape(reqPath) + reqPath, err = webdav.ResolvePath(user, reqPath) + if err != nil { + c.Status(http.StatusForbidden) + c.Abort() + return + } + perm := common.MergeRolePermissions(user, reqPath) + webdavRead := common.HasPermission(perm, common.PermWebdavRead) + if user.Disabled || (!webdavRead && (c.Request.Method != "PROPFIND" || !common.HasChildPermission(user, reqPath, common.PermWebdavRead))) { if c.Request.Method == "OPTIONS" { c.Set("user", guest) c.Next() @@ -102,31 +138,38 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!user.CanWebdavManage() || !user.CanWrite()) { + if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermWrite)) { + c.Status(http.StatusForbidden) + c.Abort() + return + } + if c.Request.Method == "MOVE" && (!common.HasPermission(perm, common.PermWebdavManage) || (!common.HasPermission(perm, common.PermMove) && !common.HasPermission(perm, common.PermRename))) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "MOVE" && (!user.CanWebdavManage() || (!user.CanMove() && !user.CanRename())) { + if c.Request.Method == "COPY" && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermCopy)) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "COPY" && (!user.CanWebdavManage() || !user.CanCopy()) { + if c.Request.Method == "DELETE" && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermRemove)) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "DELETE" && (!user.CanWebdavManage() || !user.CanRemove()) { + if c.Request.Method == "PROPPATCH" && !common.HasPermission(perm, common.PermWebdavManage) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "PROPPATCH" && !user.CanWebdavManage() { + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", user.ID, c.ClientIP())) + if err := device.Handle(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { c.Status(http.StatusForbidden) c.Abort() return } + c.Set("device_key", key) c.Set("user", user) c.Next() } diff --git a/server/webdav/file.go b/server/webdav/file.go index ac8f5c1cbfb..419c7b07207 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -14,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" ) // slashClean is equivalent to but slightly more efficient than @@ -34,10 +35,11 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int srcName := path.Base(src) dstName := path.Base(dst) user := ctx.Value("user").(*model.User) - if srcDir != dstDir && !user.CanMove() { + perm := common.MergeRolePermissions(user, src) + if srcDir != dstDir && !common.HasPermission(perm, common.PermMove) { return http.StatusForbidden, nil } - if srcName != dstName && !user.CanRename() { + if srcName != dstName && !common.HasPermission(perm, common.PermRename) { return http.StatusForbidden, nil } if srcDir == dstDir { @@ -92,6 +94,7 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn depth = 0 } meta, _ := op.GetNearestMeta(name) + user := ctx.Value("user").(*model.User) // Read directory names. objs, err := fs.List(context.WithValue(ctx, "meta", meta), name, &fs.ListArgs{}) //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) @@ -106,6 +109,9 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn for _, fileInfo := range objs { filename := path.Join(name, fileInfo.GetName()) + if !common.CanReadPathByRole(user, filename) { + continue + } if err != nil { if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { return err diff --git a/server/webdav/path.go b/server/webdav/path.go new file mode 100644 index 00000000000..9a18da9551f --- /dev/null +++ b/server/webdav/path.go @@ -0,0 +1,22 @@ +package webdav + +import ( + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// ResolvePath normalizes the provided raw path and resolves it against the user's base path +// before delegating to the user-aware JoinPath permission checks. +func ResolvePath(user *model.User, raw string) (string, error) { + cleaned := utils.FixAndCleanPath(raw) + basePath := utils.FixAndCleanPath(user.BasePath) + + if cleaned != "/" && basePath != "/" && !utils.IsSubPath(basePath, cleaned) { + cleaned = path.Join(basePath, strings.TrimPrefix(cleaned, "/")) + } + + return user.JoinPath(cleaned) +} diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index f22e15aadb9..df1d2045140 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -21,7 +21,6 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" ) @@ -194,7 +193,7 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status } ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -222,7 +221,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta // TODO: check locks for read-only access?? ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return http.StatusForbidden, err } @@ -253,10 +252,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta return http.StatusInternalServerError, fmt.Errorf("webdav proxy error: %+v", err) } } else if storage.GetStorage().WebdavProxy() && downProxyUrl != "" { - u := fmt.Sprintf("%s%s?sign=%s", - strings.Split(downProxyUrl, "\n")[0], - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)) + u := common.BuildDownProxyURL(downProxyUrl, reqPath, storage.GetStorage().DownProxySign) w.Header().Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") http.Redirect(w, r, u, http.StatusFound) } else { @@ -282,7 +278,7 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -321,7 +317,7 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, // comments in http.checkEtag. ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return http.StatusForbidden, err } @@ -375,7 +371,7 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -439,11 +435,11 @@ func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request) (status ctx := r.Context() user := ctx.Value("user").(*model.User) - src, err = user.JoinPath(src) + src, err = ResolvePath(user, src) if err != nil { return 403, err } - dst, err = user.JoinPath(dst) + dst, err = ResolvePath(user, dst) if err != nil { return 403, err } @@ -540,7 +536,7 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus if err != nil { return status, err } - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -623,7 +619,7 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status userAgent := r.Header.Get("User-Agent") ctx = context.WithValue(ctx, "userAgent", userAgent) user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -648,6 +644,98 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status mw := multistatusWriter{w: w} + if utils.PathEqual(reqPath, user.BasePath) { + hasRootPerm := false + for _, role := range user.RolesDetail { + for _, entry := range role.PermissionScopes { + if utils.PathEqual(entry.Path, user.BasePath) { + hasRootPerm = true + break + } + } + if hasRootPerm { + break + } + } + if !hasRootPerm { + basePaths := model.GetAllBasePathsFromRoles(user) + type infoItem struct { + path string + info model.Obj + } + infos := []infoItem{{reqPath, fi}} + seen := make(map[string]struct{}) + for _, p := range basePaths { + if !utils.IsSubPath(user.BasePath, p) { + continue + } + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(p), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + dir := strings.Split(rel, "/")[0] + if dir == "" { + continue + } + if _, ok := seen[dir]; ok { + continue + } + seen[dir] = struct{}{} + sp := utils.FixAndCleanPath(path.Join(user.BasePath, dir)) + info, err := fs.Get(ctx, sp, &fs.GetArgs{}) + if err != nil { + continue + } + infos = append(infos, infoItem{sp, info}) + } + for _, item := range infos { + var pstats []Propstat + if pf.Propname != nil { + pnames, err := propnames(ctx, h.LockSystem, item.info) + if err != nil { + return http.StatusInternalServerError, err + } + pstat := Propstat{Status: http.StatusOK} + for _, xmlname := range pnames { + pstat.Props = append(pstat.Props, Property{XMLName: xmlname}) + } + pstats = append(pstats, pstat) + } else if pf.Allprop != nil { + pstats, err = allprop(ctx, h.LockSystem, item.info, pf.Prop) + if err != nil { + return http.StatusInternalServerError, err + } + } else { + pstats, err = props(ctx, h.LockSystem, item.info, pf.Prop) + if err != nil { + return http.StatusInternalServerError, err + } + } + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(item.path), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + href := utils.EncodePath(path.Join("/", h.Prefix, rel), true) + if href != "/" && item.info.IsDir() { + href += "/" + } + if err := mw.write(makePropstatResponse(href, pstats)); err != nil { + return http.StatusInternalServerError, err + } + } + if err := mw.close(); err != nil { + return http.StatusInternalServerError, err + } + return 0, nil + } + } + walkFn := func(reqPath string, info model.Obj, err error) error { if err != nil { return err @@ -671,7 +759,14 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status if err != nil { return err } - href := path.Join(h.Prefix, strings.TrimPrefix(reqPath, user.BasePath)) + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(reqPath), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + href := utils.EncodePath(path.Join("/", h.Prefix, rel), true) if href != "/" && info.IsDir() { href += "/" } @@ -702,7 +797,7 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -734,7 +829,7 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu func makePropstatResponse(href string, pstats []Propstat) *response { resp := response{ - Href: []string{(&url.URL{Path: href}).EscapedPath()}, + Href: []string{href}, Propstat: make([]propstat, 0, len(pstats)), } for _, p := range pstats {