feat(chunk upload): Support uploading by chunks on drivers/local

src/lang/en/drivers.json

@@ -729,6 +729,8 @@
     "thumb_pixel-tips": "Specifies the target width for image thumbnails in pixels. The height of the thumbnail will be calculated automatically to maintain the original aspect ratio of the image.",
     "thumbnail": "Thumbnail",
     "thumbnail-tips": "enable thumbnail",
+    "upload_chunk_size_mb": "Upload chunk size",
+    "upload_chunk_size_mb-tips": "chunk size in MB for web uploads, range 0-4096, 0 disables chunk upload",
     "use_ffmpeg": "Use ffmpeg",
     "use_ffmpeg-tips": "use ffmpeg to generate thumbnail",
     "video_thumb_pos": "Video thumb pos",

src/lang/en/storages.json

@@ -45,5 +45,7 @@
     "filter_by_driver": "Filter by driver",
     "table_layout": "Table layout"
   },
+  "local_upload_chunk_size_invalid": "Upload chunk size must be an integer in MB.",
+  "local_upload_chunk_size_range": "Upload chunk size must be between {{min}} and {{max}} MB.",
   "no_storage_content": "failed get storage: storage not found; please add a storage first"
 }

src/pages/home/uploads/Upload.tsx

@@ -28,6 +28,7 @@ import { File2Upload, traverseFileTree } from "./util"
 import { SelectWrapper } from "~/components"
 import { getUploads } from "./uploads"
 import { TaskState } from "~/pages/manage/tasks/Task"
+import { uploadWithChunkPolicy } from "./chunk"
 
 enum TaskStateEnum {
   Pending,
@@ -77,19 +78,9 @@ const UploadFile = (props: UploadFileProps) => {
         <Text>{getFileSize(props.speed)}/s</Text>
       </HStack>
       <Show when={props.task_id}>
-        <HStack spacing="$2" flexWrap="wrap">
-          <Badge colorScheme="accent">{props.task_id}</Badge>
-          <Button
-            size="xs"
-            variant="subtle"
-            as="a"
-            href={joinBase("/@manage/tasks/upload")}
-            target="_blank"
-            rel="noreferrer"
-          >
-            {t("home.upload.open_task_center")}
-          </Button>
-        </HStack>
+        <Badge colorScheme="accent">
+          {t("home.upload.task_id", { id: props.task_id! })}
+        </Badge>
       </Show>
       <Progress
         w="$full"
@@ -130,8 +121,8 @@ const Upload = () => {
   }
   const hasBackgroundTask = () =>
     uploadFiles.uploads.some(({ task_id }) => !!task_id)
-  let fileInput: HTMLInputElement
-  let folderInput: HTMLInputElement
+  let fileInput!: HTMLInputElement
+  let folderInput!: HTMLInputElement
   const clearTaskPoller = (path: string) => {
     const timer = taskPollers.get(path)
     if (timer !== undefined) {
@@ -205,7 +196,7 @@ const Upload = () => {
     setUpload(path, "status", "uploading")
     const uploadPath = pathJoin(pathname(), path)
     try {
-      const result = await curUploader().upload(
+      const result = await uploadWithChunkPolicy(
         uploadPath,
         file,
         (key, value) => {
@@ -214,6 +205,7 @@ const Upload = () => {
         asTask(),
         overwrite(),
         rapid(),
+        curUploader().upload,
       )
       if (result.error) {
         setUpload(path, "status", "error")

src/pages/home/uploads/chunk.ts

@@ -0,0 +1,217 @@
+import { password } from "~/store"
+import { Resp, TaskInfo } from "~/types"
+import { r } from "~/utils"
+import { SetUpload, Upload, UploadResult } from "./types"
+import { calculateHash } from "./util"
+
+type PolicyResp = Resp<{
+  provider: string
+  chunk_size_mb: number
+  enabled: boolean
+}>
+
+type InitResp = Resp<{
+  upload_id: string
+  provider: string
+  chunk_size_mb: number
+  chunk_size_bytes: number
+}>
+
+type CompleteResp = Resp<{
+  task?: TaskInfo
+}>
+
+const toError = (e: any) => {
+  if (e instanceof Error) {
+    return e
+  }
+  if (e?.message) {
+    return new Error(e.message)
+  }
+  return new Error("upload failed")
+}
+
+const buildBaseHeaders = (
+  uploadPath: string,
+  asTask: boolean,
+  overwrite: boolean,
+  file: File,
+) => ({
+  "File-Path": encodeURIComponent(uploadPath),
+  "As-Task": asTask,
+  "Last-Modified": file.lastModified,
+  Password: password(),
+  Overwrite: overwrite.toString(),
+})
+
+const calcSpeed = (
+  now: number,
+  loaded: number,
+  oldTimestamp: number,
+  oldLoaded: number,
+  setUpload: SetUpload,
+) => {
+  const duration = (now - oldTimestamp) / 1000
+  if (duration <= 1) {
+    return { oldTimestamp, oldLoaded }
+  }
+  const deltaLoaded = loaded - oldLoaded
+  if (deltaLoaded >= 0) {
+    setUpload("speed", deltaLoaded / duration)
+  }
+  return { oldTimestamp: now, oldLoaded: loaded }
+}
+
+const cleanupFailedUpload = async (uploadID?: string) => {
+  if (!uploadID) {
+    return
+  }
+  try {
+    await r.post("/fs/upload/cancel", { upload_id: uploadID })
+  } catch {
+    // Best-effort cleanup; main error should still propagate.
+  }
+}
+
+const chunkUpload = async (
+  uploadPath: string,
+  file: File,
+  setUpload: SetUpload,
+  asTask: boolean,
+  overwrite: boolean,
+  rapid: boolean,
+  chunkSizeBytes: number,
+): Promise<UploadResult> => {
+  let uploadID = ""
+  const initHeaders: { [k: string]: any } = {
+    ...buildBaseHeaders(uploadPath, asTask, overwrite, file),
+    "Content-Type": "application/json",
+  }
+  if (rapid) {
+    const { md5, sha1, sha256 } = await calculateHash(file)
+    initHeaders["X-File-Md5"] = md5
+    initHeaders["X-File-Sha1"] = sha1
+    initHeaders["X-File-Sha256"] = sha256
+  }
+
+  const initResp: InitResp = await r.post(
+    "/fs/upload/init",
+    {
+      size: file.size,
+      as_task: asTask,
+      overwrite,
+      last_modified: file.lastModified,
+      mimetype: file.type || "application/octet-stream",
+    },
+    { headers: initHeaders },
+  )
+  if (initResp.code !== 200 || !initResp.data?.upload_id) {
+    return {
+      error: new Error(initResp.message),
+    }
+  }
+  uploadID = initResp.data.upload_id
+  const effectiveChunkSize = initResp.data.chunk_size_bytes || chunkSizeBytes
+  if (effectiveChunkSize <= 0) {
+    await cleanupFailedUpload(uploadID)
+    return {
+      error: new Error("invalid chunk size"),
+    }
+  }
+
+  let oldTimestamp = Date.now()
+  let oldLoaded = 0
+
+  try {
+    for (
+      let chunkIndex = 0, start = 0;
+      start < file.size;
+      chunkIndex++, start += effectiveChunkSize
+    ) {
+      const end = Math.min(start + effectiveChunkSize, file.size)
+      const chunk = file.slice(start, end)
+      const resp: Resp<{
+        uploaded_size: number
+        total_size: number
+      }> = await r.put("/fs/upload/chunk", chunk, {
+        headers: {
+          "Upload-Id": uploadID,
+          "Chunk-Index": chunkIndex,
+          "Content-Type": "application/octet-stream",
+        },
+        onUploadProgress: (progressEvent) => {
+          const loadedThisChunk = progressEvent.loaded ?? 0
+          const totalLoaded = Math.min(start + loadedThisChunk, file.size)
+          const complete = ((totalLoaded / file.size) * 100) | 0
+          setUpload("progress", complete)
+
+          const speedCalc = calcSpeed(
+            Date.now(),
+            totalLoaded,
+            oldTimestamp,
+            oldLoaded,
+            setUpload,
+          )
+          oldTimestamp = speedCalc.oldTimestamp
+          oldLoaded = speedCalc.oldLoaded
+        },
+      })
+      if (resp.code !== 200) {
+        throw new Error(resp.message)
+      }
+      const uploadedSize = resp.data?.uploaded_size ?? end
+      const complete = ((uploadedSize / file.size) * 100) | 0
+      setUpload("progress", Math.min(100, complete))
+    }
+
+    setUpload("status", "backending")
+    const completeResp: CompleteResp = await r.post("/fs/upload/complete", {
+      upload_id: uploadID,
+    })
+    if (completeResp.code !== 200) {
+      throw new Error(completeResp.message)
+    }
+    return {
+      task: completeResp.data?.task,
+    }
+  } catch (e: any) {
+    await cleanupFailedUpload(uploadID)
+    return {
+      error: toError(e),
+    }
+  }
+}
+
+export const uploadWithChunkPolicy = async (
+  uploadPath: string,
+  file: File,
+  setUpload: SetUpload,
+  asTask: boolean,
+  overwrite: boolean,
+  rapid: boolean,
+  fallbackUpload: Upload,
+): Promise<UploadResult> => {
+  const policyHeaders = buildBaseHeaders(uploadPath, asTask, overwrite, file)
+  const policyResp: PolicyResp = await r.post(
+    "/fs/upload/policy",
+    {},
+    { headers: policyHeaders },
+  )
+  if (policyResp.code !== 200 || !policyResp.data?.enabled) {
+    return fallbackUpload(uploadPath, file, setUpload, asTask, overwrite, rapid)
+  }
+  const chunkSizeMB = policyResp.data.chunk_size_mb ?? 0
+  const chunkSizeBytes = Math.floor(chunkSizeMB * 1024 * 1024)
+  if (chunkSizeBytes <= 0 || file.size <= chunkSizeBytes) {
+    return fallbackUpload(uploadPath, file, setUpload, asTask, overwrite, rapid)
+  }
+  return chunkUpload(
+    uploadPath,
+    file,
+    setUpload,
+    asTask,
+    overwrite,
+    rapid,
+    chunkSizeBytes,
+  )
+}

src/pages/manage/storages/AddOrEdit.tsx

@@ -43,6 +43,8 @@ function GetDefaultValue(type: Type, value?: string) {
 
 type Drivers = Record<string, DriverInfo>
 
+const maxLocalUploadChunkSizeMB = 4096
+
 const AddOrEdit = () => {
   const t = useT()
   const { params, back, to } = useRouter()
@@ -88,6 +90,31 @@ const AddOrEdit = () => {
     setStorage("addition", JSON.stringify(addition))
     return r.post(`/admin/storage/${id ? "update" : "create"}`, storage)
   })
+  const validateLocalUploadChunkSize = () => {
+    if (storage.driver !== "Local") {
+      return true
+    }
+    const raw = addition.upload_chunk_size_mb
+    if (raw === undefined || raw === null || raw === "") {
+      return true
+    }
+    const valueStr = String(raw).trim()
+    if (!/^-?\d+$/.test(valueStr)) {
+      notify.error(t("storages.local_upload_chunk_size_invalid"))
+      return false
+    }
+    const value = Number.parseInt(valueStr, 10)
+    if (value < 0 || value > maxLocalUploadChunkSizeMB) {
+      notify.error(
+        t("storages.local_upload_chunk_size_range", {
+          min: "0",
+          max: String(maxLocalUploadChunkSizeMB),
+        }),
+      )
+      return false
+    }
+    return true
+  }
   const alert = createMemo(() => {
     const i = drivers()[storage.driver]?.config.alert
     console.log(i)
@@ -178,6 +205,9 @@ const AddOrEdit = () => {
         mt="$2"
         loading={okLoading()}
         onClick={async () => {
+          if (!validateLocalUploadChunkSize()) {
+            return
+          }
           if (drivers()[storage.driver].config.need_ms) {
             notify.info(t("manage.add_storage-tips"))
             window.open(joinBase("/@manage/messenger"), "_blank")