diff --git a/src/lang/en/drivers.json b/src/lang/en/drivers.json index bc7a057cb8..d4945f7de4 100755 --- a/src/lang/en/drivers.json +++ b/src/lang/en/drivers.json @@ -729,6 +729,8 @@ "thumb_pixel-tips": "Specifies the target width for image thumbnails in pixels. The height of the thumbnail will be calculated automatically to maintain the original aspect ratio of the image.", "thumbnail": "Thumbnail", "thumbnail-tips": "enable thumbnail", + "upload_chunk_size_mb": "Upload chunk size", + "upload_chunk_size_mb-tips": "chunk size in MB for web uploads, range 0-4096, 0 disables chunk upload", "use_ffmpeg": "Use ffmpeg", "use_ffmpeg-tips": "use ffmpeg to generate thumbnail", "video_thumb_pos": "Video thumb pos", diff --git a/src/lang/en/storages.json b/src/lang/en/storages.json index 01039e337e..b8fbe24309 100644 --- a/src/lang/en/storages.json +++ b/src/lang/en/storages.json @@ -45,5 +45,7 @@ "filter_by_driver": "Filter by driver", "table_layout": "Table layout" }, + "local_upload_chunk_size_invalid": "Upload chunk size must be an integer in MB.", + "local_upload_chunk_size_range": "Upload chunk size must be between {{min}} and {{max}} MB.", "no_storage_content": "failed get storage: storage not found; please add a storage first" } diff --git a/src/pages/home/uploads/Upload.tsx b/src/pages/home/uploads/Upload.tsx index 1e81f7fbd9..095b97de13 100644 --- a/src/pages/home/uploads/Upload.tsx +++ b/src/pages/home/uploads/Upload.tsx @@ -28,6 +28,7 @@ import { File2Upload, traverseFileTree } from "./util" import { SelectWrapper } from "~/components" import { getUploads } from "./uploads" import { TaskState } from "~/pages/manage/tasks/Task" +import { uploadWithChunkPolicy } from "./chunk" enum TaskStateEnum { Pending, @@ -77,19 +78,9 @@ const UploadFile = (props: UploadFileProps) => { {getFileSize(props.speed)}/s - - {props.task_id} - - + + {t("home.upload.task_id", { id: props.task_id! })} + { } const hasBackgroundTask = () => uploadFiles.uploads.some(({ task_id }) => !!task_id) - let fileInput: HTMLInputElement - let folderInput: HTMLInputElement + let fileInput!: HTMLInputElement + let folderInput!: HTMLInputElement const clearTaskPoller = (path: string) => { const timer = taskPollers.get(path) if (timer !== undefined) { @@ -205,7 +196,7 @@ const Upload = () => { setUpload(path, "status", "uploading") const uploadPath = pathJoin(pathname(), path) try { - const result = await curUploader().upload( + const result = await uploadWithChunkPolicy( uploadPath, file, (key, value) => { @@ -214,6 +205,7 @@ const Upload = () => { asTask(), overwrite(), rapid(), + curUploader().upload, ) if (result.error) { setUpload(path, "status", "error") diff --git a/src/pages/home/uploads/chunk.ts b/src/pages/home/uploads/chunk.ts new file mode 100644 index 0000000000..d89081a693 --- /dev/null +++ b/src/pages/home/uploads/chunk.ts @@ -0,0 +1,217 @@ +import { password } from "~/store" +import { Resp, TaskInfo } from "~/types" +import { r } from "~/utils" +import { SetUpload, Upload, UploadResult } from "./types" +import { calculateHash } from "./util" + +type PolicyResp = Resp<{ + provider: string + chunk_size_mb: number + enabled: boolean +}> + +type InitResp = Resp<{ + upload_id: string + provider: string + chunk_size_mb: number + chunk_size_bytes: number +}> + +type CompleteResp = Resp<{ + task?: TaskInfo +}> + +const toError = (e: any) => { + if (e instanceof Error) { + return e + } + if (e?.message) { + return new Error(e.message) + } + return new Error("upload failed") +} + +const buildBaseHeaders = ( + uploadPath: string, + asTask: boolean, + overwrite: boolean, + file: File, +) => ({ + "File-Path": encodeURIComponent(uploadPath), + "As-Task": asTask, + "Last-Modified": file.lastModified, + Password: password(), + Overwrite: overwrite.toString(), +}) + +const calcSpeed = ( + now: number, + loaded: number, + oldTimestamp: number, + oldLoaded: number, + setUpload: SetUpload, +) => { + const duration = (now - oldTimestamp) / 1000 + if (duration <= 1) { + return { oldTimestamp, oldLoaded } + } + const deltaLoaded = loaded - oldLoaded + if (deltaLoaded >= 0) { + setUpload("speed", deltaLoaded / duration) + } + return { oldTimestamp: now, oldLoaded: loaded } +} + +const cleanupFailedUpload = async (uploadID?: string) => { + if (!uploadID) { + return + } + try { + await r.post("/fs/upload/cancel", { upload_id: uploadID }) + } catch { + // Best-effort cleanup; main error should still propagate. + } +} + +const chunkUpload = async ( + uploadPath: string, + file: File, + setUpload: SetUpload, + asTask: boolean, + overwrite: boolean, + rapid: boolean, + chunkSizeBytes: number, +): Promise => { + let uploadID = "" + const initHeaders: { [k: string]: any } = { + ...buildBaseHeaders(uploadPath, asTask, overwrite, file), + "Content-Type": "application/json", + } + if (rapid) { + const { md5, sha1, sha256 } = await calculateHash(file) + initHeaders["X-File-Md5"] = md5 + initHeaders["X-File-Sha1"] = sha1 + initHeaders["X-File-Sha256"] = sha256 + } + + const initResp: InitResp = await r.post( + "/fs/upload/init", + { + size: file.size, + as_task: asTask, + overwrite, + last_modified: file.lastModified, + mimetype: file.type || "application/octet-stream", + }, + { headers: initHeaders }, + ) + if (initResp.code !== 200 || !initResp.data?.upload_id) { + return { + error: new Error(initResp.message), + } + } + uploadID = initResp.data.upload_id + const effectiveChunkSize = initResp.data.chunk_size_bytes || chunkSizeBytes + if (effectiveChunkSize <= 0) { + await cleanupFailedUpload(uploadID) + return { + error: new Error("invalid chunk size"), + } + } + + let oldTimestamp = Date.now() + let oldLoaded = 0 + + try { + for ( + let chunkIndex = 0, start = 0; + start < file.size; + chunkIndex++, start += effectiveChunkSize + ) { + const end = Math.min(start + effectiveChunkSize, file.size) + const chunk = file.slice(start, end) + const resp: Resp<{ + uploaded_size: number + total_size: number + }> = await r.put("/fs/upload/chunk", chunk, { + headers: { + "Upload-Id": uploadID, + "Chunk-Index": chunkIndex, + "Content-Type": "application/octet-stream", + }, + onUploadProgress: (progressEvent) => { + const loadedThisChunk = progressEvent.loaded ?? 0 + const totalLoaded = Math.min(start + loadedThisChunk, file.size) + const complete = ((totalLoaded / file.size) * 100) | 0 + setUpload("progress", complete) + + const speedCalc = calcSpeed( + Date.now(), + totalLoaded, + oldTimestamp, + oldLoaded, + setUpload, + ) + oldTimestamp = speedCalc.oldTimestamp + oldLoaded = speedCalc.oldLoaded + }, + }) + if (resp.code !== 200) { + throw new Error(resp.message) + } + const uploadedSize = resp.data?.uploaded_size ?? end + const complete = ((uploadedSize / file.size) * 100) | 0 + setUpload("progress", Math.min(100, complete)) + } + + setUpload("status", "backending") + const completeResp: CompleteResp = await r.post("/fs/upload/complete", { + upload_id: uploadID, + }) + if (completeResp.code !== 200) { + throw new Error(completeResp.message) + } + return { + task: completeResp.data?.task, + } + } catch (e: any) { + await cleanupFailedUpload(uploadID) + return { + error: toError(e), + } + } +} + +export const uploadWithChunkPolicy = async ( + uploadPath: string, + file: File, + setUpload: SetUpload, + asTask: boolean, + overwrite: boolean, + rapid: boolean, + fallbackUpload: Upload, +): Promise => { + const policyHeaders = buildBaseHeaders(uploadPath, asTask, overwrite, file) + const policyResp: PolicyResp = await r.post( + "/fs/upload/policy", + {}, + { headers: policyHeaders }, + ) + if (policyResp.code !== 200 || !policyResp.data?.enabled) { + return fallbackUpload(uploadPath, file, setUpload, asTask, overwrite, rapid) + } + const chunkSizeMB = policyResp.data.chunk_size_mb ?? 0 + const chunkSizeBytes = Math.floor(chunkSizeMB * 1024 * 1024) + if (chunkSizeBytes <= 0 || file.size <= chunkSizeBytes) { + return fallbackUpload(uploadPath, file, setUpload, asTask, overwrite, rapid) + } + return chunkUpload( + uploadPath, + file, + setUpload, + asTask, + overwrite, + rapid, + chunkSizeBytes, + ) +} diff --git a/src/pages/manage/storages/AddOrEdit.tsx b/src/pages/manage/storages/AddOrEdit.tsx index e98f39485c..5dfc33d77e 100644 --- a/src/pages/manage/storages/AddOrEdit.tsx +++ b/src/pages/manage/storages/AddOrEdit.tsx @@ -43,6 +43,8 @@ function GetDefaultValue(type: Type, value?: string) { type Drivers = Record +const maxLocalUploadChunkSizeMB = 4096 + const AddOrEdit = () => { const t = useT() const { params, back, to } = useRouter() @@ -88,6 +90,31 @@ const AddOrEdit = () => { setStorage("addition", JSON.stringify(addition)) return r.post(`/admin/storage/${id ? "update" : "create"}`, storage) }) + const validateLocalUploadChunkSize = () => { + if (storage.driver !== "Local") { + return true + } + const raw = addition.upload_chunk_size_mb + if (raw === undefined || raw === null || raw === "") { + return true + } + const valueStr = String(raw).trim() + if (!/^-?\d+$/.test(valueStr)) { + notify.error(t("storages.local_upload_chunk_size_invalid")) + return false + } + const value = Number.parseInt(valueStr, 10) + if (value < 0 || value > maxLocalUploadChunkSizeMB) { + notify.error( + t("storages.local_upload_chunk_size_range", { + min: "0", + max: String(maxLocalUploadChunkSizeMB), + }), + ) + return false + } + return true + } const alert = createMemo(() => { const i = drivers()[storage.driver]?.config.alert console.log(i) @@ -178,6 +205,9 @@ const AddOrEdit = () => { mt="$2" loading={okLoading()} onClick={async () => { + if (!validateLocalUploadChunkSize()) { + return + } if (drivers()[storage.driver].config.need_ms) { notify.info(t("manage.add_storage-tips")) window.open(joinBase("/@manage/messenger"), "_blank") diff --git a/src/pages/manage/storages/Item.tsx b/src/pages/manage/storages/Item.tsx index cb56990bc4..cd48b4bd23 100644 --- a/src/pages/manage/storages/Item.tsx +++ b/src/pages/manage/storages/Item.tsx @@ -53,6 +53,8 @@ export type ItemProps = DriverItem & { } ) +const maxLocalUploadChunkSizeMB = 4096 + const Item = (props: ItemProps) => { const t = useT() const isFolderPathField = @@ -63,6 +65,10 @@ const Item = (props: ItemProps) => { props.type === Type.Number && props.driver === "Chunker" && props.name === "chunk_size" + const isLocalUploadChunkSizeField = + props.type === Type.Number && + props.driver === "Local" && + props.name === "upload_chunk_size_mb" const isChunkerRemotePathsField = props.type === Type.Text && props.driver === "Chunker" && @@ -149,10 +155,26 @@ const Item = (props: ItemProps) => { type="number" id={props.name} readOnly={props.readonly} + min={isLocalUploadChunkSizeField ? 0 : undefined} + max={ + isLocalUploadChunkSizeField + ? maxLocalUploadChunkSizeMB + : undefined + } + step={isLocalUploadChunkSizeField ? 1 : undefined} value={props.value as number} onInput={ props.type === Type.Number - ? (e) => props.onChange?.(parseInt(e.currentTarget.value)) + ? (e) => { + let nextValue = Number.parseInt( + e.currentTarget.value, + 10, + ) + if (!Number.isFinite(nextValue)) { + nextValue = 0 + } + props.onChange?.(nextValue) + } : undefined } /> @@ -294,6 +316,14 @@ const Item = (props: ItemProps) => { )} + + + {t("storages.local_upload_chunk_size_range", { + min: "0", + max: String(maxLocalUploadChunkSizeMB), + })} + + {`${(props.value as number).toLocaleString()} bytes (${getFileSize(