From 1690fdaf24fa091234bc0812556cd5d46b0b12f3 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 23 Mar 2026 11:06:51 +0100 Subject: [PATCH 1/2] wip --- .../src/ResourceFetcherV2.ts | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 packages/expo-resource-fetcher/src/ResourceFetcherV2.ts diff --git a/packages/expo-resource-fetcher/src/ResourceFetcherV2.ts b/packages/expo-resource-fetcher/src/ResourceFetcherV2.ts new file mode 100644 index 0000000000..ee1c624707 --- /dev/null +++ b/packages/expo-resource-fetcher/src/ResourceFetcherV2.ts @@ -0,0 +1,162 @@ +/** + * Resource Fetcher for Expo applications. + * + * This module provides functions to download and manage files stored in the application's document directory + * inside the `react-native-executorch/` directory. These utilities help manage storage and clean up downloaded + * files when they are no longer needed. + * + * @category Utilities - General + * + * @remarks + * **Key Functionality:** + * - **Download Control**: Pause, resume, and cancel operations through: + * - {@link pauseFetching} - Pause ongoing downloads + * - {@link resumeFetching} - Resume paused downloads + * - {@link cancelFetching} - Cancel ongoing or paused downloads + * - **File Management**: + * - {@link getFilesTotalSize} - Get total size of resources + * - {@link listDownloadedFiles} - List all downloaded files + * - {@link listDownloadedModels} - List downloaded model files (.pte) + * - {@link deleteResources} - Delete downloaded resources + * + * **Important Notes:** + * - Pause/resume/cancel operations work only for remote resources + * - Most functions accept multiple `ResourceSource` arguments (string, number, or object) + * - The {@link fetch} method accepts a progress callback (0-1) and returns file paths or null if interrupted + * + * **Technical Implementation:** + * - Maintains a `downloads` Map to track active and paused downloads + * - Successful downloads are automatically removed from the Map + * - Uses `ResourceSourceExtended` interface for pause/resume functionality with linked-list behavior + */ + +import { + ResourceSource, + ResourceFetcherAdapter, + DownloadStatus, + SourceType, + RnExecutorchError, + RnExecutorchErrorCode, +} from 'react-native-executorch'; +import { + ResourceSourceExtended, + DownloadResource, + ResourceFetcherUtils, +} from './ResourceFetcherUtils'; +import { + cacheDirectory, + createDownloadResumable, + DownloadResumable, +} from 'expo-file-system/legacy'; +import { RNEDirectory } from './constants/directories'; + +export interface DownloadMetadata { + source: ResourceSource; + type: SourceType; + remoteUri?: string; + localUri: string; + onDownloadProgress?: (downloadProgress: number) => void; +} + +export interface DownloadAsset { + downloadObj: DownloadResumable; + status: DownloadStatus; + metadata: DownloadMetadata; +} + +interface ExpoResourceFetcherInterface extends ResourceFetcherAdapter { + downloadAssets: Record; + singleFetch(source: ResourceSource): Promise; + returnOrStartNext( + sourceExtended: ResourceSourceExtended, + result: string | string[] + ): string[] | Promise; + pause(source: ResourceSource): Promise; + resume(source: ResourceSource): Promise; + cancel(source: ResourceSource): Promise; + findActive(sources: ResourceSource[]): ResourceSource; + pauseFetching(...sources: ResourceSource[]): Promise; + resumeFetching(...sources: ResourceSource[]): Promise; + cancelFetching(...sources: ResourceSource[]): Promise; + listDownloadedFiles(): Promise; + listDownloadedModels(): Promise; + deleteResources(...sources: ResourceSource[]): Promise; + getFilesTotalSize(...sources: ResourceSource[]): Promise; + handleObject(source: ResourceSource): Promise; + handleLocalFile(source: ResourceSource): string; + handleReleaseModeFile(source: DownloadAsset): Promise; + handleDevModeFile(source: DownloadAsset): Promise; + handleRemoteFile(source: DownloadAsset): Promise; + handleCachedDownload(source: ResourceSource): Promise; +} + +export const ExpoResourceFetcher: ExpoResourceFetcherInterface = { + downloadAssets: {}, + + async singleFetch(source: DownloadAsset): Promise { + switch (source.metadata.type) { + case SourceType.OBJECT: { + return await this.handleObject(source); + } + case SourceType.LOCAL_FILE: { + return this.handleLocalFile(source); + } + case SourceType.RELEASE_MODE_FILE: { + return this.handleReleaseModeFile(source); + } + case SourceType.DEV_MODE_FILE: { + return this.handleDevModeFile(source); + } + case SourceType.REMOTE_FILE: { + return this.handleRemoteFile(source); + } + } + }, + + async handleCachedDownload(source: ResourceSource) {}, + + async handleRemoteFile(source: ResourceSource) { + if (typeof source === 'object') { + throw new RnExecutorchError( + RnExecutorchErrorCode.InvalidModelSource, + 'Model source is expected to be a string or a number.' + ); + } + if (this.downloadAssets[source]) { + return this.handleCachedDownload(source); + } + + const uri = 'asda'; + const targetFilename = ResourceFetcherUtils.getFilenameFromUri(uri); + const fileUri = `${RNEDirectory}${targetFilename}`; + const cacheFileUri = `${cacheDirectory}${targetFilename}`; + + if (await ResourceFetcherUtils.checkFileExists(fileUri)) { + return ResourceFetcherUtils.removeFilePrefix(fileUri); + } + + await ResourceFetcherUtils.createDirectoryIfNoExists(); + + const downloadResumable = createDownloadResumable(uri, fileUri); + }, + + async fetch( + callback: (downloadProgress: number) => void = () => {}, + ...sources: ResourceSource[] + ) { + if (sources.length === 0) { + throw new RnExecutorchError( + RnExecutorchErrorCode.InvalidUserInput, + "Empty list given as an argument to Resource Fetcher's fetch() function!" + ); + } + + const totalFilesLength = (await ResourceFetcherUtils.getFilesSizes(sources)) + .totalLength; + const downloadedBytesCounter = 0; + for (let source of sources) { + const downloadResult = await this.singleFetch(source); + } + return ['']; + }, +}; From c996c1f51aea1a68d7cc39cc5690edc167475c63 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Fri, 27 Mar 2026 17:50:58 +0100 Subject: [PATCH 2/2] wip --- .../src/ResourceFetcherV2.ts | 341 ++++++++++++------ .../expo-resource-fetcher/src/handlers.ts | 179 +++++++++ 2 files changed, 414 insertions(+), 106 deletions(-) create mode 100644 packages/expo-resource-fetcher/src/handlers.ts diff --git a/packages/expo-resource-fetcher/src/ResourceFetcherV2.ts b/packages/expo-resource-fetcher/src/ResourceFetcherV2.ts index ee1c624707..dd77193681 100644 --- a/packages/expo-resource-fetcher/src/ResourceFetcherV2.ts +++ b/packages/expo-resource-fetcher/src/ResourceFetcherV2.ts @@ -23,140 +23,269 @@ * - Pause/resume/cancel operations work only for remote resources * - Most functions accept multiple `ResourceSource` arguments (string, number, or object) * - The {@link fetch} method accepts a progress callback (0-1) and returns file paths or null if interrupted - * - * **Technical Implementation:** - * - Maintains a `downloads` Map to track active and paused downloads - * - Successful downloads are automatically removed from the Map - * - Uses `ResourceSourceExtended` interface for pause/resume functionality with linked-list behavior */ +import { + deleteAsync, + readDirectoryAsync, + readAsStringAsync, + moveAsync, +} from 'expo-file-system/legacy'; +import { RNEDirectory } from './constants/directories'; import { ResourceSource, ResourceFetcherAdapter, - DownloadStatus, - SourceType, - RnExecutorchError, RnExecutorchErrorCode, + RnExecutorchError, } from 'react-native-executorch'; import { - ResourceSourceExtended, - DownloadResource, ResourceFetcherUtils, + HTTP_CODE, + DownloadStatus, + SourceType, } from './ResourceFetcherUtils'; import { - cacheDirectory, - createDownloadResumable, - DownloadResumable, -} from 'expo-file-system/legacy'; -import { RNEDirectory } from './constants/directories'; + type ActiveDownload, + handleObject, + handleLocalFile, + handleAsset, + handleRemote, +} from './handlers'; -export interface DownloadMetadata { - source: ResourceSource; - type: SourceType; - remoteUri?: string; - localUri: string; - onDownloadProgress?: (downloadProgress: number) => void; -} +class ExpoResourceFetcherClass implements ResourceFetcherAdapter { + private downloads = new Map(); -export interface DownloadAsset { - downloadObj: DownloadResumable; - status: DownloadStatus; - metadata: DownloadMetadata; -} + /** + * Fetches resources (remote URLs, local files or embedded assets), downloads or stores them locally + * for use by React Native ExecuTorch. + * + * @param callback - Optional callback to track progress of all downloads, reported between 0 and 1. + * @param sources - Multiple resources that can be strings, asset references, or objects. + * @returns If the fetch was successful, resolves to an array of local file paths for the + * downloaded/stored resources (without file:// prefix). + * If the fetch was interrupted by `pauseFetching` or `cancelFetching`, resolves to `null`. + */ + async fetch( + callback: (downloadProgress: number) => void = () => {}, + ...sources: ResourceSource[] + ): Promise { + if (sources.length === 0) { + throw new RnExecutorchError( + RnExecutorchErrorCode.InvalidUserInput, + 'Empty list given as an argument to Resource Fetcher' + ); + } -interface ExpoResourceFetcherInterface extends ResourceFetcherAdapter { - downloadAssets: Record; - singleFetch(source: ResourceSource): Promise; - returnOrStartNext( - sourceExtended: ResourceSourceExtended, - result: string | string[] - ): string[] | Promise; - pause(source: ResourceSource): Promise; - resume(source: ResourceSource): Promise; - cancel(source: ResourceSource): Promise; - findActive(sources: ResourceSource[]): ResourceSource; - pauseFetching(...sources: ResourceSource[]): Promise; - resumeFetching(...sources: ResourceSource[]): Promise; - cancelFetching(...sources: ResourceSource[]): Promise; - listDownloadedFiles(): Promise; - listDownloadedModels(): Promise; - deleteResources(...sources: ResourceSource[]): Promise; - getFilesTotalSize(...sources: ResourceSource[]): Promise; - handleObject(source: ResourceSource): Promise; - handleLocalFile(source: ResourceSource): string; - handleReleaseModeFile(source: DownloadAsset): Promise; - handleDevModeFile(source: DownloadAsset): Promise; - handleRemoteFile(source: DownloadAsset): Promise; - handleCachedDownload(source: ResourceSource): Promise; -} + const { results: info, totalLength } = + await ResourceFetcherUtils.getFilesSizes(sources); + // Key by source so we can look up progress info without relying on index alignment + // (getFilesSizes skips sources whose HEAD request fails) + const infoMap = new Map(info.map((entry) => [entry.source, entry])); + const results: string[] = []; -export const ExpoResourceFetcher: ExpoResourceFetcherInterface = { - downloadAssets: {}, + for (const source of sources) { + const fileInfo = infoMap.get(source); + const progressCallback = + fileInfo?.type === SourceType.REMOTE_FILE + ? ResourceFetcherUtils.calculateDownloadProgress( + totalLength, + fileInfo.previousFilesTotalLength, + fileInfo.length, + callback + ) + : () => {}; - async singleFetch(source: DownloadAsset): Promise { - switch (source.metadata.type) { - case SourceType.OBJECT: { - return await this.handleObject(source); - } - case SourceType.LOCAL_FILE: { - return this.handleLocalFile(source); - } - case SourceType.RELEASE_MODE_FILE: { - return this.handleReleaseModeFile(source); - } - case SourceType.DEV_MODE_FILE: { - return this.handleDevModeFile(source); - } - case SourceType.REMOTE_FILE: { - return this.handleRemoteFile(source); - } + const path = await this.fetchOne(source, progressCallback); + if (path === null) return null; + results.push(path); } - }, - async handleCachedDownload(source: ResourceSource) {}, + return results; + } - async handleRemoteFile(source: ResourceSource) { - if (typeof source === 'object') { - throw new RnExecutorchError( - RnExecutorchErrorCode.InvalidModelSource, - 'Model source is expected to be a string or a number.' + /** + * Reads the contents of a file as a string. + * + * @param path - Absolute file path or file URI to read. + * @returns A promise that resolves to the file contents as a string. + */ + async readAsString(path: string): Promise { + const uri = path.startsWith('file://') ? path : `file://${path}`; + return readAsStringAsync(uri); + } + + /** + * Pauses an ongoing download of files. + * + * @param sources - The resource identifiers used when calling `fetch`. + * @returns A promise that resolves once the download is paused. + */ + async pauseFetching(...sources: ResourceSource[]): Promise { + const source = this.findActive(sources); + await this.pause(source); + } + + /** + * Resumes a paused download of files. + * + * The result of the resumed download flows back through the original `fetch` promise. + * + * @param sources - The resource identifiers used when calling `fetch`. + * @returns A promise that resolves once the resume handoff is complete. + */ + async resumeFetching(...sources: ResourceSource[]): Promise { + const source = this.findActive(sources); + await this.resume(source); + } + + /** + * Cancels an ongoing/paused download of files. + * + * @param sources - The resource identifiers used when calling `fetch()`. + * @returns A promise that resolves once the download is canceled. + */ + async cancelFetching(...sources: ResourceSource[]): Promise { + const source = this.findActive(sources); + await this.cancel(source); + } + + /** + * Lists all the downloaded files used by React Native ExecuTorch. + * + * @returns A promise that resolves to an array of URIs for all the downloaded files. + */ + async listDownloadedFiles(): Promise { + const files = await readDirectoryAsync(RNEDirectory); + return files.map((file: string) => `${RNEDirectory}${file}`); + } + + /** + * Lists all the downloaded models used by React Native ExecuTorch. + * + * @returns A promise that resolves to an array of URIs for all the downloaded models. + */ + async listDownloadedModels(): Promise { + const files = await this.listDownloadedFiles(); + return files.filter((file: string) => file.endsWith('.pte')); + } + + /** + * Deletes downloaded resources from the local filesystem. + * + * @param sources - The resource identifiers used when calling `fetch`. + * @returns A promise that resolves once all specified resources have been removed. + */ + async deleteResources(...sources: ResourceSource[]): Promise { + for (const source of sources) { + const filename = ResourceFetcherUtils.getFilenameFromUri( + source as string ); + const fileUri = `${RNEDirectory}${filename}`; + if (await ResourceFetcherUtils.checkFileExists(fileUri)) { + await deleteAsync(fileUri); + } } - if (this.downloadAssets[source]) { - return this.handleCachedDownload(source); - } + } - const uri = 'asda'; - const targetFilename = ResourceFetcherUtils.getFilenameFromUri(uri); - const fileUri = `${RNEDirectory}${targetFilename}`; - const cacheFileUri = `${cacheDirectory}${targetFilename}`; + /** + * Fetches the total size of remote files. Works only for remote files. + * + * @param sources - The resource identifiers (URLs). + * @returns A promise that resolves to the combined size of files in bytes. + */ + async getFilesTotalSize(...sources: ResourceSource[]): Promise { + return (await ResourceFetcherUtils.getFilesSizes(sources)).totalLength; + } - if (await ResourceFetcherUtils.checkFileExists(fileUri)) { - return ResourceFetcherUtils.removeFilePrefix(fileUri); + private async fetchOne( + source: ResourceSource, + progressCallback: (progress: number) => void + ): Promise { + const type = ResourceFetcherUtils.getType(source); + switch (type) { + case SourceType.OBJECT: + return handleObject(source as object); + case SourceType.LOCAL_FILE: + return handleLocalFile(source as string); + case SourceType.RELEASE_MODE_FILE: + case SourceType.DEV_MODE_FILE: + return handleAsset(source as number, progressCallback, this.downloads); + default: // REMOTE_FILE + return handleRemote( + source as string, + source, + progressCallback, + this.downloads + ); } + } - await ResourceFetcherUtils.createDirectoryIfNoExists(); - - const downloadResumable = createDownloadResumable(uri, fileUri); - }, + private async pause(source: ResourceSource): Promise { + const dl = this.downloads.get(source)!; + if (dl.status === DownloadStatus.PAUSED) { + throw new RnExecutorchError( + RnExecutorchErrorCode.ResourceFetcherAlreadyPaused, + "The file download is currently paused. Can't pause the download of the same file twice." + ); + } + dl.status = DownloadStatus.PAUSED; + await dl.downloadResumable.pauseAsync(); + } - async fetch( - callback: (downloadProgress: number) => void = () => {}, - ...sources: ResourceSource[] - ) { - if (sources.length === 0) { + private async resume(source: ResourceSource): Promise { + const dl = this.downloads.get(source)!; + if (dl.status === DownloadStatus.ONGOING) { throw new RnExecutorchError( - RnExecutorchErrorCode.InvalidUserInput, - "Empty list given as an argument to Resource Fetcher's fetch() function!" + RnExecutorchErrorCode.ResourceFetcherAlreadyOngoing, + "The file download is currently ongoing. Can't resume the ongoing download." ); } + dl.status = DownloadStatus.ONGOING; + const result = await dl.downloadResumable.resumeAsync(); + const current = this.downloads.get(source); + // Paused again or canceled during resume — settle/reject handled elsewhere. + if (!current || current.status === DownloadStatus.PAUSED) return; - const totalFilesLength = (await ResourceFetcherUtils.getFilesSizes(sources)) - .totalLength; - const downloadedBytesCounter = 0; - for (let source of sources) { - const downloadResult = await this.singleFetch(source); + if ( + !result || + (result.status !== HTTP_CODE.OK && + result.status !== HTTP_CODE.PARTIAL_CONTENT) + ) { + this.downloads.delete(source); + // Propagate the failure through the original fetch() promise. + dl.reject( + new RnExecutorchError( + RnExecutorchErrorCode.ResourceFetcherDownloadFailed, + `Failed to resume download from '${dl.uri}', status: ${result?.status}` + ) + ); + return; } - return ['']; - }, -}; + + await moveAsync({ from: dl.cacheFileUri, to: dl.fileUri }); + this.downloads.delete(source); + ResourceFetcherUtils.triggerHuggingFaceDownloadCounter(dl.uri); + dl.settle(ResourceFetcherUtils.removeFilePrefix(dl.fileUri)); + } + + private async cancel(source: ResourceSource): Promise { + const dl = this.downloads.get(source)!; + await dl.downloadResumable.cancelAsync(); + this.downloads.delete(source); + dl.settle(null); + } + + private findActive(sources: ResourceSource[]): ResourceSource { + for (const source of sources) { + if (this.downloads.has(source)) { + return source; + } + } + throw new RnExecutorchError( + RnExecutorchErrorCode.ResourceFetcherNotActive, + 'None of given sources are currently during downloading process.' + ); + } +} + +export const ExpoResourceFetcher = new ExpoResourceFetcherClass(); diff --git a/packages/expo-resource-fetcher/src/handlers.ts b/packages/expo-resource-fetcher/src/handlers.ts new file mode 100644 index 0000000000..a13361bf2a --- /dev/null +++ b/packages/expo-resource-fetcher/src/handlers.ts @@ -0,0 +1,179 @@ +import { + cacheDirectory, + copyAsync, + createDownloadResumable, + moveAsync, + FileSystemSessionType, + writeAsStringAsync, + EncodingType, + type DownloadResumable, +} from 'expo-file-system/legacy'; +import { Asset } from 'expo-asset'; +import { Platform } from 'react-native'; +import { + ResourceSource, + RnExecutorchErrorCode, + RnExecutorchError, +} from 'react-native-executorch'; +import { RNEDirectory } from './constants/directories'; +import { + ResourceFetcherUtils, + HTTP_CODE, + DownloadStatus, +} from './ResourceFetcherUtils'; + +export interface ActiveDownload { + downloadResumable: DownloadResumable; + status: DownloadStatus; + uri: string; + fileUri: string; + cacheFileUri: string; + /** Resolves the pending Promise inside the fetch() loop. */ + settle: (path: string | null) => void; + /** Rejects the pending Promise inside the fetch() loop. */ + reject: (error: unknown) => void; +} + +export async function handleObject(source: object): Promise { + const jsonString = JSON.stringify(source); + const digest = ResourceFetcherUtils.hashObject(jsonString); + const path = `${RNEDirectory}${digest}.json`; + + if (await ResourceFetcherUtils.checkFileExists(path)) { + return ResourceFetcherUtils.removeFilePrefix(path); + } + + await ResourceFetcherUtils.createDirectoryIfNoExists(); + await writeAsStringAsync(path, jsonString, { encoding: EncodingType.UTF8 }); + return ResourceFetcherUtils.removeFilePrefix(path); +} + +export function handleLocalFile(source: string): string { + return ResourceFetcherUtils.removeFilePrefix(source); +} + +export async function handleAsset( + source: number, + progressCallback: (progress: number) => void, + downloads: Map +): Promise { + const asset = Asset.fromModule(source); + const uri = asset.uri; + + if (uri.startsWith('http')) { + // Dev mode: asset served from Metro dev server + return handleRemote(uri, source, progressCallback, downloads); + } + + // Release mode: asset bundled locally, copy to RNEDirectory + const filename = ResourceFetcherUtils.getFilenameFromUri(uri); + const fileUri = `${RNEDirectory}${filename}`; + // On Android, the bundled URI has no extension, so we append it manually + const fileUriWithType = + Platform.OS === 'android' ? `${fileUri}.${asset.type}` : fileUri; + + if (await ResourceFetcherUtils.checkFileExists(fileUri)) { + return ResourceFetcherUtils.removeFilePrefix(fileUri); + } + + await ResourceFetcherUtils.createDirectoryIfNoExists(); + await copyAsync({ from: uri, to: fileUriWithType }); + return ResourceFetcherUtils.removeFilePrefix(fileUriWithType); +} + +export function handleRemote( + uri: string, + source: ResourceSource, + progressCallback: (progress: number) => void, + downloads: Map +): Promise { + if (downloads.has(source)) { + throw new RnExecutorchError( + RnExecutorchErrorCode.ResourceFetcherDownloadInProgress, + 'Already downloading this file' + ); + } + + let settle!: (path: string | null) => void; + let reject!: (error: unknown) => void; + + const promise = new Promise((res, rej) => { + settle = res; + reject = rej; + }); + + void (async () => { + const filename = ResourceFetcherUtils.getFilenameFromUri(uri); + const fileUri = `${RNEDirectory}${filename}`; + const cacheFileUri = `${cacheDirectory}${filename}`; + + try { + if (await ResourceFetcherUtils.checkFileExists(fileUri)) { + settle(ResourceFetcherUtils.removeFilePrefix(fileUri)); + return; + } + + await ResourceFetcherUtils.createDirectoryIfNoExists(); + + const downloadResumable = createDownloadResumable( + uri, + cacheFileUri, + { sessionType: FileSystemSessionType.BACKGROUND }, + ({ + totalBytesWritten, + totalBytesExpectedToWrite, + }: { + totalBytesWritten: number; + totalBytesExpectedToWrite: number; + }) => { + if (totalBytesExpectedToWrite === -1) { + progressCallback(0); + } else { + progressCallback(totalBytesWritten / totalBytesExpectedToWrite); + } + } + ); + + downloads.set(source, { + downloadResumable, + status: DownloadStatus.ONGOING, + uri, + fileUri, + cacheFileUri, + settle, + reject, + }); + + const result = await downloadResumable.downloadAsync(); + const dl = downloads.get(source); + // If paused or canceled during the download, settle/reject will be called + // externally by resume() or cancel() — do nothing here. + if (!dl || dl.status === DownloadStatus.PAUSED) return; + + if ( + !result || + (result.status !== HTTP_CODE.OK && + result.status !== HTTP_CODE.PARTIAL_CONTENT) + ) { + downloads.delete(source); + reject( + new RnExecutorchError( + RnExecutorchErrorCode.ResourceFetcherDownloadFailed, + `Failed to fetch resource from '${uri}', status: ${result?.status}` + ) + ); + return; + } + + await moveAsync({ from: cacheFileUri, to: fileUri }); + downloads.delete(source); + ResourceFetcherUtils.triggerHuggingFaceDownloadCounter(uri); + settle(ResourceFetcherUtils.removeFilePrefix(fileUri)); + } catch (error) { + downloads.delete(source); + reject(error); + } + })(); + + return promise; +}