Skip to content

Commit b452de8

Browse files
bhamiltoncxwcandillon
bhamiltoncx
and
wcandillon
authored
feat(🥷): Update createImageBitmap() to accept ArrayBuffer or Blob (#318)
* feat: Update createImageBitmap() to accept ArrayBuffer or Blob On Android, React Native 0.82 and later suffers from a serious open issue where `file://` URIs to local assets cannot be loaded with `fetch()`: facebook/react-native#54626 That means it's very difficult to load bundled texture assets to use with `react-native-wgpu` in release builds on Android. Develompent builds work fine, since assets are loaded over HTTP, but release builds directly include assets which resolve to local `file://` URIs like `file:///android_res/drawable/assets_textures_foo.jpg`. Because `react-native-wgpu`'s `createImageBitmap()` currently requires a `Blob`, there's no good workaround for the React Native issue, since the only thing which can create `Blob`s is `fetch()`. (Note: I tried using the workaround from expo/expo#2402 (comment) to create an `XMLHttpRequest` to fetch the `file://` URI, but it fails in React Native 0.82 and later as well.) To resolve this issue, this PR extends `react-native-wgpu`'s `createImageBitmap()` to accept either `Blob` *or* `ArrayBuffer`s containing encoded image bytes (PNG/JPEG/etc.). In specific, this PR: - Upgrades the C++ version to C++20 for `std::span` support to avoid copies - Adds `createImageBitmapFromData(std::span<uint8_t>)` to `PlatformContext` - Refactors the existing `Blob` codepaths to resolve the `Blob` to bytes, then use the `std::span` codepath - Detects `ArrayBuffer` vs `Blob` in `RNWebGPU::createImageBitmap` at runtime - Adds a TypeScript global overload to allow calling `createImageBitmap(ArrayBuffer)` - Updates the `TexturedCube` sample to use the new `ArrayBuffer` codepath I tested this by running the Android and iOS samples in the simulator and on device. I did use Claude Code to help draft this PR, but then edited the code myself and tested it by hand (I promise I'm a human being). --------- Co-authored-by: William Candillon <wcandillon@gmail.com>
1 parent e4f07ab commit b452de8

14 files changed

Lines changed: 353 additions & 224 deletions

File tree

‎apps/example/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"pod:install:ios": "pod install --project-directory=ios",
1313
"pod:install:macos": "pod install --project-directory=macos",
1414
"build:android": "cd android && ./gradlew assembleDebug --warning-mode all",
15-
"build:ios": "react-native build-ios --scheme Example --mode Debug --extra-params \"-sdk iphonesimulator CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ GCC_OPTIMIZATION_LEVEL=0 GCC_PRECOMPILE_PREFIX_HEADER=YES ASSETCATALOG_COMPILER_OPTIMIZATION=time DEBUG_INFORMATION_FORMAT=dwarf COMPILER_INDEX_STORE_ENABLE=NO\"",
15+
"build:ios": "react-native build-ios --scheme Example --mode Debug --extra-params \"-sdk iphonesimulator CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ GCC_OPTIMIZATION_LEVEL=0 GCC_PRECOMPILE_PREFIX_HEADER=YES ASSETCATALOG_COMPILER_OPTIMIZATION=time DEBUG_INFORMATION_FORMAT=dwarf COMPILER_INDEX_STORE_ENABLE=NO CLANG_CXX_LANGUAGE_STANDARD=c++20\"",
1616
"build:macos": "react-native build-macos --scheme Example --mode Debug",
1717
"mkdist": "node -e \"require('node:fs').mkdirSync('dist', { recursive: true, mode: 0o755 })\"",
1818
"postinstall": "node -e \"if (process.platform !== 'darwin') { console.log('Skipping iOS pod install on non-macOS environment.'); process.exit(0); } const { execSync } = require('child_process'); execSync('yarn pod:install:ios', { stdio: 'inherit' });\""

‎apps/example/src/Cube/TexturedCube.tsx‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,10 @@ export const TexturedCube = () => {
102102

103103
// Fetch the image and upload it into a GPUTexture.
104104
let cubeTexture: GPUTexture;
105-
{
106-
const response = await fetchAsset(require("../assets/Di-3d.png"));
107-
const imageBitmap = await createImageBitmap(await response.blob());
105+
try {
106+
const asset = await fetchAsset(require("../assets/Di-3d.png"));
107+
const arrayBuffer = await asset.arrayBuffer();
108+
const imageBitmap = await createImageBitmap(arrayBuffer);
108109
cubeTexture = device.createTexture({
109110
size: [imageBitmap.width, imageBitmap.height, 1],
110111
format: "rgba8unorm",
@@ -116,8 +117,11 @@ export const TexturedCube = () => {
116117
device.queue.copyExternalImageToTexture(
117118
{ source: imageBitmap },
118119
{ texture: cubeTexture },
119-
[imageBitmap.width, imageBitmap.height],
120+
[imageBitmap.width, imageBitmap.height]
120121
);
122+
} catch (err) {
123+
console.error("Failed to fetch asset", err);
124+
throw err;
121125
}
122126

123127
// Create a sampler with linear filtering for smooth interpolation.

‎packages/webgpu/android/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ cmake_minimum_required(VERSION 3.4.1)
22
project(RNWGPU)
33

44
set (CMAKE_VERBOSE_MAKEFILE ON)
5-
set (CMAKE_CXX_STANDARD 17)
5+
set (CMAKE_CXX_STANDARD 20)
6+
set (CMAKE_CXX_STANDARD_REQUIRED True)
67

78
set (PACKAGE_NAME "react-native-wgpu")
89

‎packages/webgpu/android/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ android {
7171
buildConfigField "boolean", "IS_NEW_ARCHITECTURE_ENABLED", "true"
7272
externalNativeBuild {
7373
cmake {
74-
cppFlags "-fexceptions", "-frtti", "-std=c++1y", "-DONANDROID"
74+
cppFlags "-fexceptions", "-frtti", "-DONANDROID"
7575
abiFilters (*reactNativeArchitectures())
7676
arguments '-DANDROID_STL=c++_shared',
7777
"-DNODE_MODULES_DIR=${nodeModules}",

‎packages/webgpu/android/cpp/AndroidPlatformContext.h‎

Lines changed: 104 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,43 @@ class AndroidPlatformContext : public PlatformContext {
2323
private:
2424
jobject _blobModule;
2525

26+
std::vector<uint8_t> resolveBlob(JNIEnv *env, const std::string &blobId,
27+
double offset, double size) {
28+
if (!_blobModule) {
29+
throw std::runtime_error("BlobModule instance is null");
30+
}
31+
32+
jclass blobModuleClass = env->GetObjectClass(_blobModule);
33+
if (!blobModuleClass) {
34+
throw std::runtime_error("Couldn't find BlobModule class");
35+
}
36+
37+
jmethodID resolveMethod = env->GetMethodID(blobModuleClass, "resolve",
38+
"(Ljava/lang/String;II)[B");
39+
env->DeleteLocalRef(blobModuleClass);
40+
41+
if (!resolveMethod) {
42+
throw std::runtime_error("Couldn't find resolve method in BlobModule");
43+
}
44+
45+
jstring jBlobId = env->NewStringUTF(blobId.c_str());
46+
jbyteArray blobData = (jbyteArray)env->CallObjectMethod(
47+
_blobModule, resolveMethod, jBlobId, static_cast<jint>(offset),
48+
static_cast<jint>(size));
49+
env->DeleteLocalRef(jBlobId);
50+
51+
if (!blobData) {
52+
throw std::runtime_error("Couldn't retrieve blob data");
53+
}
54+
55+
jsize len = env->GetArrayLength(blobData);
56+
std::vector<uint8_t> data(len);
57+
env->GetByteArrayRegion(blobData, 0, len,
58+
reinterpret_cast<jbyte *>(data.data()));
59+
env->DeleteLocalRef(blobData);
60+
return data;
61+
}
62+
2663
public:
2764
explicit AndroidPlatformContext(jobject blobModule)
2865
: _blobModule(blobModule) {}
@@ -52,188 +89,116 @@ class AndroidPlatformContext : public PlatformContext {
5289
throw std::runtime_error("Couldn't get JNI environment");
5390
}
5491

55-
// Use the BlobModule instance from _blobModule
56-
if (!_blobModule) {
57-
throw std::runtime_error("BlobModule instance is null");
58-
}
92+
auto data = resolveBlob(env, blobId, offset, size);
93+
return createImageBitmapFromData(data);
94+
}
5995

60-
// Get the resolve method ID
61-
jclass blobModuleClass = env->GetObjectClass(_blobModule);
62-
if (!blobModuleClass) {
63-
throw std::runtime_error("Couldn't find BlobModule class");
64-
}
96+
void createImageBitmapAsync(
97+
std::string blobId, double offset, double size,
98+
std::function<void(ImageData)> onSuccess,
99+
std::function<void(std::string)> onError) override {
100+
std::thread([this, blobId = std::move(blobId), offset, size,
101+
onSuccess = std::move(onSuccess),
102+
onError = std::move(onError)]() {
103+
jni::Environment::ensureCurrentThreadIsAttached();
104+
try {
105+
JNIEnv *env = facebook::jni::Environment::current();
106+
if (!env) {
107+
throw std::runtime_error("Couldn't get JNI environment");
108+
}
109+
auto data = resolveBlob(env, blobId, offset, size);
110+
auto result = createImageBitmapFromData(data);
111+
onSuccess(std::move(result));
112+
} catch (const std::exception &e) {
113+
onError(e.what());
114+
}
115+
}).detach();
116+
}
65117

66-
jmethodID resolveMethod = env->GetMethodID(blobModuleClass, "resolve",
67-
"(Ljava/lang/String;II)[B");
68-
if (!resolveMethod) {
69-
throw std::runtime_error("Couldn't find resolve method in BlobModule");
70-
}
118+
ImageData createImageBitmapFromData(std::span<const uint8_t> data) override {
119+
jni::Environment::ensureCurrentThreadIsAttached();
71120

72-
// Resolve the blob data
73-
jstring jBlobId = env->NewStringUTF(blobId.c_str());
74-
jbyteArray blobData = (jbyteArray)env->CallObjectMethod(
75-
_blobModule, resolveMethod, jBlobId, static_cast<jint>(offset),
76-
static_cast<jint>(size));
77-
env->DeleteLocalRef(jBlobId);
121+
JNIEnv *env = facebook::jni::Environment::current();
122+
if (!env) {
123+
throw std::runtime_error("Couldn't get JNI environment");
124+
}
78125

79-
if (!blobData) {
80-
throw std::runtime_error("Couldn't retrieve blob data");
126+
// Create jbyteArray from the raw bytes
127+
jbyteArray byteArray = env->NewByteArray(static_cast<jsize>(data.size()));
128+
if (!byteArray) {
129+
throw std::runtime_error("Couldn't allocate byte array");
81130
}
131+
env->SetByteArrayRegion(byteArray, 0, static_cast<jsize>(data.size()),
132+
reinterpret_cast<const jbyte *>(data.data()));
82133

83-
// Create a Bitmap from the blob data
134+
// Decode via BitmapFactory
84135
jclass bitmapFactoryClass =
85136
env->FindClass("android/graphics/BitmapFactory");
137+
if (!bitmapFactoryClass) {
138+
env->DeleteLocalRef(byteArray);
139+
throw std::runtime_error("Couldn't find BitmapFactory class");
140+
}
86141
jmethodID decodeByteArrayMethod =
87142
env->GetStaticMethodID(bitmapFactoryClass, "decodeByteArray",
88143
"([BII)Landroid/graphics/Bitmap;");
89-
jint blobLength = env->GetArrayLength(blobData);
144+
if (!decodeByteArrayMethod) {
145+
env->DeleteLocalRef(byteArray);
146+
env->DeleteLocalRef(bitmapFactoryClass);
147+
throw std::runtime_error("Couldn't find decodeByteArray method");
148+
}
149+
jint length = static_cast<jint>(data.size());
90150
jobject bitmap = env->CallStaticObjectMethod(
91-
bitmapFactoryClass, decodeByteArrayMethod, blobData, 0, blobLength);
151+
bitmapFactoryClass, decodeByteArrayMethod, byteArray, 0, length);
152+
env->DeleteLocalRef(bitmapFactoryClass);
92153

93154
if (!bitmap) {
94-
env->DeleteLocalRef(blobData);
155+
env->DeleteLocalRef(byteArray);
95156
throw std::runtime_error("Couldn't decode image");
96157
}
97158

98-
// Get bitmap info
99159
AndroidBitmapInfo bitmapInfo;
100160
if (AndroidBitmap_getInfo(env, bitmap, &bitmapInfo) !=
101161
ANDROID_BITMAP_RESULT_SUCCESS) {
102-
env->DeleteLocalRef(blobData);
162+
env->DeleteLocalRef(byteArray);
103163
env->DeleteLocalRef(bitmap);
104164
throw std::runtime_error("Couldn't get bitmap info");
105165
}
106166

107-
// Lock the bitmap pixels
108167
void *bitmapPixels;
109168
if (AndroidBitmap_lockPixels(env, bitmap, &bitmapPixels) !=
110169
ANDROID_BITMAP_RESULT_SUCCESS) {
111-
env->DeleteLocalRef(blobData);
170+
env->DeleteLocalRef(byteArray);
112171
env->DeleteLocalRef(bitmap);
113172
throw std::runtime_error("Couldn't lock bitmap pixels");
114173
}
115174

116-
// Copy the bitmap data
117-
std::vector<uint8_t> imageData(bitmapInfo.height * bitmapInfo.stride);
118-
memcpy(imageData.data(), bitmapPixels, imageData.size());
175+
ImageData result;
176+
result.width = static_cast<int>(bitmapInfo.width);
177+
result.height = static_cast<int>(bitmapInfo.height);
178+
result.data.resize(bitmapInfo.height * bitmapInfo.stride);
179+
memcpy(result.data.data(), bitmapPixels, result.data.size());
119180

120-
// Unlock the bitmap pixels
121181
AndroidBitmap_unlockPixels(env, bitmap);
122182

123-
// Clean up JNI references
124-
env->DeleteLocalRef(blobData);
183+
env->DeleteLocalRef(byteArray);
125184
env->DeleteLocalRef(bitmap);
126185

127-
ImageData result;
128-
result.width = static_cast<int>(bitmapInfo.width);
129-
result.height = static_cast<int>(bitmapInfo.height);
130-
result.data = imageData;
131186
return result;
132187
}
133188

134-
void createImageBitmapAsync(
135-
std::string blobId, double offset, double size,
136-
std::function<void(ImageData)> onSuccess,
189+
void createImageBitmapFromDataAsync(
190+
std::span<const uint8_t> data, std::function<void(ImageData)> onSuccess,
137191
std::function<void(std::string)> onError) override {
138-
// Capture blobModule for the background thread
139-
jobject blobModule = _blobModule;
140-
141-
// Dispatch to a background thread
142-
std::thread([blobModule, blobId = std::move(blobId), offset, size,
192+
std::thread([this, ownedData = std::vector<uint8_t>(data.begin(), data.end()),
143193
onSuccess = std::move(onSuccess),
144-
onError = std::move(onError)]() {
194+
onError = std::move(onError)]() mutable {
145195
jni::Environment::ensureCurrentThreadIsAttached();
146-
147-
JNIEnv *env = facebook::jni::Environment::current();
148-
if (!env) {
149-
onError("Couldn't get JNI environment");
150-
return;
151-
}
152-
153-
if (!blobModule) {
154-
onError("BlobModule instance is null");
155-
return;
156-
}
157-
158-
// Get the resolve method ID
159-
jclass blobModuleClass = env->GetObjectClass(blobModule);
160-
if (!blobModuleClass) {
161-
onError("Couldn't find BlobModule class");
162-
return;
163-
}
164-
165-
jmethodID resolveMethod = env->GetMethodID(blobModuleClass, "resolve",
166-
"(Ljava/lang/String;II)[B");
167-
if (!resolveMethod) {
168-
onError("Couldn't find resolve method in BlobModule");
169-
return;
170-
}
171-
172-
// Resolve the blob data
173-
jstring jBlobId = env->NewStringUTF(blobId.c_str());
174-
jbyteArray blobData = (jbyteArray)env->CallObjectMethod(
175-
blobModule, resolveMethod, jBlobId, static_cast<jint>(offset),
176-
static_cast<jint>(size));
177-
env->DeleteLocalRef(jBlobId);
178-
179-
if (!blobData) {
180-
onError("Couldn't retrieve blob data");
181-
return;
196+
try {
197+
auto result = createImageBitmapFromData(ownedData);
198+
onSuccess(std::move(result));
199+
} catch (const std::exception &e) {
200+
onError(e.what());
182201
}
183-
184-
// Create a Bitmap from the blob data
185-
jclass bitmapFactoryClass =
186-
env->FindClass("android/graphics/BitmapFactory");
187-
jmethodID decodeByteArrayMethod =
188-
env->GetStaticMethodID(bitmapFactoryClass, "decodeByteArray",
189-
"([BII)Landroid/graphics/Bitmap;");
190-
jint blobLength = env->GetArrayLength(blobData);
191-
jobject bitmap = env->CallStaticObjectMethod(
192-
bitmapFactoryClass, decodeByteArrayMethod, blobData, 0, blobLength);
193-
194-
if (!bitmap) {
195-
env->DeleteLocalRef(blobData);
196-
onError("Couldn't decode image");
197-
return;
198-
}
199-
200-
// Get bitmap info
201-
AndroidBitmapInfo bitmapInfo;
202-
if (AndroidBitmap_getInfo(env, bitmap, &bitmapInfo) !=
203-
ANDROID_BITMAP_RESULT_SUCCESS) {
204-
env->DeleteLocalRef(blobData);
205-
env->DeleteLocalRef(bitmap);
206-
onError("Couldn't get bitmap info");
207-
return;
208-
}
209-
210-
// Lock the bitmap pixels
211-
void *bitmapPixels;
212-
if (AndroidBitmap_lockPixels(env, bitmap, &bitmapPixels) !=
213-
ANDROID_BITMAP_RESULT_SUCCESS) {
214-
env->DeleteLocalRef(blobData);
215-
env->DeleteLocalRef(bitmap);
216-
onError("Couldn't lock bitmap pixels");
217-
return;
218-
}
219-
220-
// Copy the bitmap data
221-
std::vector<uint8_t> imageData(bitmapInfo.height * bitmapInfo.stride);
222-
memcpy(imageData.data(), bitmapPixels, imageData.size());
223-
224-
// Unlock the bitmap pixels
225-
AndroidBitmap_unlockPixels(env, bitmap);
226-
227-
// Clean up JNI references
228-
env->DeleteLocalRef(blobData);
229-
env->DeleteLocalRef(bitmap);
230-
231-
ImageData result;
232-
result.width = static_cast<int>(bitmapInfo.width);
233-
result.height = static_cast<int>(bitmapInfo.height);
234-
result.data = std::move(imageData);
235-
236-
onSuccess(std::move(result));
237202
}).detach();
238203
}
239204
};

‎packages/webgpu/apple/ApplePlatformContext.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@ class ApplePlatformContext : public PlatformContext {
2020
std::string blobId, double offset, double size,
2121
std::function<void(ImageData)> onSuccess,
2222
std::function<void(std::string)> onError) override;
23+
24+
ImageData createImageBitmapFromData(std::span<const uint8_t> data) override;
25+
26+
void createImageBitmapFromDataAsync(
27+
std::span<const uint8_t> data, std::function<void(ImageData)> onSuccess,
28+
std::function<void(std::string)> onError) override;
2329
};
2430

2531
} // namespace rnwgpu

0 commit comments

Comments
 (0)