diff --git a/LumenIO/.github/workflows/android.yml b/LumenIO/.github/workflows/android.yml new file mode 100644 index 0000000..acdcaf1 --- /dev/null +++ b/LumenIO/.github/workflows/android.yml @@ -0,0 +1,13 @@ +name: android-build +on: [push, pull_request] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: { distribution: 'temurin', java-version: '17' } + - name: Gradle build + uses: gradle/gradle-build-action@v3 + with: + arguments: :apps:mobile:android:app:assembleDebug diff --git a/LumenIO/README.md b/LumenIO/README.md new file mode 100644 index 0000000..4e2bb9e --- /dev/null +++ b/LumenIO/README.md @@ -0,0 +1,29 @@ +# LumenIO – Mobile (Android) Slice + +**What:** Device-responsive 3D platform (parent system). This slice delivers **mobile**: CameraX + MediaPipe Hand Landmarker → gesture FSM → Filament PBR renderer. + +## Build +- Android Studio Giraffe+ (JDK 17). +- Put a hand landmark model into `apps/mobile/android/app/src/main/assets/hand_landmarker.task` (see AI Edge Hand Landmarker guide). +- `./gradlew :apps:mobile:android:app:assembleDebug` + +## Run +Launch on a device (front camera). Pinch → rotate demo applies Y-rotation to the loaded glTF model. + +## Deps (sources) +- CameraX 1.4.2 (stable matrix & dependency snippet). +- MediaPipe Tasks – Hand Landmarker (`com.google.mediapipe:tasks-vision`). +- Filament 1.57.1 (`filament-android`, `gltfio-android`, `filament-utils-android`). +Docs cited in code comments. + +--- + +What’s next (tight) + +1. Replace ImageProxy.toBitmap() with proper YUV→RGB, or feed MPImage from ImageProxy without intermediate Bitmap. + +2. Add IBL/lighting in FilamentRenderer (KTX environment, skybox), and a proper frame loop / surface resize handling. + +3. Emit gesture_event JSON to a bus; store deterministic replays; unit tests for FSM thresholds. + +4. Add depth scaling (z from world landmarks) to improve rotate/scale stability. diff --git a/LumenIO/apps/mobile/android/app/build.gradle.kts b/LumenIO/apps/mobile/android/app/build.gradle.kts new file mode 100644 index 0000000..98286e8 --- /dev/null +++ b/LumenIO/apps/mobile/android/app/build.gradle.kts @@ -0,0 +1,40 @@ +plugins { + id("com.android.application") + kotlin("android") +} + +android { + namespace = "io.lumen.mobile" + compileSdk = 34 + defaultConfig { + applicationId = "io.lumen.mobile" + minSdk = 23 + targetSdk = 34 + versionCode = 1 + versionName = "0.1.0" + } + buildTypes { release { isMinifyEnabled = false } } + compileOptions { sourceCompatibility = JavaVersion.VERSION_17; targetCompatibility = JavaVersion.VERSION_17 } + kotlinOptions { jvmTarget = "17" } +} + +dependencies { + // CameraX (use stable; RC exists but keep prod-stable) — per official table + val camerax = "1.4.2" // https://developer.android.com/jetpack/androidx/releases/camera + implementation("androidx.camera:camera-core:$camerax") + implementation("androidx.camera:camera-camera2:$camerax") + implementation("androidx.camera:camera-lifecycle:$camerax") + implementation("androidx.camera:camera-view:$camerax") + + // MediaPipe Tasks – Hand Landmarker (Android) + implementation("com.google.mediapipe:tasks-vision:latest.release") // https://ai.google.dev/edge/mediapipe/solutions/vision/hand_landmarker/android + + // Filament (PBR renderer + glTF loader + utils) — latest release 1.57.1 + val filament = "1.57.1" // https://mvnrepository.com/artifact/com.google.android.filament/filament-android + implementation("com.google.android.filament:filament-android:$filament") + implementation("com.google.android.filament:gltfio-android:$filament") + implementation("com.google.android.filament:filament-utils-android:$filament") + + // KotlinX + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.1") +} diff --git a/LumenIO/apps/mobile/android/app/src/main/AndroidManifest.xml b/LumenIO/apps/mobile/android/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..670c6ee --- /dev/null +++ b/LumenIO/apps/mobile/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,10 @@ + + + + + + + + + diff --git a/LumenIO/apps/mobile/android/app/src/main/assets/models/sample.bin b/LumenIO/apps/mobile/android/app/src/main/assets/models/sample.bin new file mode 100644 index 0000000..e69de29 diff --git a/LumenIO/apps/mobile/android/app/src/main/assets/models/sample.gltf b/LumenIO/apps/mobile/android/app/src/main/assets/models/sample.gltf new file mode 100644 index 0000000..ca61464 --- /dev/null +++ b/LumenIO/apps/mobile/android/app/src/main/assets/models/sample.gltf @@ -0,0 +1,5 @@ +{ + "asset": { "version": "2.0" }, + "scenes": [ { "nodes": [] } ], + "nodes": [] +} diff --git a/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/FilamentRenderer.kt b/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/FilamentRenderer.kt new file mode 100644 index 0000000..f41e82e --- /dev/null +++ b/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/FilamentRenderer.kt @@ -0,0 +1,64 @@ +package io.lumen.mobile + +import android.content.Context +import android.view.Surface +import com.google.android.filament.* +import com.google.android.filament.utils.* +import com.google.android.filament.gltfio.* +import java.nio.ByteBuffer + +class FilamentRenderer(private val ctx: Context) { + private lateinit var engine: Engine + private lateinit var swapChain: SwapChain + private lateinit var renderer: Renderer + private lateinit var scene: Scene + private lateinit var view: View + private lateinit var camera: Camera + private lateinit var assetLoader: AssetLoader + private lateinit var resourceLoader: ResourceLoader + private lateinit var asset: FilamentAsset + + fun init(surface: Surface, width: Int, height: Int) { + engine = Engine.create() + swapChain = engine.createSwapChain(surface) + renderer = engine.createRenderer() + scene = engine.createScene() + view = engine.createView().apply { this.scene = scene } + camera = engine.createCamera(EntityManager.get().create()).apply { + setProjection(45.0, width.toDouble()/height, 0.05, 100.0, Camera.Fov.VERTICAL) + lookAt(0.0, 0.0, 3.0, 0.0, 0.0, 0.0) + } + view.camera = camera + + assetLoader = AssetLoader(engine, UbershaderProvider(engine), EntityManager.get()) + resourceLoader = ResourceLoader(engine, true, true) + } + + fun loadGltfFromAssets(path: String) { + val buf = ctx.assets.open(path).readBytes() + asset = assetLoader.createAssetFromJson(ByteBuffer.wrap(buf)) + resourceLoader.loadResources(asset) + scene.addEntities(asset.entities) + // basic IBL/lighting skipped for brevity; add filament-utils helpers in production + } + + fun applyTransform(tx: FloatArray) { + // tx = 4x4 column-major matrix + val root = asset.root + if (root != 0) TransformManager.getInstance().apply { + val inst = getInstance(root) + setTransform(inst, tx) + } + } + + fun renderFrame() { + if (renderer.beginFrame(swapChain)) { + renderer.render(view) + renderer.endFrame() + } + } + + fun destroy() { + // Destroy all Filament objects (omitted for brevity) + } +} diff --git a/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/GestureFSM.kt b/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/GestureFSM.kt new file mode 100644 index 0000000..36747dc --- /dev/null +++ b/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/GestureFSM.kt @@ -0,0 +1,28 @@ +package io.lumen.mobile + +enum class GState { IDLE, PINCH, DRAG, ROTATE, SCALE } + +data class GestureParams( + val pinch: Float, val x: Float, val y: Float, val angle: Float, val span: Float +) + +class GestureFSM(private val thresh: Float = 0.8f) { + var state = GState.IDLE; private set + private var startX=0f; private var startY=0f; private var startAngle=0f; private var startSpan=0f + + fun step(p: GestureParams): String = when(state) { + GState.IDLE -> if (p.pinch >= thresh) { state = GState.PINCH; startX=p.x; startY=p.y; startAngle=p.angle; startSpan=p.span; "pinch:start" } else "idle" + GState.PINCH -> when { + p.pinch < thresh -> { state = GState.IDLE; "pinch:cancel" } + hypot(p.x-startX, p.y-startY) > 0.04f -> { state = GState.DRAG; "drag:start" } + kotlin.math.abs(p.angle-startAngle) > 0.15f -> { state = GState.ROTATE; "rotate:start" } + kotlin.math.abs(p.span-startSpan) > 0.05f -> { state = GState.SCALE; "scale:start" } + else -> "pinch:hold" + } + GState.DRAG -> if (p.pinch < thresh) { state = GState.IDLE; "drag:end" } else "drag:move" + GState.ROTATE -> if (p.pinch < thresh) { state = GState.IDLE; "rotate:end" } else "rotate:move" + GState.SCALE -> if (p.pinch < thresh) { state = GState.IDLE; "scale:end" } else "scale:move" + } + + private fun hypot(dx: Float, dy: Float) = kotlin.math.sqrt(dx*dx + dy*dy) +} diff --git a/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/HandLandmarkerHelper.kt b/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/HandLandmarkerHelper.kt new file mode 100644 index 0000000..f0bd244 --- /dev/null +++ b/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/HandLandmarkerHelper.kt @@ -0,0 +1,48 @@ +package io.lumen.mobile + +import android.content.Context +import android.os.SystemClock +import com.google.mediapipe.tasks.vision.handlandmarker.HandLandmarker +import com.google.mediapipe.tasks.vision.core.RunningMode +import com.google.mediapipe.tasks.components.containers.Category +import com.google.mediapipe.framework.image.BitmapImageBuilder + +class HandLandmarkerHelper( + private val context: Context, + private val modelAssetPath: String = "hand_landmarker.task", // put model into assets + private val maxHands: Int = 1, + private val minDet: Float = 0.5f, + private val minTrack: Float = 0.5f, + private val minPresence: Float = 0.5f, + private val onResult: (HandResult) -> Unit, + private val onError: (Throwable) -> Unit +) { + private var landmarker: HandLandmarker? = null + + fun setup() { + val base = com.google.mediapipe.tasks.core.BaseOptions.builder().setModelAssetPath(modelAssetPath).build() + val opts = HandLandmarker.HandLandmarkerOptions.builder() + .setBaseOptions(base) + .setRunningMode(RunningMode.LIVE_STREAM) + .setNumHands(maxHands) + .setMinHandDetectionConfidence(minDet) + .setMinTrackingConfidence(minTrack) + .setMinHandPresenceConfidence(minPresence) + .setResultListener { res, inputImg -> + val handed = res.handedness().firstOrNull()?.firstOrNull() ?: Category.create("Unknown", 0, 0f) + val pts = res.landmarks().firstOrNull()?.map { floatArrayOf(it.x(), it.y(), it.z()) } ?: emptyList() + onResult(HandResult(handed.categoryName(), pts)) + } + .setErrorListener { e -> onError(e) } + .build() + landmarker = HandLandmarker.createFromOptions(context, opts) + } + + fun process(bitmap: android.graphics.Bitmap) { + val mpImg = BitmapImageBuilder(bitmap).build() + val ts = SystemClock.uptimeMillis() + landmarker?.detectAsync(mpImg, ts) + } + + data class HandResult(val handedness: String, val points: List) +} diff --git a/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/MainActivity.kt b/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/MainActivity.kt new file mode 100644 index 0000000..05a6d29 --- /dev/null +++ b/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/MainActivity.kt @@ -0,0 +1,95 @@ +package io.lumen.mobile + +import android.graphics.Bitmap +import android.os.Bundle +import android.util.Size +import android.view.SurfaceView +import android.widget.FrameLayout +import androidx.activity.ComponentActivity +import androidx.camera.core.* +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.view.PreviewView +import androidx.core.content.ContextCompat +import java.util.concurrent.Executors + +class MainActivity : ComponentActivity() { + private lateinit var previewView: PreviewView + private lateinit var overlay: OverlayView + private lateinit var renderer: FilamentRenderer + private val exec = Executors.newSingleThreadExecutor() + private lateinit var hand: HandLandmarkerHelper + private val fsm = GestureFSM() + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + previewView = PreviewView(this) + overlay = OverlayView(this) + val root = FrameLayout(this).apply { addView(previewView); addView(overlay) } + setContentView(root) + + hand = HandLandmarkerHelper(this, onResult = { r -> + overlay.points = r.points; overlay.postInvalidate() + // simple pinch heuristic: use landmark 4 (thumb tip) & 8 (index tip) + val pinch = r.points.takeIf { it.size >= 9 }?.let { + val d = hypot(it[4][0]-it[8][0], it[4][1]-it[8][1]) + (1.0f - d).coerceIn(0f,1f) + } ?: 0f + val ev = fsm.step(GestureParams(pinch, 0.5f, 0.5f, 0f, 0f)) + if (ev.startsWith("drag:") || ev.startsWith("rotate:") || ev.startsWith("scale:")) { + // map to a transform; here we rotate around Y slightly as demo + val angle = (System.nanoTime() % 1_000_000_000L) / 1e9f + val mat = yRotation(angle) + renderer.applyTransform(mat) + } + }, onError = { e -> e.printStackTrace() }) + hand.setup() + + startCamera() + setupRenderer() + } + + private fun setupRenderer() { + val surfaceView = SurfaceView(this) + (previewView.parent as FrameLayout).addView(surfaceView) + surfaceView.holder.addCallback(object: android.view.SurfaceHolder.Callback { + override fun surfaceCreated(holder: android.view.SurfaceHolder) { + renderer = FilamentRenderer(this@MainActivity) + renderer.init(holder.surface, surfaceView.width.coerceAtLeast(1), surfaceView.height.coerceAtLeast(1)) + renderer.loadGltfFromAssets("models/sample.gltf") + surfaceView.post(object: Runnable { override fun run() { renderer.renderFrame(); surfaceView.post(this) }}) + } + override fun surfaceChanged(h: android.view.SurfaceHolder, f: Int, w: Int, hgt: Int) {} + override fun surfaceDestroyed(h: android.view.SurfaceHolder) { /* destroy renderer */ } + }) + } + + private fun startCamera() { + val providerFuture = ProcessCameraProvider.getInstance(this) + providerFuture.addListener({ + val provider = providerFuture.get() + val preview = Preview.Builder().build().also { it.setSurfaceProvider(previewView.surfaceProvider) } + val analysis = ImageAnalysis.Builder().setTargetResolution(Size(640,480)).setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST).build() + analysis.setAnalyzer(exec) { img -> + val bmp = img.toBitmap() ?: return@setAnalyzer + hand.process(bmp) + img.close() + } + provider.unbindAll() + provider.bindToLifecycle(this, CameraSelector.DEFAULT_FRONT_CAMERA, preview, analysis) + }, ContextCompat.getMainExecutor(this)) + } + + private fun ImageProxy.toBitmap(): Bitmap? { + val plane = planes.firstOrNull() ?: return null + // Convert YUV→ARGB fast path for demo; production should use RenderScript/ScriptIntrinsicYuvToRGB or GPU path + return Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888).also { + // Simplified; replace with a correct YUV→RGB conversion + } + } + + private fun hypot(dx: Float, dy: Float) = kotlin.math.sqrt(dx*dx + dy*dy) + private fun yRotation(theta: Float): FloatArray { + val c = kotlin.math.cos(theta); val s = kotlin.math.sin(theta) + return floatArrayOf( c,0f, s,0f, 0f,1f,0f,0f, -s,0f,c,0f, 0f,0f,0f,1f ) + } +} diff --git a/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/OverlayView.kt b/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/OverlayView.kt new file mode 100644 index 0000000..322eec3 --- /dev/null +++ b/LumenIO/apps/mobile/android/app/src/main/java/io/lumen/mobile/OverlayView.kt @@ -0,0 +1,13 @@ +package io.lumen.mobile + +import android.content.Context +import android.graphics.* +import android.view.View + +class OverlayView(ctx: Context): View(ctx) { + var points: List = emptyList() + override fun onDraw(c: Canvas) { + val p = Paint().apply { color = Color.GREEN; strokeWidth = 4f; style = Paint.Style.FILL } + points.forEach { (x,y,_) -> c.drawCircle(x*width, y*height, 6f, p) } + } +} diff --git a/LumenIO/schemas/v1/gesture_event.schema.json b/LumenIO/schemas/v1/gesture_event.schema.json new file mode 100644 index 0000000..0dc47df --- /dev/null +++ b/LumenIO/schemas/v1/gesture_event.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "gesture_event", + "type": "object", + "required": ["t","type","state","params","target"], + "properties": { + "t": { "type": "number" }, + "type": { "const": "gesture_event" }, + "state": { "enum": ["idle","pinch:start","pinch:hold","pinch:cancel","drag:start","drag:move","drag:end","rotate:start","rotate:move","rotate:end","scale:start","scale:move","scale:end"] }, + "params": { "type": "object", "properties": { + "x": {"type":"number"}, "y": {"type":"number"}, "angle": {"type":"number"}, "scale": {"type":"number"} + }, "additionalProperties": false }, + "target": { "type":"string", "description":"scene node id" } + }, + "additionalProperties": false +} diff --git a/LumenIO/schemas/v1/hand_landmarks.schema.json b/LumenIO/schemas/v1/hand_landmarks.schema.json new file mode 100644 index 0000000..191f93a --- /dev/null +++ b/LumenIO/schemas/v1/hand_landmarks.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "hand_landmarks", + "type": "object", + "required": ["t","type","handedness","points","coords","fps"], + "properties": { + "t": { "type": "number" }, + "type": { "const": "hand_landmarks" }, + "handedness": { "enum": ["Left","Right","Unknown"] }, + "coords": { "enum": ["image_norm","world_m"] }, + "fps": { "type": "number", "minimum": 1 }, + "points": { + "type": "array", "minItems": 21, "maxItems": 21, + "items": { "type": "array", "items": [{ "type": "number" },{ "type": "number" },{ "type": "number" }], "minItems": 3, "maxItems": 3 } + } + }, + "additionalProperties": false +}