From b078fddc93f3307e19883088e627d0a02519aa88 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Fri, 23 Jan 2026 16:12:11 +0100 Subject: [PATCH 01/72] feat(core): use a pool for frame to lower memory allocation --- .../streampack/core/elements/data/Frame.kt | 88 ++++++++---------- .../encoders/mediacodec/MediaCodecEncoder.kt | 41 +++++---- .../elements/endpoints/CombineEndpoint.kt | 2 +- .../muxers/mp4/models/TrackChunks.kt | 1 + .../endpoints/composites/muxers/ts/TsMuxer.kt | 19 ++-- .../core/elements/utils/pool/FramePool.kt | 57 ++++++++++++ .../{IFrameFactory.kt => IRawFrameFactory.kt} | 0 .../core/elements/utils/pool/ObjectPool.kt | 89 +++++++++++++++++++ .../core/elements/utils/FakeFrames.kt | 3 +- 9 files changed, 224 insertions(+), 76 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt rename core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/{IFrameFactory.kt => IRawFrameFactory.kt} (100%) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 018f4dbc1..14e337598 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -16,7 +16,6 @@ package io.github.thibaultbee.streampack.core.elements.data import android.media.MediaFormat -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removePrefixes import java.io.Closeable import java.nio.ByteBuffer @@ -48,111 +47,102 @@ data class RawFrame( } } - -data class Frame( +/** + * Encoded frame representation + */ +interface Frame { /** * Contains an audio or video frame data. */ - val rawBuffer: ByteBuffer, + val rawBuffer: ByteBuffer /** * Presentation timestamp in µs */ - val ptsInUs: Long, + val ptsInUs: Long /** * Decoded timestamp in µs (not used). */ - val dtsInUs: Long? = null, + val dtsInUs: Long? /** * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) */ - val isKeyFrame: Boolean, + val isKeyFrame: Boolean /** * Contains csd buffers for key frames and audio frames only. * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - val extra: List?, + val extra: List? /** * Contains frame format.. * TODO: to remove */ val format: MediaFormat -) { - init { - removePrefixes() - } } -/** - * Removes the [Frame.extra] prefixes from the [Frame.rawBuffer]. - * - * With MediaCodec, the encoded frames may contain prefixes like SPS, PPS for H264/H265 key frames. - * It also modifies the position of the [Frame.rawBuffer] to skip the prefixes.s - * - * @return A [ByteBuffer] without prefixes. - */ -fun Frame.removePrefixes(): ByteBuffer { - return if (extra != null) { - rawBuffer.removePrefixes(extra) - } else { - rawBuffer - } +fun Frame.copy( + rawBuffer: ByteBuffer = this.rawBuffer, + ptsInUs: Long = this.ptsInUs, + dtsInUs: Long? = this.dtsInUs, + isKeyFrame: Boolean = this.isKeyFrame, + extra: List? = this.extra, + format: MediaFormat = this.format +): Frame { + return MutableFrame( + rawBuffer = rawBuffer, + ptsInUs = ptsInUs, + dtsInUs = dtsInUs, + isKeyFrame = isKeyFrame, + extra = extra, + format = format + ) } -fun FrameWithCloseable( +/** + * A mutable [Frame] internal representation. + * + * The purpose is to get reusable [Frame] + */ +data class MutableFrame( /** * Contains an audio or video frame data. */ - rawBuffer: ByteBuffer, + override var rawBuffer: ByteBuffer, /** * Presentation timestamp in µs */ - ptsInUs: Long, + override var ptsInUs: Long, /** * Decoded timestamp in µs (not used). */ - dtsInUs: Long?, + override var dtsInUs: Long?, /** * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) */ - isKeyFrame: Boolean, + override var isKeyFrame: Boolean, /** * Contains csd buffers for key frames and audio frames only. * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - extra: List?, + override var extra: List?, /** * Contains frame format.. + * TODO: to remove */ - format: MediaFormat, - - /** - * A callback to call when frame is closed. - */ - onClosed: (FrameWithCloseable) -> Unit, -) = FrameWithCloseable( - Frame( - rawBuffer, - ptsInUs, - dtsInUs, - isKeyFrame, - extra, - format - ), - onClosed -) + override var format: MediaFormat +) : Frame /** * Frame internal representation. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index e62806ca5..1b90d17b9 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -31,6 +31,8 @@ import io.github.thibaultbee.streampack.core.elements.encoders.mediacodec.extens import io.github.thibaultbee.streampack.core.elements.encoders.mediacodec.extensions.isValid import io.github.thibaultbee.streampack.core.elements.utils.extensions.extra import io.github.thibaultbee.streampack.core.elements.utils.extensions.put +import io.github.thibaultbee.streampack.core.elements.utils.extensions.removePrefixes +import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -40,6 +42,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext +import java.io.Closeable import kotlin.math.min /** @@ -246,6 +249,8 @@ internal constructor( if (input is SurfaceInput) { input.release() } + + frameFactory.close() } catch (_: Throwable) { } finally { setState(State.RELEASED) @@ -587,9 +592,11 @@ internal constructor( class FrameFactory( private val codec: MediaCodec, private val isVideo: Boolean - ) { + ) : Closeable { private var previousPresentationTimeUs = 0L + private val pool = FramePool() + /** * Create a [Frame] from a [RawFrame] * @@ -623,30 +630,34 @@ internal constructor( tag: String ): FrameWithCloseable { val buffer = requireNotNull(codec.getOutputBuffer(index)) + val extra = if (isKeyFrame || !isVideo) { + outputFormat.extra + } else { + null + } + val rawBuffer = if (extra != null) { + buffer.removePrefixes(extra) + } else { + buffer + } + + val frame = pool.get(rawBuffer, ptsInUs, null, isKeyFrame, extra, outputFormat) + return FrameWithCloseable( - buffer, - ptsInUs, // pts - null, // dts - isKeyFrame, - try { - if (isKeyFrame || !isVideo) { - outputFormat.extra - } else { - null - } - } catch (_: Throwable) { - null - }, - outputFormat, + frame, onClosed = { try { codec.releaseOutputBuffer(index, false) + pool.put(frame) } catch (t: Throwable) { Logger.w(tag, "Failed to release output buffer for code: ${t.message}") } }) } + override fun close() { + pool.close() + } } companion object { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt index 7d7d8f721..4802f24e1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt @@ -18,6 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.utils.extensions.intersect import io.github.thibaultbee.streampack.core.logger.Logger @@ -36,7 +37,6 @@ import kotlinx.coroutines.flow.merge import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch - /** * Combines multiple endpoints into one. * diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt index 076d1c132..dca110e4f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt @@ -18,6 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaFormat import android.util.Size import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index d0ab92de7..f38231870 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -19,6 +19,7 @@ import android.media.MediaCodecInfo import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.IMuxerInternal @@ -86,16 +87,15 @@ class TsMuxer : IMuxerInternal { mimeType == MediaFormat.MIMETYPE_VIDEO_AVC -> { // Copy sps & pps before buffer if (frame.isKeyFrame) { - if (frame.extra == null) { - throw MissingFormatArgumentException("Missing extra for AVC") - } + val extra = frame.extra + ?: throw MissingFormatArgumentException("Missing extra for AVC") val buffer = - ByteBuffer.allocate(6 + frame.extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(6 + extra.sumOf { it.limit() } + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the AVC access unit buffer.putInt(0x00000001) buffer.put(0x09.toByte()) buffer.put(0xf0.toByte()) - frame.extra.forEach { buffer.put(it) } + extra.forEach { buffer.put(it) } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) @@ -107,17 +107,16 @@ class TsMuxer : IMuxerInternal { mimeType == MediaFormat.MIMETYPE_VIDEO_HEVC -> { // Copy sps & pps & vps before buffer if (frame.isKeyFrame) { - if (frame.extra == null) { - throw MissingFormatArgumentException("Missing extra for HEVC") - } + val extra = frame.extra + ?: throw MissingFormatArgumentException("Missing extra for HEVC") val buffer = - ByteBuffer.allocate(7 + frame.extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(7 + extra.sumOf { it.limit() } + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the HEVC access unit buffer.putInt(0x00000001) buffer.put(0x46.toByte()) buffer.put(0x01.toByte()) buffer.put(0x50.toByte()) - frame.extra.forEach { buffer.put(it) } + extra.forEach { buffer.put(it) } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt new file mode 100644 index 000000000..60281eeff --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.data.MutableFrame +import java.nio.ByteBuffer +import java.util.ArrayDeque + + +/** + * A pool of [MutableFrame]. + */ +internal class FramePool() : ObjectPool() { + fun get( + rawBuffer: ByteBuffer, + ptsInUs: Long, + dtsInUs: Long?, + isKeyFrame: Boolean, + extra: List?, + format: MediaFormat + ): MutableFrame { + val frame = get() + + return if (frame != null) { + frame.rawBuffer = rawBuffer + frame.ptsInUs = ptsInUs + frame.dtsInUs = dtsInUs + frame.isKeyFrame = isKeyFrame + frame.extra = extra + frame.format = format + frame + } else { + MutableFrame( + rawBuffer = rawBuffer, + ptsInUs = ptsInUs, + dtsInUs = dtsInUs, + isKeyFrame = isKeyFrame, + extra = extra, + format = format + ) + } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IFrameFactory.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt similarity index 100% rename from core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IFrameFactory.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt new file mode 100644 index 000000000..b405aa274 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt @@ -0,0 +1,89 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import java.io.Closeable +import java.util.ArrayDeque +import java.util.concurrent.atomic.AtomicBoolean + +/** + * A pool of objects. + * + * The pool is thread-safe. + * + * The implementation is required to add a `get` methods. + * + * @param T the type of object to pool + */ +internal sealed class ObjectPool() : Closeable { + private val pool = ArrayDeque() + + private val isClosed = AtomicBoolean(false) + + protected fun get(): T? { + if (isClosed.get()) { + throw IllegalStateException("ObjectPool is closed") + } + + return synchronized(pool) { + if (!pool.isEmpty()) { + pool.removeFirst() + } else { + null + } + } + } + + /** + * Puts an object in the pool. + * + * @param any the object to put + */ + fun put(any: T) { + if (isClosed.get()) { + throw IllegalStateException("ObjectPool is closed") + } + synchronized(pool) { + pool.addLast(any) + } + } + + /** + * Clears the pool. + */ + fun clear() { + if (isClosed.get()) { + return + } + synchronized(pool) { + pool.clear() + } + } + + /** + * Closes the pool. + * + * After a pool is closed, it cannot be used anymore. + */ + override fun close() { + if (isClosed.getAndSet(true)) { + return + } + synchronized(pool) { + pool.clear() + } + } +} \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt index 83139d45c..4b1a4c94b 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt @@ -18,6 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.utils import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer import kotlin.random.Random @@ -51,7 +52,7 @@ object FakeFrames { ) ) } - return Frame( + return MutableFrame( buffer, pts, dts, From bf3c1af13ffc9d8039ca777259d6d3865c5dd3ee Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sat, 24 Jan 2026 15:15:02 +0100 Subject: [PATCH 02/72] refactor(*): merge FrameWithCloseable with MutableFrame to avoid memory allocation --- .../core/elements/endpoints/DummyEndpoint.kt | 6 ++-- .../streampack/core/elements/data/Frame.kt | 34 ++++++++----------- .../core/elements/encoders/IEncoder.kt | 4 +-- .../encoders/mediacodec/MediaCodecEncoder.kt | 21 ++++++------ .../elements/endpoints/CombineEndpoint.kt | 14 ++++---- .../elements/endpoints/DynamicEndpoint.kt | 6 ++-- .../core/elements/endpoints/IEndpoint.kt | 9 +++-- .../elements/endpoints/MediaMuxerEndpoint.kt | 7 ++-- .../endpoints/composites/CompositeEndpoint.kt | 6 ++-- .../composites/muxers/IMuxerInternal.kt | 4 +-- .../composites/muxers/mp4/Mp4Muxer.kt | 7 ++-- .../endpoints/composites/muxers/ts/TsMuxer.kt | 8 ++--- .../core/elements/utils/pool/FramePool.kt | 8 +++-- .../core/elements/utils/pool/ObjectPool.kt | 1 + .../encoding/EncodingPipelineOutput.kt | 14 ++++---- .../elements/endpoints/DynamicEndpointTest.kt | 4 +-- .../composites/muxers/ts/TsMuxerTest.kt | 14 ++++---- .../core/elements/utils/FakeFrames.kt | 20 ----------- .../flv/elements/endpoints/FlvEndpoints.kt | 7 ++-- .../composites/muxer/utils/FlvTagBuilder.kt | 12 +++---- .../rtmp/elements/endpoints/RtmpEndpoint.kt | 7 ++-- 21 files changed, 91 insertions(+), 122 deletions(-) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt index b4a06c464..bf8caea0b 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt @@ -19,7 +19,6 @@ import android.content.Context import android.util.Log import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import kotlinx.coroutines.flow.MutableStateFlow @@ -58,11 +57,10 @@ class DummyEndpoint : IEndpointInternal { _isOpenFlow.emit(false) } - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { - val frame = closeableFrame.frame + override suspend fun write(frame: Frame, streamPid: Int) { Log.i(TAG, "write: $frame") _frameFlow.emit(frame) - closeableFrame.close() + frame.close() } override suspend fun addStreams(streamConfigs: List): Map { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 14e337598..6d293060c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -50,7 +50,7 @@ data class RawFrame( /** * Encoded frame representation */ -interface Frame { +interface Frame : Closeable { /** * Contains an audio or video frame data. */ @@ -85,13 +85,18 @@ interface Frame { val format: MediaFormat } +interface WithClosable { + val onClosed: (T) -> Unit +} + fun Frame.copy( rawBuffer: ByteBuffer = this.rawBuffer, ptsInUs: Long = this.ptsInUs, dtsInUs: Long? = this.dtsInUs, isKeyFrame: Boolean = this.isKeyFrame, extra: List? = this.extra, - format: MediaFormat = this.format + format: MediaFormat = this.format, + onClosed: (Frame) -> Unit = {} ): Frame { return MutableFrame( rawBuffer = rawBuffer, @@ -99,7 +104,8 @@ fun Frame.copy( dtsInUs = dtsInUs, isKeyFrame = isKeyFrame, extra = extra, - format = format + format = format, + onClosed = onClosed ) } @@ -141,16 +147,13 @@ data class MutableFrame( * Contains frame format.. * TODO: to remove */ - override var format: MediaFormat -) : Frame + override var format: MediaFormat, -/** - * Frame internal representation. - */ -data class FrameWithCloseable( - val frame: Frame, - val onClosed: (FrameWithCloseable) -> Unit -) : Closeable { + /** + * A callback to call when frame is closed. + */ + override var onClosed: (MutableFrame) -> Unit = {} +) : Frame, WithClosable { override fun close() { try { onClosed(this) @@ -159,10 +162,3 @@ data class FrameWithCloseable( } } } - -/** - * Uses the resource and unwraps the [Frame] to pass it to the given block. - */ -inline fun FrameWithCloseable.useAndUnwrap(block: (Frame) -> T) = use { - block(frame) -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt index 2bedd128a..b5d480a15 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt @@ -16,7 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.encoders import android.view.Surface -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendReleasable import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendStreamable @@ -77,7 +77,7 @@ interface IEncoderInternal : SuspendStreamable, SuspendReleasable, /** * A channel where the encoder will send encoded frames. */ - val outputChannel: SendChannel + val outputChannel: SendChannel } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index 1b90d17b9..80ec9f025 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -22,7 +22,7 @@ import android.media.MediaFormat import android.os.Bundle import android.util.Log import android.view.Surface -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.EncoderMode import io.github.thibaultbee.streampack.core.elements.encoders.IEncoderInternal @@ -249,8 +249,6 @@ internal constructor( if (input is SurfaceInput) { input.release() } - - frameFactory.close() } catch (_: Throwable) { } finally { setState(State.RELEASED) @@ -604,7 +602,7 @@ internal constructor( */ fun frame( index: Int, outputFormat: MediaFormat, info: BufferInfo, tag: String - ): FrameWithCloseable { + ): Frame { var pts = info.presentationTimeUs if (pts <= previousPresentationTimeUs) { pts = previousPresentationTimeUs + 1 @@ -628,7 +626,7 @@ internal constructor( ptsInUs: Long, isKeyFrame: Boolean, tag: String - ): FrameWithCloseable { + ): Frame { val buffer = requireNotNull(codec.getOutputBuffer(index)) val extra = if (isKeyFrame || !isVideo) { outputFormat.extra @@ -641,11 +639,14 @@ internal constructor( buffer } - val frame = pool.get(rawBuffer, ptsInUs, null, isKeyFrame, extra, outputFormat) - - return FrameWithCloseable( - frame, - onClosed = { + return pool.get( + rawBuffer, + ptsInUs, + null, + isKeyFrame, + extra, + outputFormat, + onClosed = { frame -> try { codec.releaseOutputBuffer(index, false) pool.put(frame) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt index 4802f24e1..f9d235323 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt @@ -17,7 +17,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.utils.extensions.intersect @@ -218,8 +218,7 @@ open class CombineEndpoint( * * If all endpoints write fails, it throws the exception of the first endpoint that failed. */ - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { - val frame = closeableFrame.frame + override suspend fun write(frame: Frame, streamPid: Int) { val throwables = mutableListOf() /** @@ -231,9 +230,10 @@ open class CombineEndpoint( endpointInternals.filter { it.isOpenFlow.value }.forEach { endpoint -> try { val deferred = CompletableDeferred() - val duplicatedFrame = FrameWithCloseable( - frame.copy(rawBuffer = frame.rawBuffer.duplicate()), - { deferred.complete(Unit) }) + val duplicatedFrame = frame.copy( + rawBuffer = frame.rawBuffer.duplicate(), + onClosed = { deferred.complete(Unit) } + ) val endpointStreamId = endpointsToStreamIdsMap[Pair(endpoint, streamPid)]!! deferreds += deferred @@ -246,7 +246,7 @@ open class CombineEndpoint( coroutineScope.launch { deferreds.forEach { it.await() } - closeableFrame.close() + frame.close() } if (throwables.isNotEmpty()) { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt index c242000ef..1951fabc0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.createDefaultTsServiceInfo -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.CompositeEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.TsMuxer @@ -146,8 +146,8 @@ open class DynamicEndpoint( endpoint.addStream(streamConfig) } - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) = - safeEndpoint { endpoint -> endpoint.write(closeableFrame, streamPid) } + override suspend fun write(frame: Frame, streamPid: Int) = + safeEndpoint { endpoint -> endpoint.write(frame, streamPid) } override suspend fun startStream() = safeEndpoint { endpoint -> endpoint.startStream() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt index b87e8627f..715c7d5c8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt @@ -18,7 +18,6 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.FileSink import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendCloseable @@ -46,14 +45,14 @@ interface IEndpointInternal : IEndpoint, SuspendStreamable, /** * Writes a [Frame] to the [IEndpointInternal]. * - * The [FrameWithCloseable.close] must be called when the frame has been processed and the [Frame.rawBuffer] is not used anymore. - * The [IEndpointInternal] must called [FrameWithCloseable.close] even if the frame is dropped or it somehow crashes. + * The [Frame.close] must be called when the frame has been processed and the [Frame.rawBuffer] is not used anymore. + * The [IEndpointInternal] must called [Frame.close] even if the frame is dropped or it somehow crashes. * - * @param closeableFrame the [Frame] to write + * @param frame the [Frame] to write * @param streamPid the stream id the [Frame] belongs to */ suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, streamPid: Int, ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt index 1d30fba65..cedb2462a 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt @@ -24,7 +24,7 @@ import android.media.MediaMuxer.OutputFormat import android.os.Build import android.os.ParcelFileDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider @@ -142,9 +142,8 @@ class MediaMuxerEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) = withContext(ioDispatcher) { - val frame = closeableFrame.frame mutex.withLock { try { if (state != State.STARTED && state != State.PENDING_START) { @@ -183,7 +182,7 @@ class MediaMuxerEndpoint( Logger.e(TAG, "Error while writing frame: ${t.message}") throw t } finally { - closeableFrame.close() + frame.close() } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt index 92f58fcee..e85726a65 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt @@ -17,7 +17,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal @@ -80,9 +80,9 @@ class CompositeEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, streamPid: Int - ) = muxer.write(closeableFrame, streamPid) + ) = muxer.write(frame, streamPid) override suspend fun addStreams(streamConfigs: List): Map { return mutex.withLock { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt index 1b609738b..108736b7c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt @@ -15,7 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data.Packet import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.interfaces.Releasable @@ -32,7 +32,7 @@ interface IMuxerInternal : fun onOutputFrame(packet: Packet) } - fun write(closeableFrame: FrameWithCloseable, streamPid: Int) + fun write(frame: Frame, streamPid: Int) fun addStreams(streamsConfig: List): Map diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt index 4c4f630e5..b96ca44b2 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt @@ -15,9 +15,8 @@ */ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4 -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data.Packet -import io.github.thibaultbee.streampack.core.elements.data.useAndUnwrap import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.IMuxerInternal import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.boxes.FileTypeBox @@ -58,9 +57,9 @@ class Mp4Muxer( override val streamConfigs: List get() = tracks.map { it.config } - override fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { + override fun write(frame: Frame, streamPid: Int) { synchronized(this) { - closeableFrame.useAndUnwrap { frame -> + frame.use { frame -> if (segmenter!!.mustWriteSegment(frame)) { writeSegment() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index f38231870..626eaebf8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -18,7 +18,6 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaCodecInfo import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig @@ -73,13 +72,12 @@ class TsMuxer : IMuxerInternal { /** * Encodes a frame to MPEG-TS format. * Each audio frames and each video key frames must come with an extra buffer containing sps, pps,... - * @param closeableFrame frame to mux + * @param frame frame to mux * @param streamPid Pid of frame stream. Throw a NoSuchElementException if streamPid refers to an unknown stream */ override fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame try { val pes = getPes(streamPid.toShort()) val mimeType = pes.stream.config.mimeType @@ -160,7 +158,7 @@ class TsMuxer : IMuxerInternal { generateStreams(newFrame, pes) } } finally { - closeableFrame.close() + frame.close() } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt index 60281eeff..112b0d73a 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -18,7 +18,6 @@ package io.github.thibaultbee.streampack.core.elements.utils.pool import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer -import java.util.ArrayDeque /** @@ -31,7 +30,8 @@ internal class FramePool() : ObjectPool() { dtsInUs: Long?, isKeyFrame: Boolean, extra: List?, - format: MediaFormat + format: MediaFormat, + onClosed: (MutableFrame) -> Unit ): MutableFrame { val frame = get() @@ -42,6 +42,7 @@ internal class FramePool() : ObjectPool() { frame.isKeyFrame = isKeyFrame frame.extra = extra frame.format = format + frame.onClosed = onClosed frame } else { MutableFrame( @@ -50,7 +51,8 @@ internal class FramePool() : ObjectPool() { dtsInUs = dtsInUs, isKeyFrame = isKeyFrame, extra = extra, - format = format + format = format, + onClosed = onClosed ) } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt index b405aa274..73bc0b337 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt @@ -15,6 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.utils.pool +import io.github.thibaultbee.streampack.core.logger.Logger import java.io.Closeable import java.util.ArrayDeque import java.util.concurrent.atomic.AtomicBoolean diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt index cedbad83b..d3b8a2a05 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.core.pipelines.outputs.encoding import android.content.Context import android.view.Surface import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig @@ -239,7 +239,7 @@ internal class EncodingPipelineOutput( } override val outputChannel = - Channel(Channel.UNLIMITED, onUndeliveredElement = { + Channel(Channel.UNLIMITED, onUndeliveredElement = { it.close() }) } @@ -250,7 +250,7 @@ internal class EncodingPipelineOutput( } override val outputChannel = - Channel(Channel.UNLIMITED, onUndeliveredElement = { + Channel(Channel.UNLIMITED, onUndeliveredElement = { it.close() }) } @@ -259,10 +259,10 @@ internal class EncodingPipelineOutput( if (withAudio) { coroutineScope.launch(audioOutputDispatcher) { // Audio - audioEncoderListener.outputChannel.consumeEach { closeableFrame -> + audioEncoderListener.outputChannel.consumeEach { frame -> try { audioStreamId?.let { - endpointInternal.write(closeableFrame, it) + endpointInternal.write(frame, it) } ?: Logger.w(TAG, "Audio frame received but audio stream is not set") } catch (t: Throwable) { onInternalError(t) @@ -273,10 +273,10 @@ internal class EncodingPipelineOutput( if (withVideo) { coroutineScope.launch(videoOutputDispatcher) { // Video - videoEncoderListener.outputChannel.consumeEach { closeableFrame -> + videoEncoderListener.outputChannel.consumeEach { frame -> try { videoStreamId?.let { - endpointInternal.write(closeableFrame, it) + endpointInternal.write(frame, it) } ?: Logger.w(TAG, "Video frame received but video stream is not set") } catch (t: Throwable) { onInternalError(t) diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt index 2b8e3428b..3b96e15f9 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt @@ -6,7 +6,7 @@ import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor import io.github.thibaultbee.streampack.core.elements.utils.DescriptorUtils -import io.github.thibaultbee.streampack.core.elements.utils.FakeFramesWithCloseable +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrames import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.test.runTest import org.junit.Assert.assertFalse @@ -85,7 +85,7 @@ class DynamicEndpointTest { val dynamicEndpoint = DynamicEndpoint(context, Dispatchers.Default, Dispatchers.IO) try { dynamicEndpoint.write( - FakeFramesWithCloseable.create(MediaFormat.MIMETYPE_AUDIO_AAC), + FakeFrames.create(MediaFormat.MIMETYPE_AUDIO_AAC), 0 ) fail("Throwable expected") diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt index 1a5b3dc8d..b3fa4dd23 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt @@ -21,7 +21,7 @@ import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.TSConst import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.Utils.createFakeServiceInfo -import io.github.thibaultbee.streampack.core.elements.utils.FakeFramesWithCloseable +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrames import io.github.thibaultbee.streampack.core.elements.utils.MockUtils import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -193,7 +193,7 @@ class TsMuxerTest { val tsMux = TsMuxer() try { tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), -1 ) fail() @@ -206,7 +206,7 @@ class TsMuxerTest { val tsMux = TsMuxer() try { tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), -1 ) fail() @@ -231,11 +231,11 @@ class TsMuxerTest { tsMux.addStreams(service, listOf(config))[config]!! tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid ) tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid ) } @@ -251,10 +251,10 @@ class TsMuxerTest { tsMux.addStreams(createFakeServiceInfo(), listOf(config))[config]!! tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid ) tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid ) } } \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt index 4b1a4c94b..b31ab2b44 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt @@ -17,7 +17,6 @@ package io.github.thibaultbee.streampack.core.elements.utils import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer import kotlin.random.Random @@ -66,22 +65,3 @@ object FakeFrames { ) } } - -object FakeFramesWithCloseable { - fun create( - mimeType: String, - buffer: ByteBuffer = ByteBuffer.wrap(Random.nextBytes(1024)), - pts: Long = Random.nextLong(), - dts: Long? = null, - isKeyFrame: Boolean = false - ) = FrameWithCloseable( - FakeFrames.create( - mimeType, - buffer, - pts, - dts, - isKeyFrame - ), - {/* Nothing to do */ } - ) -} \ No newline at end of file diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt index 91bf48b07..661315379 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt @@ -22,7 +22,7 @@ import io.github.komedia.komuxer.flv.encode import io.github.komedia.komuxer.flv.tags.FLVTag import io.github.komedia.komuxer.flv.tags.script.OnMetadata import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.endpoints.MediaSinkType @@ -126,12 +126,11 @@ sealed class FlvEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame val startUpTimestamp = getStartUpTimestamp(frame.ptsInUs) val ts = (frame.ptsInUs - startUpTimestamp) / 1000 - flvTagBuilder.write(closeableFrame, ts.toInt(), streamPid) + flvTagBuilder.write(frame, ts.toInt(), streamPid) } override suspend fun addStreams(streamConfigs: List): Map { diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt index 14df85e5b..a450972d7 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.m import io.github.komedia.komuxer.flv.tags.FLVTag import io.github.komedia.komuxer.flv.tags.script.Metadata import io.github.komedia.komuxer.logger.KomuxerLogger -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig @@ -106,21 +106,19 @@ class FlvTagBuilder(val channel: ChannelWithCloseableData) { } suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, ts: Int, streamPid: Int ) { if (ts < 0) { Logger.w( TAG, - "Negative timestamp $ts for frame ${closeableFrame.frame}. Frame will be dropped." + "Negative timestamp $ts for frame $frame. Frame will be dropped." ) - closeableFrame.close() + frame.close() return } - val frame = closeableFrame.frame - val flvDatas = when (streamPid) { AUDIO_STREAM_PID -> audioStream?.create(frame) ?: throw IllegalStateException("Audio stream not added") @@ -133,7 +131,7 @@ class FlvTagBuilder(val channel: ChannelWithCloseableData) { flvDatas.forEachIndexed { index, flvData -> if (index == flvDatas.lastIndex) { // Pass the close callback on the last element - channel.send(FLVTag(ts, flvData), { closeableFrame.close() }) + channel.send(FLVTag(ts, flvData), { frame.close() }) } else { channel.send(FLVTag(ts, flvData)) } diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt index b0d3d8eea..eab0e0eb9 100644 --- a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt @@ -22,7 +22,7 @@ import io.github.komedia.komuxer.rtmp.client.RtmpClient import io.github.komedia.komuxer.rtmp.connect import io.github.komedia.komuxer.rtmp.messages.command.StreamPublishType import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.ClosedException import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint @@ -164,12 +164,11 @@ class RtmpEndpoint internal constructor( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame val startUpTimestamp = getStartUpTimestamp(frame.ptsInUs) val ts = (frame.ptsInUs - startUpTimestamp) / 1000 - flvTagBuilder.write(closeableFrame, ts.toInt(), streamPid) + flvTagBuilder.write(frame, ts.toInt(), streamPid) } override suspend fun addStreams(streamConfigs: List): Map { From 7de246989d76b30e5c50e4a7c5ebbb176ce5406a Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:06:28 +0100 Subject: [PATCH 03/72] chore(core): frame copy use frame from pool for memory optimization --- .../streampack/core/elements/data/Frame.kt | 22 ++++--- .../muxers/mp4/models/TrackChunks.kt | 58 ++++++++++--------- .../endpoints/composites/muxers/ts/TsMuxer.kt | 10 +++- .../core/elements/utils/pool/FramePool.kt | 7 +++ 4 files changed, 59 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 6d293060c..e7505acd2 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.data import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool import java.io.Closeable import java.nio.ByteBuffer @@ -89,6 +90,11 @@ interface WithClosable { val onClosed: (T) -> Unit } +/** + * Copy a [Frame] to a new [Frame]. + * + * For better memory allocation, you should close the returned frame after usage. + */ fun Frame.copy( rawBuffer: ByteBuffer = this.rawBuffer, ptsInUs: Long = this.ptsInUs, @@ -98,15 +104,13 @@ fun Frame.copy( format: MediaFormat = this.format, onClosed: (Frame) -> Unit = {} ): Frame { - return MutableFrame( - rawBuffer = rawBuffer, - ptsInUs = ptsInUs, - dtsInUs = dtsInUs, - isKeyFrame = isKeyFrame, - extra = extra, - format = format, - onClosed = onClosed - ) + val pool = FramePool.default + return pool.get( + rawBuffer, ptsInUs, dtsInUs, isKeyFrame, extra, format, + { frame -> + pool.put(frame) + onClosed(frame) + }) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt index dca110e4f..33cea1f13 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt @@ -56,7 +56,6 @@ import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxer import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.boxes.VPCodecConfigurationBox import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.utils.createHandlerBox import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.utils.createTypeMediaHeaderBox -import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.elements.utils.av.audio.opus.OpusCsdParser import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.AudioSpecificConfigDescriptor import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.ESDescriptor @@ -70,6 +69,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAvcc import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.resolution import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize +import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import java.nio.ByteBuffer /** @@ -184,34 +184,38 @@ class TrackChunks( fun write() { chunks.forEach { chunk -> chunk.writeTo { frame -> - when (track.config.mimeType) { - MediaFormat.MIMETYPE_VIDEO_HEVC, - MediaFormat.MIMETYPE_VIDEO_AVC -> { - if (frame.rawBuffer.isAnnexB) { - // Replace start code with size (from Annex B to AVCC) - val noStartCodeBuffer = frame.rawBuffer.removeStartCode() - val sizeBuffer = ByteBuffer.allocate(4) - sizeBuffer.putInt(0, noStartCodeBuffer.remaining()) - onNewSample(sizeBuffer) - onNewSample(noStartCodeBuffer) - } else if (frame.rawBuffer.isAvcc) { - onNewSample(frame.rawBuffer) - } else { - throw IllegalArgumentException( - "Unsupported buffer format: buffer start with 0x${ - frame.rawBuffer.get( - 0 - ).toString(16) - }, 0x${frame.rawBuffer.get(1).toString(16)}, 0x${ - frame.rawBuffer.get(2).toString(16) - }, 0x${frame.rawBuffer.get(3).toString(16)}" - ) + try { + when (track.config.mimeType) { + MediaFormat.MIMETYPE_VIDEO_HEVC, + MediaFormat.MIMETYPE_VIDEO_AVC -> { + if (frame.rawBuffer.isAnnexB) { + // Replace start code with size (from Annex B to AVCC) + val noStartCodeBuffer = frame.rawBuffer.removeStartCode() + val sizeBuffer = ByteBuffer.allocate(4) + sizeBuffer.putInt(0, noStartCodeBuffer.remaining()) + onNewSample(sizeBuffer) + onNewSample(noStartCodeBuffer) + } else if (frame.rawBuffer.isAvcc) { + onNewSample(frame.rawBuffer) + } else { + throw IllegalArgumentException( + "Unsupported buffer format: buffer start with 0x${ + frame.rawBuffer.get( + 0 + ).toString(16) + }, 0x${frame.rawBuffer.get(1).toString(16)}, 0x${ + frame.rawBuffer.get(2).toString(16) + }, 0x${frame.rawBuffer.get(3).toString(16)}" + ) + } } - } - else -> { - onNewSample(frame.rawBuffer) - } // Nothing + else -> { + onNewSample(frame.rawBuffer) + } // Nothing + } + } finally { + frame.close() } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index 626eaebf8..ba17e1291 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -154,8 +154,14 @@ class TsMuxer : IMuxerInternal { else -> throw IllegalArgumentException("Unsupported mimeType $mimeType") } - synchronized(this) { - generateStreams(newFrame, pes) + try { + synchronized(this) { + generateStreams(newFrame, pes) + } + } finally { + if (frame != newFrame) { + newFrame.close() + } } } finally { frame.close() diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt index 112b0d73a..264917486 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -56,4 +56,11 @@ internal class FramePool() : ObjectPool() { ) } } + + companion object { + /** + * The default frame pool. + */ + internal val default by lazy { FramePool() } + } } \ No newline at end of file From 7882782cdc0c8e10093bb59b796e62b3f13c3dc2 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:16:49 +0100 Subject: [PATCH 04/72] chore(core): codec: reusable extra for memory optimization --- .../encoders/mediacodec/MediaCodecEncoder.kt | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index 80ec9f025..deb443df4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -43,6 +43,7 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import java.io.Closeable +import java.nio.ByteBuffer import kotlin.math.min /** @@ -63,6 +64,12 @@ internal constructor( private val mediaCodec: MediaCodec private val format: MediaFormat private var outputFormat: MediaFormat? = null + set(value) { + extra = value?.extra + field = value + } + private var extra: List? = null + private val frameFactory by lazy { FrameFactory(mediaCodec, isVideo) } private val isVideo = encoderConfig.isVideo @@ -353,7 +360,7 @@ internal constructor( info.isValid -> { try { val frame = frameFactory.frame( - index, outputFormat!!, info, tag + index, extra, outputFormat!!, info, tag ) try { listener.outputChannel.send(frame) @@ -601,7 +608,11 @@ internal constructor( * @return the created frame */ fun frame( - index: Int, outputFormat: MediaFormat, info: BufferInfo, tag: String + index: Int, + extra: List?, + outputFormat: MediaFormat, + info: BufferInfo, + tag: String ): Frame { var pts = info.presentationTimeUs if (pts <= previousPresentationTimeUs) { @@ -609,7 +620,7 @@ internal constructor( Logger.w(tag, "Correcting timestamp: $pts <= $previousPresentationTimeUs") } previousPresentationTimeUs = pts - return createFrame(codec, index, outputFormat, pts, info.isKeyFrame, tag) + return createFrame(codec, index, extra, outputFormat, pts, info.isKeyFrame, tag) } /** @@ -622,6 +633,7 @@ internal constructor( private fun createFrame( codec: MediaCodec, index: Int, + extra: List?, outputFormat: MediaFormat, ptsInUs: Long, isKeyFrame: Boolean, @@ -629,7 +641,7 @@ internal constructor( ): Frame { val buffer = requireNotNull(codec.getOutputBuffer(index)) val extra = if (isKeyFrame || !isVideo) { - outputFormat.extra + extra!!.map { it.duplicate() } } else { null } From ad13a5b43967acfff86e6d000160a76572b7dd6e Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sun, 25 Jan 2026 21:52:19 +0100 Subject: [PATCH 05/72] refactor(core): move dummy endpoint to main source for test purpose --- .../encoding/EncodingPipelineOutputTest.kt | 3 --- .../core/elements/endpoints/DummyEndpoint.kt | 18 +++--------------- 2 files changed, 3 insertions(+), 18 deletions(-) rename core/src/{androidTest => main}/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt (86%) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt index 1531e0257..917940c47 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt @@ -335,9 +335,6 @@ class EncodingPipelineOutputTest { Random.nextLong() ) ) - - // Wait for frame to be received - assertNotNull(dummyEndpoint.frameFlow.filterNotNull().first()) } companion object { diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt similarity index 86% rename from core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt index bf8caea0b..429b0a078 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt @@ -16,7 +16,6 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context -import android.util.Log import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig @@ -25,19 +24,16 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +/** + * A dummy endpoint for testing. + */ class DummyEndpoint : IEndpointInternal { private val _isOpenFlow = MutableStateFlow(false) override val isOpenFlow = _isOpenFlow.asStateFlow() - private val _frameFlow = MutableStateFlow(null) - val frameFlow = _frameFlow.asStateFlow() - private val _isStreamingFlow = MutableStateFlow(false) val isStreamingFlow = _isStreamingFlow.asStateFlow() - private val _configFlow = MutableStateFlow(null) - val configFlow = _configFlow.asStateFlow() - override val info: IEndpoint.IEndpointInfo get() = TODO("Not yet implemented") @@ -58,18 +54,14 @@ class DummyEndpoint : IEndpointInternal { } override suspend fun write(frame: Frame, streamPid: Int) { - Log.i(TAG, "write: $frame") - _frameFlow.emit(frame) frame.close() } override suspend fun addStreams(streamConfigs: List): Map { - streamConfigs.forEach { _configFlow.emit(it) } return streamConfigs.associateWith { it.hashCode() } } override suspend fun addStream(streamConfig: CodecConfig): Int { - _configFlow.emit(streamConfig) return streamConfig.hashCode() } @@ -80,10 +72,6 @@ class DummyEndpoint : IEndpointInternal { override suspend fun stopStream() { _isStreamingFlow.emit(false) } - - companion object { - private const val TAG = "DummyEndpoint" - } } /** From 50976ddaf8c087fd426e77c75052b3572b87599d Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 26 Jan 2026 22:23:54 +0100 Subject: [PATCH 06/72] refactor(core): avoid slice to skip start code to reduce memory allocation --- .../encoders/mediacodec/MediaCodecEncoder.kt | 2 +- .../muxers/mp4/models/TrackChunks.kt | 4 +- .../avc/AVCDecoderConfigurationRecord.kt | 21 ++- .../hevc/HEVCDecoderConfigurationRecord.kt | 4 +- .../utils/extensions/ByteBufferExtensions.kt | 154 ++++++++++++------ .../extensions/ByteBufferExtensionsKtTest.kt | 45 ++++- 6 files changed, 160 insertions(+), 70 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index deb443df4..fe033fb96 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -531,7 +531,7 @@ internal constructor( } val inputBuffer = requireNotNull(mediaCodec.getInputBuffer(inputBufferId)) val size = min(frame.rawBuffer.remaining(), inputBuffer.remaining()) - inputBuffer.put(frame.rawBuffer, frame.rawBuffer.position(), size) + inputBuffer.put(frame.rawBuffer, 0, size) mediaCodec.queueInputBuffer( inputBufferId, 0, size, frame.timestampInUs, 0 ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt index 33cea1f13..cbccabd76 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt @@ -66,7 +66,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.video.vpx.VPCodec import io.github.thibaultbee.streampack.core.elements.utils.extensions.clone import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAnnexB import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAvcc -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.resolution import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils @@ -190,7 +190,7 @@ class TrackChunks( MediaFormat.MIMETYPE_VIDEO_AVC -> { if (frame.rawBuffer.isAnnexB) { // Replace start code with size (from Annex B to AVCC) - val noStartCodeBuffer = frame.rawBuffer.removeStartCode() + val noStartCodeBuffer = frame.rawBuffer.skipStartCode() val sizeBuffer = ByteBuffer.allocate(4) sizeBuffer.putInt(0, noStartCodeBuffer.remaining()) onNewSample(sizeBuffer) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt index 75281c053..0b0c0fd3f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt @@ -19,8 +19,8 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.buffer.ByteBuffer import io.github.thibaultbee.streampack.core.elements.utils.av.video.ChromaFormat import io.github.thibaultbee.streampack.core.elements.utils.extensions.put import io.github.thibaultbee.streampack.core.elements.utils.extensions.putShort -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.shl +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize import java.nio.ByteBuffer @@ -33,8 +33,8 @@ data class AVCDecoderConfigurationRecord( private val sps: List, private val pps: List ) : ByteBufferWriter() { - private val spsNoStartCode: List = sps.map { it.removeStartCode() } - private val ppsNoStartCode: List = pps.map { it.removeStartCode() } + private val spsNoStartCode: List = sps.map { it.skipStartCode() } + private val ppsNoStartCode: List = pps.map { it.skipStartCode() } override val size: Int = getSize(spsNoStartCode, ppsNoStartCode) @@ -86,11 +86,13 @@ data class AVCDecoderConfigurationRecord( sps: List, pps: List ): AVCDecoderConfigurationRecord { - val spsNoStartCode = sps.map { it.removeStartCode() } - val ppsNoStartCode = pps.map { it.removeStartCode() } - val profileIdc: Byte = spsNoStartCode[0].get(1) - val profileCompatibility = spsNoStartCode[0].get(2) - val levelIdc = spsNoStartCode[0].get(3) + val spsNoStartCode = sps.map { it.skipStartCode() } + val ppsNoStartCode = pps.map { it.skipStartCode() } + val firstSpsNoStartCode = spsNoStartCode[0] + val firstSpsNoStartCodePosition = firstSpsNoStartCode.position() + val profileIdc: Byte = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 1) + val profileCompatibility = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 2) + val levelIdc = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 3) return AVCDecoderConfigurationRecord( profileIdc = profileIdc, profileCompatibility = profileCompatibility, @@ -112,7 +114,8 @@ data class AVCDecoderConfigurationRecord( size += 2 + it.remaining() - it.startCodeSize } val spsStartCodeSize = sps[0].startCodeSize - val profileIdc = sps[0].get(spsStartCodeSize + 1).toInt() + val spsPosition = sps[0].position() + val profileIdc = sps[0].get(spsPosition + spsStartCodeSize + 1).toInt() if ((profileIdc == 100) || (profileIdc == 110) || (profileIdc == 122) || (profileIdc == 144)) { size += 4 } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt index 28a11669f..7bd86d908 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt @@ -20,7 +20,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.video.ChromaForma import io.github.thibaultbee.streampack.core.elements.utils.extensions.put import io.github.thibaultbee.streampack.core.elements.utils.extensions.putLong48 import io.github.thibaultbee.streampack.core.elements.utils.extensions.putShort -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.shl import io.github.thibaultbee.streampack.core.elements.utils.extensions.shr import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize @@ -195,7 +195,7 @@ data class HEVCDecoderConfigurationRecord( } data class NalUnit(val type: Type, val data: ByteBuffer, val completeness: Boolean = true) { - val noStartCodeData: ByteBuffer = data.removeStartCode() + val noStartCodeData: ByteBuffer = data.skipStartCode() fun write(buffer: ByteBuffer) { buffer.put((completeness shl 7) or type.value.toInt()) // array_completeness + reserved 1bit + naluType 6 bytes diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt index 66da15b18..e4beee7ab 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt @@ -86,12 +86,14 @@ fun ByteBuffer.put3x3Matrix(matrix: IntArray) { matrix.forEach { putInt(it) } } -fun ByteBuffer.put(buffer: ByteBuffer, offset: Int, length: Int) { - val limit = buffer.limit() - buffer.position(offset) - buffer.limit(offset + length) - this.put(buffer) - buffer.limit(limit) +fun ByteBuffer.put(src: ByteBuffer, offset: Int, length: Int) { + val limit = src.limit() + if (offset != 0) { + src.position(src.position() + offset) + } + src.limit(src.position() + offset + length) + this.put(src) + src.limit(limit) } fun ByteBuffer.getString(size: Int = this.remaining()): String { @@ -111,22 +113,38 @@ fun ByteBuffer.getLong(isLittleEndian: Boolean): Long { return value } -fun ByteBuffer.indicesOf(prefix: ByteArray): List { - if (prefix.isEmpty()) { - return emptyList() +/** + * Finds all occurrences of the given [needle] byte array within the ByteBuffer. + * @param needle The byte array sequence to search for. + * @return A list of starting indices for every match found. + */ +fun ByteBuffer.indicesOf(needle: ByteArray): List { + if (needle.isEmpty()) return emptyList() + + val results = mutableListOf() + val end = limit() - needle.size + var i = position() + + while (i <= end) { + if (match(i, needle)) { + results.add(i) + // Move forward by the needle's length to find the next non-overlapping match + // Or use i++ if you want to allow overlapping matches (e.g., "AAA" in "AAAA") + i += needle.size + } else { + i++ + } } + return results +} - val indices = mutableListOf() - - outer@ for (i in 0 until this.limit() - prefix.size + 1) { - for (j in prefix.indices) { - if (this.get(i + j) != prefix[j]) { - continue@outer - } +private fun ByteBuffer.match(start: Int, needle: ByteArray): Boolean { + for (idx in needle.indices) { + if (get(start + idx) != needle[idx]) { + return false } - indices.add(i) } - return indices + return true } /** @@ -137,18 +155,18 @@ fun ByteBuffer.slices(prefix: ByteArray): List { // Get all occurrence of prefix in buffer val indexes = this.indicesOf(prefix) + // Get slices indexes.forEachIndexed { index, i -> val nextPosition = if (indexes.indices.contains(index + 1)) { - indexes[index + 1] - 1 + indexes[index + 1] } else { - this.limit() - 1 + this.limit() } slices.add(Pair(i, nextPosition)) } - val array = this.array() return slices.map { - ByteBuffer.wrap(array.sliceArray(IntRange(it.first, it.second))) + this.slice(from = it.first, to = it.second) } } @@ -258,51 +276,72 @@ fun ByteBuffer.clone(): ByteBuffer { /** - * Get start code size of [ByteBuffer]. + * Get start code size of [ByteBuffer] from the current position */ val ByteBuffer.startCodeSize: Int - get() { - return if (this.get(0) == 0x00.toByte() && this.get(1) == 0x00.toByte() - && this.get(2) == 0x00.toByte() && this.get(3) == 0x01.toByte() - ) { - 4 - } else if (this.get(0) == 0x00.toByte() && this.get(1) == 0x00.toByte() - && this.get(2) == 0x01.toByte() - ) { - 3 - } else { - 0 - } + get() = getStartCodeSize(this.position()) + +/** + * Get start code size of [ByteBuffer] from the given [position]. + */ +fun ByteBuffer.getStartCodeSize( + position: Int +): Int { + return if (remaining() >= 4 && this.get(position) == 0x00.toByte() && this.get(position + 1) == 0x00.toByte() + && this.get(position + 2) == 0x00.toByte() && this.get(position + 3) == 0x01.toByte() + ) { + 4 + } else if (remaining() >= 3 && this.get(position) == 0x00.toByte() && this.get(position + 1) == 0x00.toByte() + && this.get(position + 2) == 0x01.toByte() + ) { + 3 + } else { + 0 } +} -fun ByteBuffer.removeStartCode(): ByteBuffer { +/** + * Moves the position after the start code. + */ +fun ByteBuffer.skipStartCode(): ByteBuffer { val startCodeSize = this.startCodeSize - this.position(startCodeSize) - return this.slice() + if (startCodeSize > 0) { + this.position(this.position() + startCodeSize) + } + return this } +private val emulationPreventionThreeByte = byteArrayOf(0x00, 0x00, 0x03) + +/** + * Removes all emulation prevention three bytes from [ByteBuffer]. + * + * @param headerLength [Int] of the header length before writing the [ByteBuffer]. + * @return [ByteBuffer] without emulation prevention three bytes + */ fun ByteBuffer.extractRbsp(headerLength: Int): ByteBuffer { - val rbsp = ByteBuffer.allocateDirect(this.remaining()) + val indices = this.indicesOf(emulationPreventionThreeByte) - val indices = this.indicesOf(byteArrayOf(0x00, 0x00, 0x03)) + val rbspSize = + this.remaining() - indices.size // remove 0x3 bytes for each emulation prevention three bytes + val rbsp = ByteBuffer.allocateDirect(rbspSize) - rbsp.put(this, this.startCodeSize, headerLength) + // Write header to new buffer + rbsp.put(this, 0, headerLength + this.startCodeSize) - var previous = this.position() indices.forEach { - rbsp.put(this, previous, it + 2 - previous) - previous = it + 3 // skip emulation_prevention_three_byte + rbsp.put(this, 0, it + 2 - this.position()) + this.position(this.position() + 1) // skip emulation_prevention_three_byte } - rbsp.put(this, previous, this.limit() - previous) + rbsp.put(this, 0, this.limit() - this.position()) - rbsp.limit(rbsp.position()) rbsp.rewind() return rbsp } /** * Remove all [prefixes] from [ByteBuffer] whatever their order. - * It slices [ByteBuffer] so it does not copy data. + * It moves the [position] of the [ByteBuffer]. * * Once a prefix is found, it is removed from the [prefixes] list. * @@ -324,7 +363,7 @@ fun ByteBuffer.removePrefixes(prefixes: List): ByteBuffer { } } - return this.slice().order(this.order()) + return this } /** @@ -343,4 +382,21 @@ val ByteBuffer.isAvcc: Boolean get() { val size = this.getInt(0) return size == (this.remaining() - 4) - } \ No newline at end of file + } + +/** + * Slices the buffer from [from] position to [to] position. + * + * @param from start position + * @param to end position + */ +fun ByteBuffer.slice(from: Int, to: Int): ByteBuffer { + val currentPosition = this.position() + val currentLimit = this.limit() + this.position(from) + this.limit(to) + val newBuffer = this.slice() + this.position(currentPosition) + this.limit(currentLimit) + return newBuffer +} diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt index 9825800e9..3ceff1035 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt @@ -134,12 +134,16 @@ class ByteBufferExtensionsKtTest { val resultBuffers = testBuffer.slices( byteArrayOf(0, 0, 0, 1) ) + assertEquals( 3, resultBuffers.size ) - var resultArray = ByteArray(0) - resultBuffers.forEach { resultArray += it.array() } // concat all arrays - assertArrayEquals(testBuffer.array(), resultArray) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array(), resultBuffer.array()) } @Test @@ -172,7 +176,12 @@ class ByteBufferExtensionsKtTest { assertEquals( 1, resultBuffers.size ) - assertArrayEquals(testBuffer.array(), resultBuffers[0].array()) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array(), resultBuffer.array()) } @Test @@ -189,9 +198,12 @@ class ByteBufferExtensionsKtTest { assertEquals( 1, resultBuffers.size ) - assertArrayEquals( - testBuffer.array().sliceArray(IntRange(3, 8)), resultBuffers[0].array() - ) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array().sliceArray(IntRange(3, 8)), resultBuffer.array()) } @Test @@ -263,6 +275,10 @@ class ByteBufferExtensionsKtTest { ) ) val expectedArray = byteArrayOf( + 0, + 0, + 0, + 1, 66, 1, 1, @@ -336,4 +352,19 @@ class ByteBufferExtensionsKtTest { assertEquals(0, prefixBuffer.position()) assertEquals(0, testBuffer.position()) } + + @Test + fun `slice from to test`() { + val testBuffer = ByteBuffer.wrap("ABCDE".toByteArray()) + val slice = testBuffer.slice(1, 4) + + assertArrayEquals( + "BCDE".toByteArray(), slice.toByteArray() + ) + assertEquals(0, slice.position()) + assertEquals(4, slice.limit()) + + assertEquals(0, testBuffer.position()) + assertEquals(5, testBuffer.limit()) + } } \ No newline at end of file From f2a5d1fbe238f03ab5a4e3f487a7f6e3c39b5bf3 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 27 Jan 2026 14:48:16 +0100 Subject: [PATCH 07/72] feat(core): pool make frame put back themself in the pool after close --- .../thibaultbee/streampack/core/elements/data/Frame.kt | 1 - .../elements/encoders/mediacodec/MediaCodecEncoder.kt | 3 +-- .../streampack/core/elements/utils/pool/FramePool.kt | 9 +++++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index e7505acd2..74b154dbc 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -108,7 +108,6 @@ fun Frame.copy( return pool.get( rawBuffer, ptsInUs, dtsInUs, isKeyFrame, extra, format, { frame -> - pool.put(frame) onClosed(frame) }) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index fe033fb96..9e2d9b97c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -658,10 +658,9 @@ internal constructor( isKeyFrame, extra, outputFormat, - onClosed = { frame -> + onClosed = { try { codec.releaseOutputBuffer(index, false) - pool.put(frame) } catch (t: Throwable) { Logger.w(tag, "Failed to release output buffer for code: ${t.message}") } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt index 264917486..7302ef272 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -35,6 +35,11 @@ internal class FramePool() : ObjectPool() { ): MutableFrame { val frame = get() + val onClosedHook = { frame: MutableFrame -> + onClosed(frame) + put(frame) + } + return if (frame != null) { frame.rawBuffer = rawBuffer frame.ptsInUs = ptsInUs @@ -42,7 +47,7 @@ internal class FramePool() : ObjectPool() { frame.isKeyFrame = isKeyFrame frame.extra = extra frame.format = format - frame.onClosed = onClosed + frame.onClosed = onClosedHook frame } else { MutableFrame( @@ -52,7 +57,7 @@ internal class FramePool() : ObjectPool() { isKeyFrame = isKeyFrame, extra = extra, format = format, - onClosed = onClosed + onClosed = onClosedHook ) } } From 9677e067b1be001fdeb9c57a3bba75efb2fde126 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 27 Jan 2026 21:59:38 +0100 Subject: [PATCH 08/72] feat(core): add a raw frame pool to reuse raw frame --- .../core/elements/sources/StubAudioSource.kt | 11 ++-- .../core/elements/sources/StubVideoSource.kt | 7 ++- .../encoding/EncodingPipelineOutputTest.kt | 6 +- .../streampack/core/elements/data/Frame.kt | 48 +++++++++++++-- .../elements/processing/RawFramePullPush.kt | 33 ++++++----- .../audio/IAudioFrameSourceInternal.kt | 20 +++---- .../audio/audiorecord/AudioRecordSource.kt | 35 +++++------ .../sources/video/IVideoFrameSource.kt | 10 ++-- .../elements/utils/pool/IRawFrameFactory.kt | 31 ---------- .../elements/utils/pool/RawFrameFactory.kt | 53 ----------------- .../core/elements/utils/pool/RawFramePool.kt | 58 +++++++++++++++++++ .../core/pipelines/StreamerPipeline.kt | 26 +++------ .../core/pipelines/inputs/AudioInput.kt | 40 ++++++------- .../elements/sources/AudioCaptureUnitTest.kt | 4 +- .../elements/utils/StubRawFrameFactory.kt | 35 ----------- 15 files changed, 184 insertions(+), 233 deletions(-) delete mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt delete mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt delete mode 100644 core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt index f929a92b6..4405578e4 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt @@ -1,12 +1,11 @@ package io.github.thibaultbee.streampack.core.elements.sources import android.content.Context -import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer class StubAudioSource : IAudioSourceInternal { private val _isStreamingFlow = MutableStateFlow(false) @@ -15,13 +14,11 @@ class StubAudioSource : IAudioSourceInternal { private val _configurationFlow = MutableStateFlow(null) val configurationFlow = _configurationFlow.asStateFlow() - override fun fillAudioFrame(frame: RawFrame): RawFrame { - return frame + override val minBufferSize = 0 + override fun fillAudioFrame(buffer: ByteBuffer): Long { + return 0L } - override fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory) = - frameFactory.create(8192, 0) - override suspend fun startStream() { _isStreamingFlow.value = true } diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt index 22eca54d4..883c6ada1 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt @@ -8,12 +8,12 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.ISurfaceSour import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoFrameSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.VideoSourceConfig -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory import io.github.thibaultbee.streampack.core.elements.utils.time.Timebase import io.github.thibaultbee.streampack.core.pipelines.IVideoDispatcherProvider import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer class StubVideoSurfaceSource(override val timebase: Timebase = Timebase.REALTIME) : StubVideoSource(), @@ -44,8 +44,9 @@ class StubVideoSurfaceSource(override val timebase: Timebase = Timebase.REALTIME } class StubVideoFrameSource : StubVideoSource(), IVideoFrameSourceInternal { - override fun getVideoFrame(frameFactory: IReadOnlyRawFrameFactory) = - frameFactory.create(8192, 0L) + override fun getVideoFrame(buffer: ByteBuffer): Long { + return 0L + } class Factory : IVideoSourceInternal.Factory { override suspend fun create( diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt index 917940c47..b2f1d4daf 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt @@ -20,7 +20,7 @@ import android.util.Log import androidx.core.net.toFile import androidx.test.platform.app.InstrumentationRegistry import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.MutableRawFrame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.DummyEndpoint @@ -317,7 +317,7 @@ class EncodingPipelineOutputTest { try { output.queueAudioFrame( - RawFrame( + MutableRawFrame( ByteBuffer.allocateDirect(16384), Random.nextLong() ) @@ -330,7 +330,7 @@ class EncodingPipelineOutputTest { output.startStream(descriptor) output.queueAudioFrame( - RawFrame( + MutableRawFrame( ByteBuffer.allocateDirect(16384), Random.nextLong() ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 74b154dbc..c1a952932 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -17,28 +17,64 @@ package io.github.thibaultbee.streampack.core.elements.data import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import java.io.Closeable import java.nio.ByteBuffer /** - * A raw frame internal representation. + * Encoded frame representation */ -data class RawFrame( +interface RawFrame : Closeable { /** * Contains an audio or video frame data. */ - val rawBuffer: ByteBuffer, + val rawBuffer: ByteBuffer /** * Presentation timestamp in µs */ - var timestampInUs: Long, + val timestampInUs: Long +} + + +/** + * Copy a [RawFrame] to a new [RawFrame]. + * + * For better memory allocation, you should close the returned frame after usage. + */ +fun RawFrame.copy( + rawBuffer: ByteBuffer = this.rawBuffer, + timestampInUs: Long = this.timestampInUs, + onClosed: (RawFrame) -> Unit = {} +): RawFrame { + val pool = RawFramePool.default + return pool.get( + rawBuffer, timestampInUs, + { frame -> + onClosed(frame) + }) +} + +/** + * A mutable [RawFrame] internal representation. + * + * The purpose is to get reusable [RawFrame] + */ +data class MutableRawFrame( + /** + * Contains an audio or video frame data. + */ + override var rawBuffer: ByteBuffer, + /** + * Presentation timestamp in µs + */ + override var timestampInUs: Long, /** * A callback to call when frame is closed. */ - val onClosed: (RawFrame) -> Unit = {} -) : Closeable { + override var onClosed: (MutableRawFrame) -> Unit = {} +) : RawFrame, WithClosable { override fun close() { try { onClosed(this) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt index 3dd308c16..f87b531ad 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt @@ -16,8 +16,9 @@ package io.github.thibaultbee.streampack.core.elements.processing import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFrameFactory +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioFrameSourceInternal +import io.github.thibaultbee.streampack.core.elements.utils.pool.ByteBufferPool +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -35,40 +36,42 @@ fun RawFramePullPush( onFrame: suspend (RawFrame) -> Unit, processDispatcher: CoroutineDispatcher, isDirect: Boolean = true -) = RawFramePullPush(frameProcessor, onFrame, RawFrameFactory(isDirect), processDispatcher) +) = RawFramePullPush(frameProcessor, onFrame, ByteBufferPool(isDirect), processDispatcher) /** * A component that pull a frame from an input and push it to [onFrame] output. * * @param frameProcessor the frame processor * @param onFrame the output frame callback - * @param frameFactory the frame factory to create frames + * @param bufferPool the [ByteBuffer] pool * @param processDispatcher the dispatcher to process frames on */ class RawFramePullPush( private val frameProcessor: IFrameProcessor, val onFrame: suspend (RawFrame) -> Unit, - private val frameFactory: IRawFrameFactory, + private val bufferPool: ByteBufferPool, private val processDispatcher: CoroutineDispatcher, ) { private val coroutineScope = CoroutineScope(SupervisorJob() + processDispatcher) private val mutex = Mutex() - private var getFrame: (suspend (frameFactory: IRawFrameFactory) -> RawFrame)? = null + private val pool = RawFramePool() + + private var source: IAudioFrameSourceInternal? = null private val isReleaseRequested = AtomicBoolean(false) private var job: Job? = null - suspend fun setInput(getFrame: suspend (frameFactory: IRawFrameFactory) -> RawFrame) { + suspend fun setInput(source: IAudioFrameSourceInternal) { mutex.withLock { - this.getFrame = getFrame + this.source = source } } suspend fun removeInput() { mutex.withLock { - this.getFrame = null + this.source = null } } @@ -80,9 +83,11 @@ class RawFramePullPush( job = coroutineScope.launch { while (isActive) { val rawFrame = mutex.withLock { - val listener = getFrame ?: return@withLock null + val unwrapSource = source ?: return@withLock null try { - listener(frameFactory) + val buffer = bufferPool.get(unwrapSource.minBufferSize) + val timestampInUs = unwrapSource.fillAudioFrame(buffer) + pool.get(buffer, timestampInUs) } catch (t: Throwable) { Logger.e(TAG, "Failed to get frame: ${t.message}") null @@ -115,7 +120,8 @@ class RawFramePullPush( job?.cancel() job = null - frameFactory.clear() + pool.clear() + bufferPool.clear() } fun release() { @@ -127,7 +133,8 @@ class RawFramePullPush( } coroutineScope.cancel() - frameFactory.close() + pool.close() + bufferPool.close() } companion object { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt index 18c89bb6c..209ae9ea8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt @@ -15,26 +15,20 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.audio -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import java.nio.ByteBuffer interface IAudioFrameSourceInternal { /** - * Gets an audio frame from the source. - * - * @param frame the [RawFrame] to fill with audio data. - * @return a [RawFrame] containing audio data. + * Gets the size of the buffer to allocate. + * When using encoder is callback mode, it's unused. */ - fun fillAudioFrame(frame: RawFrame): RawFrame + val minBufferSize: Int /** * Gets an audio frame from the source. * - * The [RawFrame] to fill with audio data is created by the [frameFactory]. - * - * @param frameFactory a [IRawFrameFactory] to create [RawFrame]. - * @return a [RawFrame] containing audio data. + * @param buffer the [ByteBuffer] to fill with audio data. + * @return the timestamp in microseconds. */ - fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame + fun fillAudioFrame(buffer: ByteBuffer): Long } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt index 324438aaf..cc1746614 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt @@ -22,18 +22,18 @@ import android.media.AudioTimestamp import android.media.audiofx.AudioEffect import android.os.Build import androidx.annotation.RequiresPermission -import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordEffect.Companion.isValidUUID import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordEffect.Factory.Companion.getFactoryForEffectType +import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordSource.Companion.availableEffect import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordSource.Companion.isEffectAvailable -import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.elements.utils.extensions.type -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer import java.util.UUID /** @@ -42,7 +42,9 @@ import java.util.UUID */ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSource { private var audioRecord: AudioRecord? = null - private var bufferSize: Int? = null + private var _minBufferSize: Int? = null + override val minBufferSize: Int + get() = _minBufferSize ?: throw IllegalStateException("Audio source is not initialized") private var processor: EffectProcessor? = null private var pendingAudioEffects = mutableListOf() @@ -74,9 +76,9 @@ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSour } } - bufferSize = getMinBufferSize(config) + _minBufferSize = getMinBufferSize(config) - audioRecord = buildAudioRecord(config, bufferSize!!).also { + audioRecord = buildAudioRecord(config, _minBufferSize!!).also { val previousEffects = processor?.getAll() ?: emptyList() processor?.clear() @@ -157,30 +159,19 @@ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSour } - override fun fillAudioFrame(frame: RawFrame): RawFrame { + override fun fillAudioFrame( + buffer: ByteBuffer + ): Long { val audioRecord = requireNotNull(audioRecord) { "Audio source is not initialized" } if (audioRecord.recordingState != AudioRecord.RECORDSTATE_RECORDING) { throw IllegalStateException("Audio source is not recording") } - val buffer = frame.rawBuffer val length = audioRecord.read(buffer, buffer.remaining()) - if (length > 0) { - frame.timestampInUs = getTimestampInUs(audioRecord) - return frame - } else { - frame.close() + if (length <= 0) { throw IllegalArgumentException(audioRecordErrorToString(length)) } - } - - override fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame { - val bufferSize = requireNotNull(bufferSize) { "Buffer size is not initialized" } - - /** - * Dummy timestamp: it is overwritten later. - */ - return fillAudioFrame(frameFactory.create(bufferSize, 0)) + return getTimestampInUs(audioRecord) } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt index 3c206900d..a90d61103 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt @@ -15,17 +15,15 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.video -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import java.nio.ByteBuffer interface IVideoFrameSourceInternal { /** * Gets a video frame from a source. * - * @param frameFactory a [IRawFrameFactory] to create [RawFrame]. - * @return a [RawFrame] containing video data. + * @param buffer The buffer to fill with the video frame. + * @return The timestamp in microseconds of the video frame. */ - fun getVideoFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame + fun getVideoFrame(buffer: ByteBuffer): Long } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt deleted file mode 100644 index 759419483..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt +++ /dev/null @@ -1,31 +0,0 @@ -package io.github.thibaultbee.streampack.core.elements.utils.pool - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame - -interface IReadOnlyRawFrameFactory { - /** - * Creates a [RawFrame]. - * - * The returned frame must be released by calling [RawFrame.close] when it is not used anymore. - * - * @param bufferSize the buffer size - * @param timestampInUs the frame timestamp in µs - * @return a frame - */ - fun create(bufferSize: Int, timestampInUs: Long): RawFrame -} - -/** - * A pool of frames. - */ -interface IRawFrameFactory : IReadOnlyRawFrameFactory { - /** - * Clears the factory. - */ - fun clear() - - /** - * Closes the factory. - */ - fun close() -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt deleted file mode 100644 index dc4b3032b..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2025 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.elements.utils.pool - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.logger.Logger - -/** - * A factory to create [RawFrame]. - */ -fun RawFrameFactory(isDirect: Boolean): RawFrameFactory { - return RawFrameFactory(ByteBufferPool(isDirect)) -} - -/** - * A factory to create [RawFrame]. - */ -class RawFrameFactory(private val bufferPool: ByteBufferPool) : IRawFrameFactory { - override fun create(bufferSize: Int, timestampInUs: Long): RawFrame { - return RawFrame(bufferPool.get(bufferSize), timestampInUs) { rawFrame -> - try { - bufferPool.put(rawFrame.rawBuffer) - } catch (t: Throwable) { - Logger.w(TAG, "Error while putting buffer in pool: $t") - } - } - } - - override fun clear() { - bufferPool.clear() - } - - override fun close() { - bufferPool.close() - } - - companion object { - private const val TAG = "RawFramePool" - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt new file mode 100644 index 000000000..0c7b4e6a0 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import io.github.thibaultbee.streampack.core.elements.data.MutableRawFrame +import java.nio.ByteBuffer + + +/** + * A pool of [MutableRawFrame]. + */ +internal class RawFramePool() : ObjectPool() { + fun get( + rawBuffer: ByteBuffer, + timestampInUs: Long, + onClosed: (MutableRawFrame) -> Unit = {} + ): MutableRawFrame { + val frame = get() + + val onClosedHook = { frame: MutableRawFrame -> + onClosed(frame) + put(frame) + } + + return if (frame != null) { + frame.rawBuffer = rawBuffer + frame.timestampInUs = timestampInUs + frame.onClosed = onClosedHook + frame + } else { + MutableRawFrame( + rawBuffer = rawBuffer, + timestampInUs = timestampInUs, + onClosed = onClosedHook + ) + } + } + + companion object { + /** + * The default frame pool. + */ + internal val default by lazy { RawFramePool() } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt index bb745019c..26d417892 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.core.pipelines import android.content.Context import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory @@ -70,7 +71,6 @@ import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.io.Closeable import java.util.concurrent.atomic.AtomicBoolean /** @@ -246,25 +246,17 @@ open class StreamerPipeline( Logger.e(TAG, "Error while queueing audio frame to output: $t") } } else { - // Hook to close frame when all outputs have processed it - var numOfClosed = 0 - val onClosed = { frame: Closeable -> - numOfClosed++ - if (numOfClosed == audioStreamingOutput.size) { - frame.close() - } - } audioStreamingOutput.forEachIndexed { index, output -> try { output.queueAudioFrame( - frame.copy( - rawBuffer = if (index == audioStreamingOutput.lastIndex) { - frame.rawBuffer - } else { - frame.rawBuffer.duplicate() - }, - onClosed = onClosed - ) + if (index == audioStreamingOutput.lastIndex) { + frame + } else { + frame.copy( + rawBuffer = + frame.rawBuffer.duplicate() + ) + } ) } catch (t: Throwable) { Logger.e(TAG, "Error while queueing audio frame to output $output: $t") diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 05812e4d8..9560378e1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -24,10 +24,11 @@ import io.github.thibaultbee.streampack.core.elements.processing.RawFramePullPus import io.github.thibaultbee.streampack.core.elements.processing.audio.AudioFrameProcessor import io.github.thibaultbee.streampack.core.elements.processing.audio.IAudioFrameProcessor import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioFrameSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.utils.ConflatedJob -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Companion.THREAD_NAME_AUDIO_PREPROCESSING import io.github.thibaultbee.streampack.core.pipelines.IAudioDispatcherProvider @@ -200,15 +201,7 @@ internal class AudioInput( } } - when (port) { - is PushAudioPort -> { - port.setInput(newAudioSource::getAudioFrame) - } - - is CallbackAudioPort -> { - port.setInput(newAudioSource::fillAudioFrame) - } - } + port.setInput(newAudioSource) // Replace audio source sourceInternalFlow.emit(newAudioSource) @@ -380,8 +373,8 @@ internal class AudioInput( internal class CallbackConfig : Config() } -private sealed interface IAudioPort : Streamable, Releasable { - suspend fun setInput(getFrame: T) +private sealed interface IAudioPort : Streamable, Releasable { + suspend fun setInput(source: IAudioFrameSourceInternal) suspend fun removeInput() } @@ -389,7 +382,7 @@ private class PushAudioPort( audioFrameProcessor: AudioFrameProcessor, config: PushConfig, dispatcherProvider: IAudioDispatcherProvider -) : IAudioPort<(frameFactory: IRawFrameFactory) -> RawFrame> { +) : IAudioPort { private val audioPullPush = RawFramePullPush( audioFrameProcessor, config.onFrame, @@ -399,8 +392,8 @@ private class PushAudioPort( ) ) - override suspend fun setInput(getFrame: (frameFactory: IRawFrameFactory) -> RawFrame) { - audioPullPush.setInput(getFrame) + override suspend fun setInput(source: IAudioFrameSourceInternal) { + audioPullPush.setInput(source) } override suspend fun removeInput() { @@ -421,32 +414,35 @@ private class PushAudioPort( } private class CallbackAudioPort(private val audioFrameProcessor: AudioFrameProcessor) : - IAudioPort<(frame: RawFrame) -> RawFrame> { - private var getFrame: ((frame: RawFrame) -> RawFrame)? = null + IAudioPort { private val mutex = Mutex() + private val pool = RawFramePool() + + private var source: IAudioFrameSourceInternal? = null var audioFrameRequestedListener: OnFrameRequestedListener = object : OnFrameRequestedListener { override suspend fun onFrameRequested(buffer: ByteBuffer): RawFrame { val frame = mutex.withLock { - val getFrame = requireNotNull(getFrame) { + val source = requireNotNull(source) { "Audio frame requested listener is not set yet" } - getFrame(RawFrame(buffer, 0)) + val timestampInUs = source.fillAudioFrame(buffer) + pool.get(buffer, timestampInUs) } return audioFrameProcessor.processFrame(frame) } } - override suspend fun setInput(getFrame: (frame: RawFrame) -> RawFrame) { + override suspend fun setInput(source: IAudioFrameSourceInternal) { mutex.withLock { - this.getFrame = getFrame + this.source = source } } override suspend fun removeInput() { mutex.withLock { - this.getFrame = null + this.source = null } } diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt index 2c579cdcd..393c6a473 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt @@ -18,11 +18,11 @@ package io.github.thibaultbee.streampack.core.elements.sources import android.media.MediaRecorder import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSource import io.github.thibaultbee.streampack.core.elements.utils.StubLogger -import io.github.thibaultbee.streampack.core.elements.utils.StubRawFrameFactory import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.test.runTest import org.junit.Assert import org.junit.Test +import java.nio.ByteBuffer class MicrophoneSourceUnitTest { init { @@ -38,7 +38,7 @@ class MicrophoneSourceUnitTest { } catch (_: Throwable) { } try { - microphoneSource.getAudioFrame(StubRawFrameFactory()) + microphoneSource.fillAudioFrame(ByteBuffer.allocate(microphoneSource.minBufferSize)) Assert.fail() } catch (_: Throwable) { } diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt deleted file mode 100644 index a4d24568a..000000000 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2025 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.elements.utils - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IGetOnlyBufferPool -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory -import java.nio.ByteBuffer - -/** - * Stub buffer pool for testing. - * - * It always returns a new allocated buffer. - */ -class StubRawFrameFactory(private val bufferPool: IGetOnlyBufferPool = StubBufferPool()) : - IReadOnlyRawFrameFactory { - override fun create(bufferSize: Int, timestampInUs: Long): RawFrame { - return RawFrame(bufferPool.get(bufferSize), timestampInUs) { rawFrame -> - // Do nothing - } - } -} \ No newline at end of file From aad0dbe04f4ed4fbf4cc55818b2b8c5bf457b51a Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 28 Jan 2026 11:14:05 +0100 Subject: [PATCH 09/72] feat(*): avoid to duplicate extra for each frames --- .../streampack/core/elements/data/Frame.kt | 51 +++++++++++++++++-- .../encoders/mediacodec/MediaCodecEncoder.kt | 42 ++++++++------- .../composites/muxers/mp4/models/Chunk.kt | 4 +- .../endpoints/composites/muxers/ts/TsMuxer.kt | 24 +++++---- .../core/elements/utils/pool/FramePool.kt | 3 +- .../core/elements/utils/FakeFrames.kt | 9 ++-- .../muxer/utils/FlvAudioDataFactory.kt | 7 +-- .../muxer/utils/FlvVideoDataFactory.kt | 16 +++--- 8 files changed, 110 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index c1a952932..c9c08acea 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -109,11 +109,12 @@ interface Frame : Closeable { val isKeyFrame: Boolean /** - * Contains csd buffers for key frames and audio frames only. + * Gets the csd buffers for key frames and audio frames. + * * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - val extra: List? + val extra: Extra? /** * Contains frame format.. @@ -136,7 +137,7 @@ fun Frame.copy( ptsInUs: Long = this.ptsInUs, dtsInUs: Long? = this.dtsInUs, isKeyFrame: Boolean = this.isKeyFrame, - extra: List? = this.extra, + extra: Extra? = this.extra, format: MediaFormat = this.format, onClosed: (Frame) -> Unit = {} ): Frame { @@ -180,7 +181,7 @@ data class MutableFrame( * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - override var extra: List?, + override var extra: Extra?, /** * Contains frame format.. @@ -201,3 +202,45 @@ data class MutableFrame( } } } + +/** + * Ensures that extra are not used at the same time. + * + * After accessing the extra, they are automatically rewind for a new usage. + */ +class Extra(private val extraBuffers: List) { + private val lock = Any() + + val _length by lazy { extraBuffers.sumOf { it.remaining() } } + + fun getLength(): Int { + return synchronized(lock) { + _length + } + } + + fun get(extra: List.() -> T): T { + return synchronized(lock) { + val result = extraBuffers.extra() + extraBuffers.forEach { it.rewind() } + result + } + } +} + +/** + * Gets the duplicated extra. + * + * Prefers to use [Extra.get] as it does not create new resources. + */ +val Extra.extra: List + get() = get { this.map { it.duplicate() } } + +/** + * Gets the duplicated extra at [index] + * + * Prefers to use [Extra.get] as it does not create new resources. + */ +fun Extra.get(index: Int): ByteBuffer { + return get { this[index].duplicate() } +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index 9e2d9b97c..9ebbde6a6 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -22,6 +22,7 @@ import android.media.MediaFormat import android.os.Bundle import android.util.Log import android.view.Surface +import io.github.thibaultbee.streampack.core.elements.data.Extra import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.EncoderMode @@ -63,14 +64,8 @@ internal constructor( private val mediaCodec: MediaCodec private val format: MediaFormat - private var outputFormat: MediaFormat? = null - set(value) { - extra = value?.extra - field = value - } - private var extra: List? = null - private val frameFactory by lazy { FrameFactory(mediaCodec, isVideo) } + private val frameFactory by lazy { FrameFactory(mediaCodec, isVideo, mediaCodec.outputFormat) } private val isVideo = encoderConfig.isVideo private val tag = if (isVideo) VIDEO_ENCODER_TAG else AUDIO_ENCODER_TAG + "(${this.hashCode()})" @@ -360,7 +355,7 @@ internal constructor( info.isValid -> { try { val frame = frameFactory.frame( - index, extra, outputFormat!!, info, tag + index, info, tag ) try { listener.outputChannel.send(frame) @@ -444,7 +439,6 @@ internal constructor( } override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { - outputFormat = format Logger.i(tag, "Format changed : $format") } @@ -566,7 +560,6 @@ internal constructor( if (outputBufferId >= 0) { processOutputFrameUnsafe(mediaCodec, outputBufferId, bufferInfo) } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - outputFormat = mediaCodec.outputFormat Logger.i(tag, "Format changed: ${mediaCodec.outputFormat}") } } @@ -587,6 +580,18 @@ internal constructor( } } + fun FrameFactory( + codec: MediaCodec, + isVideo: Boolean, + outputFormat: MediaFormat + ): FrameFactory { + val extraBuffers: List? by lazy { outputFormat.extra } + val extra: Extra? by lazy { + extraBuffers?.map { it.duplicate() }?.let { Extra(it) } + } + return FrameFactory(codec, isVideo, outputFormat, extra, extraBuffers) + } + /** * A workaround to address the fact that some AAC encoders do not provide frame with `presentationTimeUs` in order. * If a frame is received with a timestamp lower or equal to the previous one, it is corrected by adding 1 to the previous timestamp. @@ -596,7 +601,10 @@ internal constructor( */ class FrameFactory( private val codec: MediaCodec, - private val isVideo: Boolean + private val isVideo: Boolean, + private val outputFormat: MediaFormat, + private val extra: Extra?, + private val extraBuffers: List? ) : Closeable { private var previousPresentationTimeUs = 0L @@ -609,8 +617,6 @@ internal constructor( */ fun frame( index: Int, - extra: List?, - outputFormat: MediaFormat, info: BufferInfo, tag: String ): Frame { @@ -620,7 +626,7 @@ internal constructor( Logger.w(tag, "Correcting timestamp: $pts <= $previousPresentationTimeUs") } previousPresentationTimeUs = pts - return createFrame(codec, index, extra, outputFormat, pts, info.isKeyFrame, tag) + return createFrame(codec, index, pts, info.isKeyFrame, tag) } /** @@ -633,20 +639,18 @@ internal constructor( private fun createFrame( codec: MediaCodec, index: Int, - extra: List?, - outputFormat: MediaFormat, ptsInUs: Long, isKeyFrame: Boolean, tag: String ): Frame { val buffer = requireNotNull(codec.getOutputBuffer(index)) val extra = if (isKeyFrame || !isVideo) { - extra!!.map { it.duplicate() } + extra!! } else { null } - val rawBuffer = if (extra != null) { - buffer.removePrefixes(extra) + val rawBuffer = if (extraBuffers != null) { + buffer.removePrefixes(extraBuffers) } else { buffer } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt index 2ff9387c4..373b6eb42 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.extra import io.github.thibaultbee.streampack.core.elements.utils.extensions.unzip import java.nio.ByteBuffer @@ -50,8 +51,9 @@ class Chunk(val id: Int) { private val sampleSizes: List get() = samples.map { it.frame.rawBuffer.remaining() } + // TODO: pass `Extra` instead of `List` val extra: List> - get() = samples.mapNotNull { it.frame.extra }.unzip() + get() = samples.mapNotNull { it.frame.extra?.extra }.unzip() val format: List get() = samples.map { it.frame.format } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index ba17e1291..127662e43 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -88,12 +88,14 @@ class TsMuxer : IMuxerInternal { val extra = frame.extra ?: throw MissingFormatArgumentException("Missing extra for AVC") val buffer = - ByteBuffer.allocate(6 + extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(6 + extra.getLength() + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the AVC access unit buffer.putInt(0x00000001) buffer.put(0x09.toByte()) buffer.put(0xf0.toByte()) - extra.forEach { buffer.put(it) } + extra.get { + forEach { buffer.put(it) } + } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) @@ -108,13 +110,15 @@ class TsMuxer : IMuxerInternal { val extra = frame.extra ?: throw MissingFormatArgumentException("Missing extra for HEVC") val buffer = - ByteBuffer.allocate(7 + extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(7 + extra.getLength() + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the HEVC access unit buffer.putInt(0x00000001) buffer.put(0x46.toByte()) buffer.put(0x01.toByte()) buffer.put(0x50.toByte()) - extra.forEach { buffer.put(it) } + extra.get { + forEach { buffer.put(it) } + } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) @@ -130,11 +134,13 @@ class TsMuxer : IMuxerInternal { frame.rawBuffer, pes.stream.config as AudioCodecConfig ).toByteBuffer() } else { - LATMFrameWriter.fromDecoderSpecificInfo( - frame.rawBuffer, - frame.extra!!.first() - ) - .toByteBuffer() + frame.extra!!.get { + LATMFrameWriter.fromDecoderSpecificInfo( + frame.rawBuffer, + this.first() + ) + .toByteBuffer() + } } ) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt index 7302ef272..c1e824e97 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.utils.pool import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.data.Extra import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer @@ -29,7 +30,7 @@ internal class FramePool() : ObjectPool() { ptsInUs: Long, dtsInUs: Long?, isKeyFrame: Boolean, - extra: List?, + extra: Extra?, format: MediaFormat, onClosed: (MutableFrame) -> Unit ): MutableFrame { diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt index b31ab2b44..542ef0a8e 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.utils import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.data.Extra import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer @@ -56,9 +57,11 @@ object FakeFrames { pts, dts, isKeyFrame, - listOf( - ByteBuffer.wrap( - Random.nextBytes(10) + Extra( + listOf( + ByteBuffer.wrap( + Random.nextBytes(10) + ) ) ), format = format diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt index b0d1013b5..54df39c08 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt @@ -24,6 +24,7 @@ import io.github.komedia.komuxer.flv.tags.audio.ExtendedAudioDataFactory import io.github.komedia.komuxer.flv.tags.audio.codedFrame import io.github.komedia.komuxer.flv.tags.audio.sequenceStart import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.get import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.utils.av.audio.opus.OpusCsdParser import io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.muxer.AudioFlvMuxerInfo @@ -72,7 +73,7 @@ internal class FlvAudioDataFactory { val flvDatas = mutableListOf() if (withSequenceStart) { - val decoderConfigurationRecordBuffer = frame.extra!![0] + val decoderConfigurationRecordBuffer = frame.extra!!.get(0) flvDatas.add( aacAudioDataFactory.sequenceStart( decoderConfigurationRecordBuffer @@ -155,7 +156,7 @@ internal class FlvAudioDataFactory { return FlvExtendedAudioDataFactory( ExtendedAudioDataFactory(AudioFourCC.AAC), onSequenceStart = { frame -> - frame.extra!![0] + frame.extra!!.get(0) } ) } @@ -165,7 +166,7 @@ internal class FlvAudioDataFactory { ExtendedAudioDataFactory(AudioFourCC.OPUS), onSequenceStart = { frame -> frame.extra?.let { - OpusCsdParser.findIdentificationHeader(it[0]) + OpusCsdParser.findIdentificationHeader(it.get(0)) } } ) diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt index 6b541bc92..dccc05e0d 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt @@ -28,6 +28,8 @@ import io.github.komedia.komuxer.flv.tags.video.VideoFrameType import io.github.komedia.komuxer.flv.tags.video.codedFrame import io.github.komedia.komuxer.flv.tags.video.sequenceStart import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.extra +import io.github.thibaultbee.streampack.core.elements.data.get import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.utils.av.video.avc.AVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.hevc.HEVCDecoderConfigurationRecord @@ -64,10 +66,11 @@ internal class FlvVideoDataFactory { VideoFrameType.INTER } if (frame.isKeyFrame && withSequenceStart) { + val extra = frame.extra!!.extra val decoderConfigurationRecordBuffer = AVCDecoderConfigurationRecord.fromParameterSets( - frame.extra!![0], - frame.extra!![1] + extra[0], + extra[1] ).toByteBuffer() flvDatas.add( factory.sequenceStart( @@ -185,10 +188,11 @@ internal class FlvVideoDataFactory { private fun createHEVCFactory(): IVideoDataFactory { return FlvExtendedVideoDataFactory(HEVCExtendedVideoDataFactory()) { frame -> // Extra is VPS, SPS, PPS + val extra = frame.extra!!.extra HEVCDecoderConfigurationRecord.fromParameterSets( - frame.extra!![0], - frame.extra!![1], - frame.extra!![2] + extra[0], + extra[1], + extra[2] ).toByteBuffer() } } @@ -196,7 +200,7 @@ internal class FlvVideoDataFactory { private fun createAV1Factory(): IVideoDataFactory { return FlvExtendedVideoDataFactory(ExtendedVideoDataFactory(VideoFourCC.AV1)) { frame -> // Extra is AV1CodecConfigurationRecord - frame.extra!![0] + frame.extra!!.get(0) } } From 212840536499e644ce18f71df58517fbe3f4c1e2 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:08:26 +0100 Subject: [PATCH 10/72] refactor(core): camera: move `applyRepeatingSessionSync` method to the camera settings --- .../sources/video/camera/CameraSettings.kt | 60 +++++++++- .../camera/controllers/CameraController.kt | 15 ++- .../controllers/CameraSessionController.kt | 107 ++++-------------- 3 files changed, 89 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index bc87cc182..c532ff208 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -22,6 +22,7 @@ import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraMetadata import android.hardware.camera2.CaptureRequest import android.hardware.camera2.CaptureResult +import android.hardware.camera2.TotalCaptureResult import android.hardware.camera2.params.ColorSpaceTransform import android.hardware.camera2.params.MeteringRectangle import android.hardware.camera2.params.RggbChannelVector @@ -32,6 +33,7 @@ import androidx.annotation.IntRange import androidx.annotation.RequiresApi import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.is90or270 import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraController +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoExposureModes import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoFocusModes import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoWhiteBalanceModes @@ -55,10 +57,12 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.launchIn import io.github.thibaultbee.streampack.core.elements.utils.extensions.normalize import io.github.thibaultbee.streampack.core.elements.utils.extensions.rotate import io.github.thibaultbee.streampack.core.logger.Logger +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import java.util.concurrent.atomic.AtomicLong /** @@ -129,6 +133,8 @@ class CameraSettings internal constructor( val focusMetering = FocusMetering(coroutineScope, characteristics, this, zoom, focus, exposure, whiteBalance) + private val tagBundleFactory = TagBundle.TagBundleFactory() + /** * Directly gets a [CaptureRequest] from the camera. * @@ -153,13 +159,65 @@ class CameraSettings internal constructor( * * This method returns when the capture callback is received with the passed request. */ - suspend fun applyRepeatingSessionSync() = cameraController.setRepeatingSessionSync() + suspend fun applyRepeatingSessionSync() { + val deferred = CompletableDeferred() + + val tag = tagBundleFactory.create() + val captureCallback = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + val resultTag = result.request.tag as? TagBundle + val keyId = resultTag?.keyId ?: return false + if (keyId >= tag.keyId) { + deferred.complete(Unit) + return true + } + return false + } + } + + cameraController.addCaptureCallbackListener(captureCallback) + cameraController.setRepeatingSession(tag) + deferred.await() + } /** * Applies settings to the camera repeatedly. */ suspend fun applyRepeatingSession() = cameraController.setRepeatingSession() + private class TagBundle private constructor(val keyId: Long) { + private val tagMap = mutableMapOf().apply { + put(TAG_KEY_ID, keyId) + } + + val keys: Set + get() = tagMap.keys + + fun setTag(key: String, value: Any?) { + tagMap[key] = value + } + + companion object { + private const val TAG_KEY_ID = "TAG_KEY_ID" + } + + /** + * Factory for [TagBundle]. + * + * The purpose is to make sure the tag always contains an increasing id. + */ + class TagBundleFactory { + /** + * Next session id. + */ + private val nextSessionUpdateId = AtomicLong(0) + + fun create(): TagBundle { + return TagBundle(nextSessionUpdateId.getAndIncrement()) + } + } + } + class Flash( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index 13c33e653..82713b589 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -21,6 +21,7 @@ import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest import android.util.Range import androidx.annotation.RequiresPermission +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.CameraCaptureSessionCompatBuilder import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraDispatcherProvider import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface @@ -378,21 +379,23 @@ internal class CameraController( } /** - * Sets a repeating session sync with the current capture request. + * Adds a capture callback listener to the current capture session. * - * It returns only when the capture callback has been called for the first time. + * The listener is removed when the [CaptureResultListener] returns true. */ - suspend fun setRepeatingSessionSync() { + fun addCaptureCallbackListener(listener: CaptureResultListener) { val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setRepeatingSessionSync() + sessionController.addCaptureCallbackListener(listener) } /** * Sets a repeating session with the current capture request. + * + * @param tag A tag to associate with the session. */ - suspend fun setRepeatingSession() { + suspend fun setRepeatingSession(tag: Any? = null) { val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setRepeatingSession() + sessionController.setRepeatingSession(tag) } private suspend fun closeControllers() { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index fc3843e3f..80d768435 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -37,9 +37,6 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.util.concurrent.atomic.AtomicLong -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine internal class CameraSessionController private constructor( private val coroutineScope: CoroutineScope, @@ -58,8 +55,6 @@ internal class CameraSessionController private constructor( private val requestTargetMutex = Mutex() - private val nextSessionUpdateId = AtomicLong(0) - /** * A default capture callback that logs the failure reason. */ @@ -75,7 +70,7 @@ internal class CameraSessionController private constructor( private val sessionCallback = CameraControlSessionCallback(coroutineScope) private val captureCallbacks = - mutableSetOf(captureCallback, sessionCallback) + setOf(captureCallback, sessionCallback) suspend fun isEmpty() = withContext(coroutineScope.coroutineContext) { requestTargetMutex.withLock { captureRequestBuilder.isEmpty() } @@ -250,72 +245,21 @@ internal class CameraSessionController private constructor( } } - suspend fun removeCaptureCallback( - cameraCaptureCallback: CaptureCallback - ) { - withContext(coroutineScope.coroutineContext) { - captureSessionMutex.withLock { - if (isClosed) { - Logger.w(TAG, "Camera session controller is released") - return@withContext - } - - captureCallbacks.remove(cameraCaptureCallback) - } - } - } - /** - * Sets a repeating session with the current capture request. + * Adds a capture callback listener to the current capture session. + * + * The listener is removed when the session is closed. */ - suspend fun setRepeatingSessionSync() { - if (captureRequestBuilder.isEmpty()) { - Logger.w(TAG, "Capture request is empty") - return - } - withContext(coroutineScope.coroutineContext) { - captureSessionMutex.withLock { - if (isClosed) { - Logger.w(TAG, "Camera session controller is released") - return@withContext - } - - suspendCoroutine { continuation -> - val nextSessionId = nextSessionUpdateId.getAndIncrement() - - val captureCallback = object : CaptureResultListener { - override fun onCaptureResult(result: TotalCaptureResult): Boolean { - val tag = result.request.tag as? TagBundle - val keyId = tag?.keyId ?: return false - if (keyId >= nextSessionId) { - continuation.resume(Unit) - return true - } - return false - } - } - - sessionCallback.addListener(captureCallback) - - captureRequestBuilder.setTag( - TagBundle().apply { - keyId = nextSessionId - } - ) - sessionCompat.setRepeatingSingleRequest( - captureSession, - captureRequestBuilder.build(), - MultiCaptureCallback(captureCallbacks) - ) - } - } - } + fun addCaptureCallbackListener(listener: CaptureResultListener) { + sessionCallback.addListener(listener) } /** * Sets a repeating session with the current capture request. + * + * @param tag A tag to associate with the session. */ - suspend fun setRepeatingSession() { + suspend fun setRepeatingSession(tag: Any? = null) { if (captureRequestBuilder.isEmpty()) { Logger.w(TAG, "Capture request is empty") return @@ -327,6 +271,8 @@ internal class CameraSessionController private constructor( return@withContext } + tag?.let { captureRequestBuilder.setTag(it) } + sessionCompat.setRepeatingSingleRequest( captureSession, captureRequestBuilder.build(), @@ -514,27 +460,6 @@ internal class CameraSessionController private constructor( } } - private class TagBundle { - private val tagMap = mutableMapOf() - - var keyId: Long? - get() = tagMap[TAG_KEY_ID] as? Long - set(value) { - tagMap[TAG_KEY_ID] = value - } - - val keys: Set - get() = tagMap.keys - - fun setTag(key: String, value: Any?) { - tagMap[key] = value - } - - companion object { - private const val TAG_KEY_ID = "TAG_KEY_ID" - } - } - interface CaptureResultListener { /** * Called when a capture result is received. @@ -545,11 +470,21 @@ internal class CameraSessionController private constructor( fun onCaptureResult(result: TotalCaptureResult): Boolean } + /** + * A capture callback that wraps multiple [CaptureResultListener]. + * + * @param coroutineScope The coroutine scope to use. + */ private class CameraControlSessionCallback(private val coroutineScope: CoroutineScope) : CaptureCallback() { /* synthetic accessor */ private val resultListeners = mutableSetOf() + /** + * Adds a capture result listener. + * + * The listener is removed when [removeListener] is explicitly called or when [CaptureResultListener] returns true. + */ fun addListener(listener: CaptureResultListener) { resultListeners.add(listener) } From f18dea725f839a3ce604cf5c20842ce269771484 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:21:46 +0100 Subject: [PATCH 11/72] feat(core): add a flow to get the current physical camera Id --- .../sources/video/camera/CameraSettings.kt | 26 +++++++++++++++++++ .../camera/controllers/CameraController.kt | 15 ++++++++++- .../controllers/CameraSessionController.kt | 22 +++++++++++----- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index c532ff208..9b0a41b1a 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -60,6 +60,11 @@ import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.callbackFlow +import kotlinx.coroutines.flow.conflate +import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.atomic.AtomicLong @@ -81,6 +86,27 @@ class CameraSettings internal constructor( */ val cameraId = cameraController.cameraId + @RequiresApi(Build.VERSION_CODES.Q) + private fun getPhysicalCameraIdCallbackFlow() = callbackFlow { + val captureCallback = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + trySend(result.get(CaptureResult.LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_ID)!!) + return false + } + } + cameraController.addCaptureCallbackListener(captureCallback) + awaitClose { + cameraController.removeCaptureCallbackListener(captureCallback) + } + }.conflate().distinctUntilChanged() + + /** + * Current physical camera id. + */ + val physicalCameraIdFlow: Flow + @RequiresApi(Build.VERSION_CODES.Q) + get() = getPhysicalCameraIdCallbackFlow() + /** * Whether the camera is available. * To be used before calling any camera settings. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index 82713b589..5f7ffce5c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -381,13 +381,26 @@ internal class CameraController( /** * Adds a capture callback listener to the current capture session. * - * The listener is removed when the [CaptureResultListener] returns true. + * The listener is removed when the [CaptureResultListener] returns true or [removeCaptureCallbackListener] is called. + * + * @param listener The listener to add */ fun addCaptureCallbackListener(listener: CaptureResultListener) { val sessionController = requireNotNull(sessionController) { "SessionController is null" } sessionController.addCaptureCallbackListener(listener) } + /** + * Removes a capture callback listener from the current capture session. + * + * @param listener The listener to remove + */ + fun removeCaptureCallbackListener(listener: CaptureResultListener) { + val sessionController = requireNotNull(sessionController) { "SessionController is null" } + sessionController.removeCaptureCallbackListener(listener) + } + + /** * Sets a repeating session with the current capture request. * diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index 80d768435..ab014f2eb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -40,9 +40,10 @@ import kotlinx.coroutines.withContext internal class CameraSessionController private constructor( private val coroutineScope: CoroutineScope, + private val captureSession: CameraCaptureSession, private val captureRequestBuilder: CaptureRequestWithTargetsBuilder, + private val sessionCallback: CameraControlSessionCallback, private val sessionCompat: ICameraCaptureSessionCompat, - private val captureSession: CameraCaptureSession, private val outputs: List, val dynamicRange: Long, val cameraIsClosedFlow: StateFlow, @@ -67,8 +68,6 @@ internal class CameraSessionController private constructor( } } - private val sessionCallback = CameraControlSessionCallback(coroutineScope) - private val captureCallbacks = setOf(captureCallback, sessionCallback) @@ -248,12 +247,21 @@ internal class CameraSessionController private constructor( /** * Adds a capture callback listener to the current capture session. * - * The listener is removed when the session is closed. + * The listener is removed when it returns true or [removeCaptureCallbackListener] is called. */ fun addCaptureCallbackListener(listener: CaptureResultListener) { sessionCallback.addListener(listener) } + /** + * Removes a capture callback listener from the current capture session. + * + * @param listener The listener to remove + */ + fun removeCaptureCallbackListener(listener: CaptureResultListener) { + sessionCallback.removeListener(listener) + } + /** * Sets a repeating session with the current capture request. * @@ -360,9 +368,10 @@ internal class CameraSessionController private constructor( val controller = CameraSessionController( coroutineScope, + newCaptureSession, captureRequestBuilder, + sessionCallback, sessionCompat, - newCaptureSession, outputs, dynamicRange, cameraDeviceController.isClosedFlow, @@ -411,9 +420,10 @@ internal class CameraSessionController private constructor( } return CameraSessionController( coroutineScope, + captureSession, captureRequestBuilder, + CameraControlSessionCallback(coroutineScope), sessionCompat, - captureSession, outputs, dynamicRange, cameraDeviceController.isClosedFlow, From 41dd38eb7179836352fc02e9619861288f203d60 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:52:06 +0100 Subject: [PATCH 12/72] refactor(core): camera: move session callback out of session controller --- .../camera/controllers/CameraController.kt | 17 +++-- .../controllers/CameraSessionController.kt | 69 ++----------------- .../camera/utils/CameraSessionCallback.kt | 67 ++++++++++++++++++ 3 files changed, 80 insertions(+), 73 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index 5f7ffce5c..ec7000895 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -24,6 +24,7 @@ import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.CameraCaptureSessionCompatBuilder import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraDispatcherProvider +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder @@ -58,6 +59,8 @@ internal class CameraController( private var deviceController: CameraDeviceController? = null private var sessionController: CameraSessionController? = null + private val sessionCallback = CameraSessionCallback(coroutineScope) + private val controllerMutex = Mutex() private val outputs = mutableMapOf() @@ -161,8 +164,9 @@ internal class CameraController( val deviceController = getDeviceController() CameraSessionController.create( coroutineScope, - sessionCompat, deviceController, + sessionCallback, + sessionCompat, outputs.values.toList(), dynamicRange = dynamicRangeProfile.dynamicRange, fpsRange = fpsRange, @@ -381,13 +385,10 @@ internal class CameraController( /** * Adds a capture callback listener to the current capture session. * - * The listener is removed when the [CaptureResultListener] returns true or [removeCaptureCallbackListener] is called. - * - * @param listener The listener to add + * The listener is removed when it returns true or [removeCaptureCallbackListener] is called. */ fun addCaptureCallbackListener(listener: CaptureResultListener) { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.addCaptureCallbackListener(listener) + sessionCallback.addListener(listener) } /** @@ -396,11 +397,9 @@ internal class CameraController( * @param listener The listener to remove */ fun removeCaptureCallbackListener(listener: CaptureResultListener) { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.removeCaptureCallbackListener(listener) + sessionCallback.removeListener(listener) } - /** * Sets a repeating session with the current capture request. * diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index ab014f2eb..ab193b4ad 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -23,6 +23,7 @@ import android.hardware.camera2.TotalCaptureResult import android.util.Range import android.view.Surface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.ICameraCaptureSessionCompat +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder @@ -42,7 +43,7 @@ internal class CameraSessionController private constructor( private val coroutineScope: CoroutineScope, private val captureSession: CameraCaptureSession, private val captureRequestBuilder: CaptureRequestWithTargetsBuilder, - private val sessionCallback: CameraControlSessionCallback, + private val sessionCallback: CameraSessionCallback, private val sessionCompat: ICameraCaptureSessionCompat, private val outputs: List, val dynamicRange: Long, @@ -244,24 +245,6 @@ internal class CameraSessionController private constructor( } } - /** - * Adds a capture callback listener to the current capture session. - * - * The listener is removed when it returns true or [removeCaptureCallbackListener] is called. - */ - fun addCaptureCallbackListener(listener: CaptureResultListener) { - sessionCallback.addListener(listener) - } - - /** - * Removes a capture callback listener from the current capture session. - * - * @param listener The listener to remove - */ - fun removeCaptureCallbackListener(listener: CaptureResultListener) { - sessionCallback.removeListener(listener) - } - /** * Sets a repeating session with the current capture request. * @@ -391,8 +374,9 @@ internal class CameraSessionController private constructor( suspend fun create( coroutineScope: CoroutineScope, - sessionCompat: ICameraCaptureSessionCompat, cameraDeviceController: CameraDeviceController, + sessionCallback: CameraSessionCallback, + sessionCompat: ICameraCaptureSessionCompat, outputs: List, dynamicRange: Long, fpsRange: Range, @@ -422,7 +406,7 @@ internal class CameraSessionController private constructor( coroutineScope, captureSession, captureRequestBuilder, - CameraControlSessionCallback(coroutineScope), + sessionCallback, sessionCompat, outputs, dynamicRange, @@ -479,47 +463,4 @@ internal class CameraSessionController private constructor( */ fun onCaptureResult(result: TotalCaptureResult): Boolean } - - /** - * A capture callback that wraps multiple [CaptureResultListener]. - * - * @param coroutineScope The coroutine scope to use. - */ - private class CameraControlSessionCallback(private val coroutineScope: CoroutineScope) : - CaptureCallback() { - /* synthetic accessor */ - private val resultListeners = mutableSetOf() - - /** - * Adds a capture result listener. - * - * The listener is removed when [removeListener] is explicitly called or when [CaptureResultListener] returns true. - */ - fun addListener(listener: CaptureResultListener) { - resultListeners.add(listener) - } - - fun removeListener(listener: CaptureResultListener) { - resultListeners.remove(listener) - } - - override fun onCaptureCompleted( - session: CameraCaptureSession, - request: CaptureRequest, - result: TotalCaptureResult - ) { - coroutineScope.launch { - val removeSet = mutableSetOf() - for (listener in resultListeners) { - val isFinished: Boolean = listener.onCaptureResult(result) - if (isFinished) { - removeSet.add(listener) - } - } - if (!removeSet.isEmpty()) { - resultListeners.removeAll(removeSet) - } - } - } - } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt new file mode 100644 index 000000000..ae8a8ad2c --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils + +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCaptureSession.CaptureCallback +import android.hardware.camera2.CaptureRequest +import android.hardware.camera2.TotalCaptureResult +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch + +/** + * A capture callback that wraps multiple [CaptureResultListener]. + * + * @param coroutineScope The coroutine scope to use. + */ +internal class CameraSessionCallback(private val coroutineScope: CoroutineScope) : + CaptureCallback() { + /* synthetic accessor */ + private val resultListeners = mutableSetOf() + + /** + * Adds a capture result listener. + * + * The listener is removed when [removeListener] is explicitly called or when [CaptureResultListener] returns true. + */ + fun addListener(listener: CaptureResultListener) { + resultListeners.add(listener) + } + + fun removeListener(listener: CaptureResultListener) { + resultListeners.remove(listener) + } + + override fun onCaptureCompleted( + session: CameraCaptureSession, + request: CaptureRequest, + result: TotalCaptureResult + ) { + coroutineScope.launch { + val removeSet = mutableSetOf() + for (listener in resultListeners) { + val isFinished: Boolean = listener.onCaptureResult(result) + if (isFinished) { + removeSet.add(listener) + } + } + if (!removeSet.isEmpty()) { + resultListeners.removeAll(removeSet) + } + } + } +} \ No newline at end of file From f647a8fd34d080a36447afd72ef7de9bca48f251 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Fri, 23 Jan 2026 16:51:40 +0100 Subject: [PATCH 13/72] feat(core): add custom audio effect API in the audio processor --- .../elements/processing/IEffectProcessor.kt | 62 ++++++++++++++++ .../elements/processing/IFrameProcessor.kt | 28 -------- .../elements/processing/RawFramePullPush.kt | 6 +- .../elements/processing/audio/AudioEffects.kt | 37 +++++++--- .../processing/audio/AudioFrameProcessor.kt | 72 ++++++++++++++++--- .../processing/audio/IAudioFrameProcessor.kt | 2 +- .../core/pipelines/inputs/AudioInput.kt | 19 +++-- 7 files changed, 170 insertions(+), 56 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt delete mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt new file mode 100644 index 000000000..0bad99623 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt @@ -0,0 +1,62 @@ +/* + * Copyright 2025 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.processing + +/** + * Interface to process a data and returns a result. + * + * @param T type of data to proces) + */ +interface IProcessor { + /** + * Process a data and returns a result. + * + * @param data data to process + * @return processed data + */ + fun process(data: T): T +} + +/** + * Interface to process a data and returns a result. + * + * @param T type of data to proces) + */ +interface IEffectProcessor { + /** + * Process a data and returns a result. + * + * @param isMuted whether the data contains only 0 + * @param data data to process + * @return processed data + */ + fun process(isMuted: Boolean, data: T): T +} + +/** + * Interface to process a data. + * + * @param T type of data to process + */ +interface IEffectConsumer { + /** + * Process a data. + * + * @param isMuted whether the data contains only 0 + * @param data data to process + */ + fun consume(isMuted: Boolean, data: T) +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt deleted file mode 100644 index dc9375392..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2025 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.elements.processing - -import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.RawFrame - -/** - * Interface to process a frame. - * - * @param T type of frame to process (probably [RawFrame] or [Frame]) - */ -interface IFrameProcessor { - fun processFrame(frame: T): T -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt index f87b531ad..eca215f31 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt @@ -32,7 +32,7 @@ import kotlinx.coroutines.sync.withLock import java.util.concurrent.atomic.AtomicBoolean fun RawFramePullPush( - frameProcessor: IFrameProcessor, + frameProcessor: IProcessor, onFrame: suspend (RawFrame) -> Unit, processDispatcher: CoroutineDispatcher, isDirect: Boolean = true @@ -47,7 +47,7 @@ fun RawFramePullPush( * @param processDispatcher the dispatcher to process frames on */ class RawFramePullPush( - private val frameProcessor: IFrameProcessor, + private val frameProcessor: IProcessor, val onFrame: suspend (RawFrame) -> Unit, private val bufferPool: ByteBufferPool, private val processDispatcher: CoroutineDispatcher, @@ -99,7 +99,7 @@ class RawFramePullPush( // Process buffer with effects val processedFrame = try { - frameProcessor.processFrame(rawFrame) + frameProcessor.process(rawFrame) } catch (t: Throwable) { Logger.e(TAG, "Failed to pre-process frame: ${t.message}") continue diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt index 83af9a5a7..bde293f80 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt @@ -16,30 +16,47 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.processing.IFrameProcessor +import io.github.thibaultbee.streampack.core.elements.processing.IEffectConsumer +import io.github.thibaultbee.streampack.core.elements.processing.IEffectProcessor import java.io.Closeable /** * The base audio effect. */ -interface IAudioEffect : IFrameProcessor, Closeable +sealed interface IAudioEffect : Closeable /** - * Mute audio effect. + * An audio effect that can be dispatched to another thread. The result is not use by the audio pipeline. + * Example: a VU meter. */ -class MuteEffect : IAudioEffect { +interface IConsumerAudioEffect : IAudioEffect, IEffectConsumer + +/** + * An audio effect that can't be dispatched to another thread. The result is used by the audio pipeline. + * + * The [RawFrame.rawBuffer] can't be modified. + */ +interface IProcessorAudioEffect : IAudioEffect, IEffectProcessor + +/** + * An audio effect that mute the audio. + */ +class MuteEffect : IProcessorAudioEffect { private var mutedByteArray: ByteArray? = null - override fun processFrame(frame: RawFrame): RawFrame { - val remaining = frame.rawBuffer.remaining() - val position = frame.rawBuffer.position() + override fun process(isMuted: Boolean, data: RawFrame): RawFrame { + if (!isMuted) { + return data + } + val remaining = data.rawBuffer.remaining() + val position = data.rawBuffer.position() if (remaining != mutedByteArray?.size) { mutedByteArray = ByteArray(remaining) } - frame.rawBuffer.put(mutedByteArray!!) - frame.rawBuffer.position(position) + data.rawBuffer.put(mutedByteArray!!) + data.rawBuffer.position(position) - return frame + return data } override fun close() { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index 4cd6e7b7a..99c586a22 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -16,22 +16,76 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.processing.IFrameProcessor +import io.github.thibaultbee.streampack.core.elements.processing.IProcessor +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import java.io.Closeable +import java.util.function.IntFunction /** * Audio frame processor. * - * Only supports mute effect for now. + * It is not thread-safe. */ -class AudioFrameProcessor : IFrameProcessor, - IAudioFrameProcessor { +class AudioFrameProcessor( + dispatcher: CoroutineDispatcher, + private val effects: MutableList = mutableListOf() +) : IProcessor, IAudioFrameProcessor, Closeable, MutableList by effects { + private val coroutineScope = CoroutineScope(dispatcher + SupervisorJob()) + + /** + * Whether the audio is muted. + * + * When the audio is muted, the audio effect are not processed. Only consumer effects are processed. + */ override var isMuted = false private val muteEffect = MuteEffect() - override fun processFrame(frame: RawFrame): RawFrame { - if (isMuted) { - return muteEffect.processFrame(frame) + private fun launchConsumerEffect( + effect: IConsumerAudioEffect, + isMuted: Boolean, + data: RawFrame + ) { + coroutineScope.launch { + val consumeFrame = + data.copy( + rawBuffer = data.rawBuffer.duplicate().asReadOnlyBuffer() + ) + effect.consume(isMuted, consumeFrame) } - return frame } -} \ No newline at end of file + + override fun process(data: RawFrame): RawFrame { + val isMuted = isMuted + + var processedFrame = muteEffect.process(isMuted, data) + + effects.forEach { + if (it is IProcessorAudioEffect) { + processedFrame = it.process(isMuted, processedFrame) + } else if (it is IConsumerAudioEffect) { + launchConsumerEffect(it, isMuted, processedFrame) + } + } + + return processedFrame + } + + override fun close() { + effects.forEach { it.close() } + effects.clear() + + muteEffect.close() + + coroutineScope.cancel() + } + + @Deprecated("'fun toArray(generator: IntFunction!>!): Array<(out) T!>!' is deprecated. This declaration is redundant in Kotlin and might be removed soon.") + @Suppress("DEPRECATION") + override fun toArray(generator: IntFunction?>): Array { + return super.toArray(generator) + } +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt index 24c7606e1..19cc6ba9c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt @@ -20,7 +20,7 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio */ interface IAudioFrameProcessor { /** - * Mute audio. + * Whether the processor is muted. */ var isMuted: Boolean } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 9560378e1..85ac22509 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -135,12 +135,12 @@ internal class AudioInput( /** * The audio processor. */ - private val frameProcessorInternal = AudioFrameProcessor() - override val processor: IAudioFrameProcessor = frameProcessorInternal + private val processorInternal = AudioFrameProcessor(dispatcherProvider.default) + override val processor: IAudioFrameProcessor = processorInternal private val port = if (config is PushConfig) { - PushAudioPort(frameProcessorInternal, config, dispatcherProvider) + PushAudioPort(processorInternal, config, dispatcherProvider) } else { - CallbackAudioPort(frameProcessorInternal) // No threading needed, called from encoder thread + CallbackAudioPort(processorInternal) // No threading needed, called from encoder thread } // CONFIG @@ -355,6 +355,15 @@ internal class AudioInput( ) } + try { + processorInternal.close() + } catch (t: Throwable) { + Logger.w( + TAG, + "release: Can't close audio processor: ${t.message}" + ) + } + isStreamingJob.cancel() } coroutineScope.coroutineContext.cancelChildren() @@ -430,7 +439,7 @@ private class CallbackAudioPort(private val audioFrameProcessor: AudioFrameProce val timestampInUs = source.fillAudioFrame(buffer) pool.get(buffer, timestampInUs) } - return audioFrameProcessor.processFrame(frame) + return audioFrameProcessor.process(frame) } } From 07296840c29017428ed315e090a46735a2ff69a7 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 2 Feb 2026 09:45:25 +0100 Subject: [PATCH 14/72] fix(core): processor: duplicate raw buffer dispatching --- .../elements/processing/audio/AudioFrameProcessor.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index 99c586a22..f6ccc4934 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.processing.IProcessor import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -49,11 +50,11 @@ class AudioFrameProcessor( isMuted: Boolean, data: RawFrame ) { + val consumeFrame = + data.copy( + rawBuffer = data.rawBuffer.duplicate().asReadOnlyBuffer() + ) coroutineScope.launch { - val consumeFrame = - data.copy( - rawBuffer = data.rawBuffer.duplicate().asReadOnlyBuffer() - ) effect.consume(isMuted, consumeFrame) } } From 4de808455a0731b43ba2293568422b126c396452 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 2 Feb 2026 10:40:52 +0100 Subject: [PATCH 15/72] refactor(core): rename `clone` to `deepCopy` --- .../endpoints/composites/muxers/mp4/models/TrackChunks.kt | 4 ++-- .../elements/utils/extensions/ByteBufferExtensions.kt | 8 ++++---- .../utils/extensions/ByteBufferExtensionsKtTest.kt | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt index cbccabd76..28dde5d47 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt @@ -63,7 +63,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.SLCon import io.github.thibaultbee.streampack.core.elements.utils.av.video.avc.AVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.hevc.HEVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.vpx.VPCodecConfigurationRecord -import io.github.thibaultbee.streampack.core.elements.utils.extensions.clone +import io.github.thibaultbee.streampack.core.elements.utils.extensions.deepCopy import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAnnexB import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAvcc import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode @@ -176,7 +176,7 @@ class TrackChunks( } val frameCopy = - frame.copy(rawBuffer = frame.rawBuffer.clone()) // Do not keep mediacodec buffer + frame.copy(rawBuffer = frame.rawBuffer.deepCopy()) // Do not keep mediacodec buffer chunks.last().add(frameId, frameCopy) frameId++ } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt index e4beee7ab..88dd17ffc 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt @@ -253,18 +253,18 @@ fun ByteBuffer.toByteArray(): ByteArray { } /** - * Clone [ByteBuffer]. + * Deep copy of [ByteBuffer]. * The position of the original [ByteBuffer] will be 0 after the clone. */ -fun ByteBuffer.clone(): ByteBuffer { +fun ByteBuffer.deepCopy(): ByteBuffer { val originalPosition = this.position() try { - val clone = if (isDirect) { + val copy = if (isDirect) { ByteBuffer.allocateDirect(this.remaining()) } else { ByteBuffer.allocate(this.remaining()) } - return clone.put(this).apply { rewind() } + return copy.put(this).apply { rewind() } } finally { this.position(originalPosition) } diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt index 3ceff1035..489364031 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt @@ -117,7 +117,7 @@ class ByteBufferExtensionsKtTest { ) testBuffer.position(2) - val clonedBuffer = testBuffer.clone() + val clonedBuffer = testBuffer.deepCopy() assertArrayEquals( testBuffer.toByteArray(), clonedBuffer.toByteArray() ) From 6ad8378f410b91b4f8808f98f8f100ff0cd9a786 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 2 Feb 2026 11:14:42 +0100 Subject: [PATCH 16/72] fix(core): processor: consumer audio effect must receive a deep copy of the current audio buffer. Because in callback mode, the rawBuffer belongs to the codec input buffer pool and it could be released before the effect processes the buffer --- .../streampack/core/elements/data/Frame.kt | 20 +++++++++++++++++++ .../elements/processing/RawFramePullPush.kt | 7 ------- .../processing/audio/AudioFrameProcessor.kt | 10 +++++----- .../utils/extensions/ByteBufferExtensions.kt | 20 +++++++++++++++++++ .../elements/utils/pool/ByteBufferPool.kt | 2 +- .../core/pipelines/inputs/AudioInput.kt | 9 +++++++-- 6 files changed, 53 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index c9c08acea..ab615b1ad 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -16,7 +16,9 @@ package io.github.thibaultbee.streampack.core.elements.data import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.utils.extensions.deepCopy import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import java.io.Closeable import java.nio.ByteBuffer @@ -36,6 +38,24 @@ interface RawFrame : Closeable { val timestampInUs: Long } +/** + * Deep copy the [RawFrame.rawBuffer] into a new [RawFrame]. + * + * For better memory allocation, you should close the returned frame after usage. + */ +fun RawFrame.deepCopy( + bufferPool: IBufferPool, + timestampInUs: Long = this.timestampInUs, + onClosed: (RawFrame) -> Unit = {} +): RawFrame { + val copy = this.rawBuffer.deepCopy(bufferPool) + return copy( + rawBuffer = copy, timestampInUs = timestampInUs, onClosed = { + onClosed(it) + bufferPool.put(copy) + } + ) +} /** * Copy a [RawFrame] to a new [RawFrame]. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt index eca215f31..aa5bfedd5 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt @@ -31,13 +31,6 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.atomic.AtomicBoolean -fun RawFramePullPush( - frameProcessor: IProcessor, - onFrame: suspend (RawFrame) -> Unit, - processDispatcher: CoroutineDispatcher, - isDirect: Boolean = true -) = RawFramePullPush(frameProcessor, onFrame, ByteBufferPool(isDirect), processDispatcher) - /** * A component that pull a frame from an input and push it to [onFrame] output. * diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index f6ccc4934..ab249b756 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -16,14 +16,16 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.data.copy +import io.github.thibaultbee.streampack.core.elements.data.deepCopy import io.github.thibaultbee.streampack.core.elements.processing.IProcessor +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.launch import java.io.Closeable +import java.nio.ByteBuffer import java.util.function.IntFunction /** @@ -32,6 +34,7 @@ import java.util.function.IntFunction * It is not thread-safe. */ class AudioFrameProcessor( + private val bufferPool: IBufferPool, dispatcher: CoroutineDispatcher, private val effects: MutableList = mutableListOf() ) : IProcessor, IAudioFrameProcessor, Closeable, MutableList by effects { @@ -50,10 +53,7 @@ class AudioFrameProcessor( isMuted: Boolean, data: RawFrame ) { - val consumeFrame = - data.copy( - rawBuffer = data.rawBuffer.duplicate().asReadOnlyBuffer() - ) + val consumeFrame = data.deepCopy(bufferPool) coroutineScope.launch { effect.consume(isMuted, consumeFrame) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt index 88dd17ffc..09cee1ff1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt @@ -15,6 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.utils.extensions +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.charset.StandardCharsets @@ -256,6 +257,7 @@ fun ByteBuffer.toByteArray(): ByteArray { * Deep copy of [ByteBuffer]. * The position of the original [ByteBuffer] will be 0 after the clone. */ +@Deprecated("Use ByteBufferPool instead") fun ByteBuffer.deepCopy(): ByteBuffer { val originalPosition = this.position() try { @@ -270,6 +272,24 @@ fun ByteBuffer.deepCopy(): ByteBuffer { } } +/** + * Deep copy of [ByteBuffer] from [IBufferPool]. + * + * Don't forget to put the returned [ByteBuffer] to the buffer pool when you are done with it. + * + * @param pool [IBufferPool] to use + * @return [ByteBuffer] deep copy + */ +fun ByteBuffer.deepCopy(pool: IBufferPool): ByteBuffer { + val originalPosition = this.position() + try { + val copy = pool.get(this.remaining()) + return copy.put(this).apply { rewind() } + } finally { + this.position(originalPosition) + } +} + /** * For AVC and HEVC */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt index ce748def0..f3036b9b8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt @@ -87,4 +87,4 @@ class ByteBufferPool(private val isDirect: Boolean) : IBufferPool, C buffers.clear() } } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 85ac22509..999bc16e8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -28,6 +28,7 @@ import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioFrameS import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.utils.ConflatedJob +import io.github.thibaultbee.streampack.core.elements.utils.pool.ByteBufferPool import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Companion.THREAD_NAME_AUDIO_PREPROCESSING @@ -132,13 +133,15 @@ internal class AudioInput( } // PROCESSOR + private val bufferPool = ByteBufferPool(true) + /** * The audio processor. */ - private val processorInternal = AudioFrameProcessor(dispatcherProvider.default) + private val processorInternal = AudioFrameProcessor(bufferPool, dispatcherProvider.default) override val processor: IAudioFrameProcessor = processorInternal private val port = if (config is PushConfig) { - PushAudioPort(processorInternal, config, dispatcherProvider) + PushAudioPort(processorInternal, config, bufferPool, dispatcherProvider) } else { CallbackAudioPort(processorInternal) // No threading needed, called from encoder thread } @@ -390,11 +393,13 @@ private sealed interface IAudioPort : Streamable, Releasable { private class PushAudioPort( audioFrameProcessor: AudioFrameProcessor, config: PushConfig, + bufferPool: ByteBufferPool, dispatcherProvider: IAudioDispatcherProvider ) : IAudioPort { private val audioPullPush = RawFramePullPush( audioFrameProcessor, config.onFrame, + bufferPool, dispatcherProvider.createAudioDispatcher( 1, THREAD_NAME_AUDIO_PREPROCESSING From 057959451132bc7f916a5005af9a471e0d46aa4f Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 2 Feb 2026 12:08:32 +0100 Subject: [PATCH 17/72] fix(core): processor: use `CopyOnWriteArrayList` instead of MutableList to avoid `ConcurrentModificationException` --- .../core/elements/processing/audio/AudioFrameProcessor.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index ab249b756..f10f6af43 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -26,17 +26,16 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.launch import java.io.Closeable import java.nio.ByteBuffer +import java.util.concurrent.CopyOnWriteArrayList import java.util.function.IntFunction /** * Audio frame processor. - * - * It is not thread-safe. */ class AudioFrameProcessor( private val bufferPool: IBufferPool, dispatcher: CoroutineDispatcher, - private val effects: MutableList = mutableListOf() + private val effects: CopyOnWriteArrayList = CopyOnWriteArrayList() ) : IProcessor, IAudioFrameProcessor, Closeable, MutableList by effects { private val coroutineScope = CoroutineScope(dispatcher + SupervisorJob()) From e1ba66449224a483e2af7991cfbcf30b84614110 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Fri, 30 Jan 2026 11:33:55 +0100 Subject: [PATCH 18/72] feat(ui): add a composable preview view --- gradle/libs.versions.toml | 9 ++ settings.libs.gradle.kts | 10 +- ui/{ => compose}/.gitignore | 0 ui/compose/build.gradle.kts | 35 ++++++ ui/{ => compose}/proguard-rules.pro | 0 ui/{ => compose}/src/main/AndroidManifest.xml | 0 .../streampack/compose/PreviewView.kt | 105 ++++++++++++++++++ .../streampack/compose/utils/BitmapUtils.kt | 41 +++++++ ui/ui/.gitignore | 1 + ui/{ => ui}/build.gradle.kts | 0 ui/ui/proguard-rules.pro | 21 ++++ ui/ui/src/main/AndroidManifest.xml | 1 + .../streampack/ui/views/AutoFitSurfaceView.kt | 0 .../streampack/ui/views/PreviewView.kt | 0 .../streampack/ui/views/StreamerExtensions.kt | 0 .../ui/views/VideoSourceExtensions.kt | 0 .../streampack/ui/views/ViewExtensions.kt | 0 ui/{ => ui}/src/main/res/values/attrs.xml | 0 18 files changed, 221 insertions(+), 2 deletions(-) rename ui/{ => compose}/.gitignore (100%) create mode 100644 ui/compose/build.gradle.kts rename ui/{ => compose}/proguard-rules.pro (100%) rename ui/{ => compose}/src/main/AndroidManifest.xml (100%) create mode 100644 ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt create mode 100644 ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt create mode 100644 ui/ui/.gitignore rename ui/{ => ui}/build.gradle.kts (100%) create mode 100644 ui/ui/proguard-rules.pro create mode 100644 ui/ui/src/main/AndroidManifest.xml rename ui/{ => ui}/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt (100%) rename ui/{ => ui}/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt (100%) rename ui/{ => ui}/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt (100%) rename ui/{ => ui}/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt (100%) rename ui/{ => ui}/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt (100%) rename ui/{ => ui}/src/main/res/values/attrs.xml (100%) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4602a3dcf..2cb2b2401 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,6 +7,7 @@ videoApiClient = "1.6.7" androidxActivity = "1.10.1" androidxAppcompat = "1.7.1" androidxCamera = "1.4.0-alpha13" +androidxComposeBom = "2026.01.00" androidxConstraintlayout = "2.2.1" androidxCore = "1.17.0" androidxDatabinding = "8.13.0" @@ -29,6 +30,7 @@ robolectric = "4.16" komuxer = "0.3.4" srtdroid = "1.9.5" junitKtx = "1.3.0" +compose = "1.10.1" [libraries] android-documentation-plugin = { module = "org.jetbrains.dokka:android-documentation-plugin", version.ref = "dokka" } @@ -42,6 +44,7 @@ android-material = { module = "com.google.android.material:material", version.re androidx-activity = { module = "androidx.activity:activity", version.ref = "androidxActivity" } androidx-appcompat = { module = "androidx.appcompat:appcompat", version.ref = "androidxAppcompat" } androidx-camera-viewfinder-view = { module = "androidx.camera.viewfinder:viewfinder-view", version.ref = "androidxCamera" } +androidx-compose-bom = { module = "androidx.compose:compose-bom", version.ref = "androidxComposeBom" } androidx-concurrent-futures = { module = "androidx.concurrent:concurrent-futures", version.ref = "concurrentFutures" } androidx-constraintlayout = { module = "androidx.constraintlayout:constraintlayout", version.ref = "androidxConstraintlayout" } androidx-core-ktx = { module = "androidx.core:core-ktx", version.ref = "androidxCore" } @@ -70,10 +73,16 @@ kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.re mockk = { module = "io.mockk:mockk", version.ref = "mockk" } robolectric = { module = "org.robolectric:robolectric", version.ref = "robolectric" } srtdroid-ktx = { module = "io.github.thibaultbee.srtdroid:srtdroid-ktx", version.ref = "srtdroid" } +androidx-compose-runtime = { group = "androidx.compose.runtime", name = "runtime", version.ref = "compose" } +androidx-compose-ui = { group = "androidx.compose.ui", name = "ui", version.ref = "compose" } +androidx-compose-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview", version.ref = "compose" } +androidx-compose-ui-tooling = { group = "androidx.compose.ui", name = "ui-tooling", version.ref = "compose" } +androidx-compose-foundation = { group = "androidx.compose.foundation", name = "foundation", version.ref = "compose" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } android-library = { id = "com.android.library", version.ref = "agp" } +compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } kotlin-kapt = { id = "org.jetbrains.kotlin.kapt", version.ref = "kotlin" } dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } diff --git a/settings.libs.gradle.kts b/settings.libs.gradle.kts index c42addcde..7a8a4328c 100644 --- a/settings.libs.gradle.kts +++ b/settings.libs.gradle.kts @@ -1,11 +1,17 @@ // StreamPack libraries include(":core") project(":core").name = "streampack-core" -include(":ui") -project(":ui").name = "streampack-ui" include(":services") project(":services").name = "streampack-services" +// UI +include(":ui") +project(":ui").projectDir = File(rootDir, "ui/ui") +project(":ui").name = "streampack-ui" +include(":compose") +project(":compose").projectDir = File(rootDir, "ui/compose") +project(":compose").name = "streampack-compose" + // Extensions include(":extension-flv") project(":extension-flv").projectDir = File(rootDir, "extensions/flv") diff --git a/ui/.gitignore b/ui/compose/.gitignore similarity index 100% rename from ui/.gitignore rename to ui/compose/.gitignore diff --git a/ui/compose/build.gradle.kts b/ui/compose/build.gradle.kts new file mode 100644 index 000000000..de8c2b318 --- /dev/null +++ b/ui/compose/build.gradle.kts @@ -0,0 +1,35 @@ +plugins { + id(libs.plugins.android.library.get().pluginId) + id(libs.plugins.kotlin.android.get().pluginId) + id("android-library-convention") + alias(libs.plugins.compose.compiler) +} + +description = "Jetpack compose components for StreamPack." + +android { + namespace = "io.github.thibaultbee.streampack.compose" + + defaultConfig { + minSdk = 23 + } + buildFeatures { + compose = true + } +} + +dependencies { + implementation(project(":streampack-core")) + implementation(project(":streampack-ui")) + + val composeBom = platform(libs.androidx.compose.bom) + implementation(composeBom) + implementation(libs.androidx.compose.runtime) + implementation(libs.androidx.compose.ui) + implementation(libs.androidx.compose.ui.tooling.preview) + implementation(libs.androidx.compose.foundation) + + androidTestImplementation(composeBom) + + debugImplementation(libs.androidx.compose.ui.tooling) +} diff --git a/ui/proguard-rules.pro b/ui/compose/proguard-rules.pro similarity index 100% rename from ui/proguard-rules.pro rename to ui/compose/proguard-rules.pro diff --git a/ui/src/main/AndroidManifest.xml b/ui/compose/src/main/AndroidManifest.xml similarity index 100% rename from ui/src/main/AndroidManifest.xml rename to ui/compose/src/main/AndroidManifest.xml diff --git a/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt new file mode 100644 index 000000000..379649bb7 --- /dev/null +++ b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.compose + +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.viewinterop.AndroidView +import io.github.thibaultbee.streampack.compose.utils.BitmapUtils +import io.github.thibaultbee.streampack.core.elements.sources.video.IPreviewableSource +import io.github.thibaultbee.streampack.core.elements.sources.video.bitmap.BitmapSourceFactory +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings.FocusMetering.Companion.DEFAULT_AUTO_CANCEL_DURATION_MS +import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource +import io.github.thibaultbee.streampack.core.logger.Logger +import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.ui.views.PreviewView +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.withLock + +private const val TAG = "ComposePreviewView" + +/** + * Displays the preview of a [IWithVideoSource]. + * + * A [IWithVideoSource] must have a video sourc. + * + * @param videoSource the [IWithVideoSource] to preview + * @param modifier the [Modifier] to apply to the [PreviewView] + * @param enableZoomOnPinch enable zoom on pinch gesture + * @param enableTapToFocus enable tap to focus + * @param onTapToFocusTimeoutMs the duration in milliseconds after which the focus area set by tap-to-focus is cleared + */ +@Composable +fun PreviewScreen( + videoSource: IWithVideoSource, + modifier: Modifier = Modifier, + enableZoomOnPinch: Boolean = true, + enableTapToFocus: Boolean = true, + onTapToFocusTimeoutMs: Long = DEFAULT_AUTO_CANCEL_DURATION_MS +) { + val scope = rememberCoroutineScope() + + AndroidView( + factory = { context -> + PreviewView(context).apply { + this.enableZoomOnPinch = enableZoomOnPinch + this.enableTapToFocus = enableTapToFocus + this.onTapToFocusTimeoutMs = onTapToFocusTimeoutMs + + scope.launch { + try { + setVideoSourceProvider(videoSource) + } catch (e: Exception) { + Logger.e(TAG, "Failed to start preview", e) + } + } + } + }, + modifier = modifier, + onRelease = { + scope.launch { + val source = videoSource.videoInput?.sourceFlow?.value as? IPreviewableSource + source?.previewMutex?.withLock { + source.stopPreview() + source.resetPreview() + } + } + }) +} + +@Preview +@Composable +fun PreviewScreenPreview() { + val context = LocalContext.current + val streamer = SingleStreamer(context) + LaunchedEffect(Unit) { + streamer.setVideoSource( + BitmapSourceFactory( + BitmapUtils.createImage( + 1280, + 720 + ) + ) + ) + } + + PreviewScreen(streamer, modifier = Modifier.fillMaxSize()) +} \ No newline at end of file diff --git a/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt new file mode 100644 index 000000000..c1cfec02b --- /dev/null +++ b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.compose.utils + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint + +object BitmapUtils { + /** + * Creates a bitmap with the given width, height and color. + * + * @param width The width of the bitmap. + * @param height The height of the bitmap. + * @param color The color of the bitmap. + * @return The created bitmap. + */ + fun createImage(width: Int, height: Int, color: Int = Color.RED): Bitmap { + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bitmap) + val paint = Paint().apply { + this.color = color + } + canvas.drawRect(0f, 0f, width.toFloat(), height.toFloat(), paint) + return bitmap + } +} diff --git a/ui/ui/.gitignore b/ui/ui/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/ui/ui/.gitignore @@ -0,0 +1 @@ +/build diff --git a/ui/build.gradle.kts b/ui/ui/build.gradle.kts similarity index 100% rename from ui/build.gradle.kts rename to ui/ui/build.gradle.kts diff --git a/ui/ui/proguard-rules.pro b/ui/ui/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/ui/ui/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/ui/ui/src/main/AndroidManifest.xml b/ui/ui/src/main/AndroidManifest.xml new file mode 100644 index 000000000..cc947c567 --- /dev/null +++ b/ui/ui/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt diff --git a/ui/src/main/res/values/attrs.xml b/ui/ui/src/main/res/values/attrs.xml similarity index 100% rename from ui/src/main/res/values/attrs.xml rename to ui/ui/src/main/res/values/attrs.xml From 6f3d60e8870ad83d9cb9820a4a94f348d57b6eb1 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 12:12:34 +0100 Subject: [PATCH 19/72] feat(core): camera: add a way to access the TotalCaptureResult --- .../sources/video/camera/CameraSettings.kt | 89 +++++++++++++++---- .../camera/controllers/CameraController.kt | 6 +- .../controllers/CameraSessionController.kt | 11 --- .../camera/utils/CameraSessionCallback.kt | 30 ++++--- .../camera/utils/CaptureResultListener.kt | 28 ++++++ 5 files changed, 123 insertions(+), 41 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 9b0a41b1a..c662fdd21 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -20,6 +20,10 @@ import android.graphics.PointF import android.graphics.Rect import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraMetadata +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_ACTIVE_SCAN +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_FOCUSED_LOCKED +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_INACTIVE +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED import android.hardware.camera2.CaptureRequest import android.hardware.camera2.CaptureResult import android.hardware.camera2.TotalCaptureResult @@ -33,7 +37,6 @@ import androidx.annotation.IntRange import androidx.annotation.RequiresApi import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.is90or270 import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraController -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoExposureModes import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoFocusModes import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoWhiteBalanceModes @@ -50,6 +53,7 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.scalerMaxZoom import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.sensitivityRange import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.zoomRatioRange +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.utils.extensions.clamp import io.github.thibaultbee.streampack.core.elements.utils.extensions.isApplicationPortrait import io.github.thibaultbee.streampack.core.elements.utils.extensions.isNormalized @@ -65,8 +69,10 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.flow.conflate import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import java.util.concurrent.CancellationException import java.util.concurrent.atomic.AtomicLong @@ -96,7 +102,9 @@ class CameraSettings internal constructor( } cameraController.addCaptureCallbackListener(captureCallback) awaitClose { - cameraController.removeCaptureCallbackListener(captureCallback) + runBlocking { + cameraController.removeCaptureCallbackListener(captureCallback) + } } }.conflate().distinctUntilChanged() @@ -159,8 +167,6 @@ class CameraSettings internal constructor( val focusMetering = FocusMetering(coroutineScope, characteristics, this, zoom, focus, exposure, whiteBalance) - private val tagBundleFactory = TagBundle.TagBundleFactory() - /** * Directly gets a [CaptureRequest] from the camera. * @@ -184,18 +190,35 @@ class CameraSettings internal constructor( * Applies settings to the camera repeatedly in a synchronized way. * * This method returns when the capture callback is received with the passed request. + * + * @return the total capture result */ - suspend fun applyRepeatingSessionSync() { - val deferred = CompletableDeferred() + suspend fun applyRepeatingSessionSync(): TotalCaptureResult { + val deferred = CompletableDeferred() - val tag = tagBundleFactory.create() + val captureResult = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + deferred.complete(result) + return false + } + } + applyRepeatingSession(captureResult) + return deferred.await() + } + + /** + * Applies settings to the camera repeatedly. + * + * @param onCaptureResult the capture result callback. Return `true` to stop the callback. + */ + suspend fun applyRepeatingSession(onCaptureResult: CaptureResultListener) { + val tag = TagBundle.Factory.default.create() val captureCallback = object : CaptureResultListener { override fun onCaptureResult(result: TotalCaptureResult): Boolean { val resultTag = result.request.tag as? TagBundle val keyId = resultTag?.keyId ?: return false if (keyId >= tag.keyId) { - deferred.complete(Unit) - return true + return onCaptureResult.onCaptureResult(result) } return false } @@ -203,7 +226,6 @@ class CameraSettings internal constructor( cameraController.addCaptureCallbackListener(captureCallback) cameraController.setRepeatingSession(tag) - deferred.await() } /** @@ -211,7 +233,7 @@ class CameraSettings internal constructor( */ suspend fun applyRepeatingSession() = cameraController.setRepeatingSession() - private class TagBundle private constructor(val keyId: Long) { + private class TagBundle(val keyId: Long) { private val tagMap = mutableMapOf().apply { put(TAG_KEY_ID, keyId) } @@ -232,7 +254,7 @@ class CameraSettings internal constructor( * * The purpose is to make sure the tag always contains an increasing id. */ - class TagBundleFactory { + class Factory private constructor() { /** * Next session id. */ @@ -241,6 +263,10 @@ class CameraSettings internal constructor( fun create(): TagBundle { return TagBundle(nextSessionUpdateId.getAndIncrement()) } + + companion object { + val default = Factory() + } } } @@ -1035,7 +1061,35 @@ class CameraSettings internal constructor( ) cameraSettings.set(CaptureRequest.CONTROL_AE_MODE, aeMode) } - cameraSettings.applyRepeatingSession() + + val deferred = CompletableDeferred() + val captureResult = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + return when (val afState = result[CaptureResult.CONTROL_AF_STATE]) { + CONTROL_AF_STATE_FOCUSED_LOCKED -> { + deferred.complete(Unit) + true + } + + CONTROL_AF_STATE_NOT_FOCUSED_LOCKED -> { + deferred.completeExceptionally(Exception("AF not focused")) + true + } + + CONTROL_AF_STATE_INACTIVE -> { + deferred.completeExceptionally(CancellationException("AF has been cancelled")) + true + } + + CONTROL_AF_STATE_ACTIVE_SCAN -> false + else -> { + deferred.completeExceptionally(IllegalStateException("AF is not in an expected state $afState")) + } + } + } + } + cameraSettings.applyRepeatingSession(captureResult) + deferred.await() } private suspend fun executeMetering( @@ -1052,7 +1106,7 @@ class CameraSettings internal constructor( */ cameraSettings.set( CaptureRequest.CONTROL_AF_TRIGGER, - CameraMetadata.CONTROL_AF_TRIGGER_IDLE + CameraMetadata.CONTROL_AF_TRIGGER_CANCEL ) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { cameraSettings.set( @@ -1063,9 +1117,6 @@ class CameraSettings internal constructor( cameraSettings.applyRepeatingSessionSync() addFocusMetering(afRectangles, aeRectangles, awbRectangles) - if (afRectangles.isNotEmpty()) { - triggerAf(true) - } // Auto cancel AF trigger after timeoutDurationMs if (timeoutDurationMs > 0) { @@ -1078,6 +1129,10 @@ class CameraSettings internal constructor( } } } + + if (afRectangles.isNotEmpty()) { + triggerAf(true) + } } private suspend fun startFocusAndMetering( diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index ec7000895..a2fca02b0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -21,13 +21,13 @@ import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest import android.util.Range import androidx.annotation.RequiresPermission -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.CameraCaptureSessionCompatBuilder import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraDispatcherProvider import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.utils.av.video.DynamicRangeProfile import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineScope @@ -387,7 +387,7 @@ internal class CameraController( * * The listener is removed when it returns true or [removeCaptureCallbackListener] is called. */ - fun addCaptureCallbackListener(listener: CaptureResultListener) { + suspend fun addCaptureCallbackListener(listener: CaptureResultListener) { sessionCallback.addListener(listener) } @@ -396,7 +396,7 @@ internal class CameraController( * * @param listener The listener to remove */ - fun removeCaptureCallbackListener(listener: CaptureResultListener) { + suspend fun removeCaptureCallbackListener(listener: CaptureResultListener) { sessionCallback.removeListener(listener) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index ab193b4ad..b42274eca 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -34,7 +34,6 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.first -import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext @@ -453,14 +452,4 @@ internal class CameraSessionController private constructor( } } } - - interface CaptureResultListener { - /** - * Called when a capture result is received. - * - * @param result The capture result. - * @return true if the listener is finished and should be removed, false otherwise. - */ - fun onCaptureResult(result: TotalCaptureResult): Boolean - } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt index ae8a8ad2c..67b8b086d 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt @@ -19,9 +19,10 @@ import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCaptureSession.CaptureCallback import android.hardware.camera2.CaptureRequest import android.hardware.camera2.TotalCaptureResult -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock /** * A capture callback that wraps multiple [CaptureResultListener]. @@ -32,18 +33,23 @@ internal class CameraSessionCallback(private val coroutineScope: CoroutineScope) CaptureCallback() { /* synthetic accessor */ private val resultListeners = mutableSetOf() + private val mutex = Mutex() /** * Adds a capture result listener. * * The listener is removed when [removeListener] is explicitly called or when [CaptureResultListener] returns true. */ - fun addListener(listener: CaptureResultListener) { - resultListeners.add(listener) + suspend fun addListener(listener: CaptureResultListener) { + mutex.withLock { + resultListeners.add(listener) + } } - fun removeListener(listener: CaptureResultListener) { - resultListeners.remove(listener) + suspend fun removeListener(listener: CaptureResultListener) { + mutex.withLock { + resultListeners.remove(listener) + } } override fun onCaptureCompleted( @@ -53,14 +59,18 @@ internal class CameraSessionCallback(private val coroutineScope: CoroutineScope) ) { coroutineScope.launch { val removeSet = mutableSetOf() - for (listener in resultListeners) { - val isFinished: Boolean = listener.onCaptureResult(result) - if (isFinished) { - removeSet.add(listener) + mutex.withLock { + for (listener in resultListeners) { + val isFinished: Boolean = listener.onCaptureResult(result) + if (isFinished) { + removeSet.add(listener) + } } } if (!removeSet.isEmpty()) { - resultListeners.removeAll(removeSet) + mutex.withLock { + resultListeners.removeAll(removeSet) + } } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt new file mode 100644 index 000000000..c97eeda2f --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils + +import android.hardware.camera2.TotalCaptureResult + +interface CaptureResultListener { + /** + * Called when a capture result is received. + * + * @param result The capture result. + * @return true if the listener is finished and should be removed, false otherwise. + */ + fun onCaptureResult(result: TotalCaptureResult): Boolean +} \ No newline at end of file From 7f4cd195b2060c618e0bf1bdca7f499245c2e77e Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 14:27:53 +0100 Subject: [PATCH 20/72] fix(core): camera: internalizes constructor of camera settings --- .../sources/video/camera/CameraSettings.kt | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index c662fdd21..2310d5c54 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -270,7 +270,7 @@ class CameraSettings internal constructor( } } - class Flash( + class Flash internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -322,7 +322,7 @@ class CameraSettings internal constructor( } } - class WhiteBalance( + class WhiteBalance internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -400,7 +400,7 @@ class CameraSettings internal constructor( } } - class Iso( + class Iso internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -448,7 +448,7 @@ class CameraSettings internal constructor( } } - class ColorCorrection( + class ColorCorrection internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -503,7 +503,7 @@ class CameraSettings internal constructor( } } - class Exposure( + class Exposure internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -679,7 +679,7 @@ class CameraSettings internal constructor( } } - class CropScalerRegionZoom( + class CropScalerRegionZoom internal constructor( characteristics: CameraCharacteristics, cameraSettings: CameraSettings ) : Zoom(characteristics, cameraSettings) { @@ -754,7 +754,10 @@ class CameraSettings internal constructor( } @RequiresApi(Build.VERSION_CODES.R) - class RZoom(characteristics: CameraCharacteristics, cameraSettings: CameraSettings) : + class RZoom internal constructor( + characteristics: CameraCharacteristics, + cameraSettings: CameraSettings + ) : Zoom(characteristics, cameraSettings) { override val availableRatioRange: Range get() = characteristics.zoomRatioRange @@ -781,7 +784,7 @@ class CameraSettings internal constructor( const val DEFAULT_ZOOM_RATIO = 1f val DEFAULT_ZOOM_RATIO_RANGE = Range(DEFAULT_ZOOM_RATIO, DEFAULT_ZOOM_RATIO) - fun build( + internal fun build( characteristics: CameraCharacteristics, cameraSettings: CameraSettings ): Zoom { @@ -795,7 +798,7 @@ class CameraSettings internal constructor( } - class Focus( + class Focus internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -905,7 +908,7 @@ class CameraSettings internal constructor( } } - class Stabilization( + class Stabilization internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -989,7 +992,7 @@ class CameraSettings internal constructor( } } - class FocusMetering( + class FocusMetering internal constructor( private val coroutineScope: CoroutineScope, private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings, From 0ece354c8e53611b38cf2715cbd89d64826a77c7 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 14:37:00 +0100 Subject: [PATCH 21/72] feat(core): camera: add an API to set torch strength level --- .../sources/video/camera/CameraSettings.kt | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 2310d5c54..14e7229cc 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -320,6 +320,31 @@ class CameraSettings internal constructor( cameraSettings.set(CaptureRequest.FLASH_MODE, mode) cameraSettings.applyRepeatingSession() } + + /** + * Gets the range of supported flash strength. + * Range is from [1-x]. + * + * Use the range to call [setStrengthLevel] + */ + val strengthLevelRange: Range + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + get() = Range( + 1, + characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] ?: 1 + ) + + /** + * Sets the flash strength. + * + * @param level flash strength. Range is from [1-x]. + * @see [strengthLevelRange] + */ + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + suspend fun setStrengthLevel(level: Int) { + cameraSettings.set(CaptureRequest.FLASH_STRENGTH_LEVEL, level) + cameraSettings.applyRepeatingSession() + } } class WhiteBalance internal constructor( From ecde6595937deb56c89b6208f63f71e2fed83d45 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 14:54:07 +0100 Subject: [PATCH 22/72] refactor(core): camera: use lazy initializer for available properties --- .../sources/video/camera/CameraSettings.kt | 87 ++++++++++--------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 14e7229cc..c0026d17e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -279,8 +279,7 @@ class CameraSettings internal constructor( * * @return `true` if camera has a flash device, [Boolean.Companion.toString] otherwise. */ - val isAvailable: Boolean - get() = characteristics.isFlashAvailable + val isAvailable: Boolean by lazy { characteristics.isFlashAvailable } /** * Enables or disables flash. @@ -327,12 +326,22 @@ class CameraSettings internal constructor( * * Use the range to call [setStrengthLevel] */ - val strengthLevelRange: Range - @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) - get() = Range( + @delegate:RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + val strengthLevelRange: Range by lazy { + Range( 1, characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] ?: 1 ) + } + + val strengthLevel: Int + /** + * Gets the flash strength. + * + * @return the flash strength + */ + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + get() = cameraSettings.get(CaptureRequest.FLASH_STRENGTH_LEVEL) ?: 1 /** * Sets the flash strength. @@ -356,8 +365,7 @@ class CameraSettings internal constructor( * * @return list of supported white balance modes. */ - val availableAutoModes: List - get() = characteristics.autoWhiteBalanceModes + val availableAutoModes: List by lazy { characteristics.autoWhiteBalanceModes } /** * Gets the auto white balance mode. @@ -388,8 +396,7 @@ class CameraSettings internal constructor( /** * Get maximum number of available white balance metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfWhiteBalanceMeteringRegions + val maxNumOfMeteringRegions: Int by lazy { characteristics.maxNumberOfWhiteBalanceMeteringRegions } /** * Gets the white balance metering regions. @@ -436,8 +443,9 @@ class CameraSettings internal constructor( * * @see [sensorSensitivity] */ - val availableSensorSensitivityRange: Range - get() = characteristics.sensitivityRange ?: DEFAULT_SENSITIVITY_RANGE + val availableSensorSensitivityRange: Range by lazy { + characteristics.sensitivityRange ?: DEFAULT_SENSITIVITY_RANGE + } /** * Gets lens focus distance. @@ -482,11 +490,10 @@ class CameraSettings internal constructor( * * @return `true` if camera has a flash device, `false` otherwise. */ - val isAvailable: Boolean - get() { - return characteristics[CameraCharacteristics.COLOR_CORRECTION_AVAILABLE_ABERRATION_MODES] - ?.contains(CaptureRequest.COLOR_CORRECTION_MODE_TRANSFORM_MATRIX) == true - } + val isAvailable: Boolean by lazy { + characteristics[CameraCharacteristics.COLOR_CORRECTION_AVAILABLE_ABERRATION_MODES] + ?.contains(CaptureRequest.COLOR_CORRECTION_MODE_TRANSFORM_MATRIX) == true + } /** * Gets color correction gain. @@ -539,9 +546,7 @@ class CameraSettings internal constructor( * * @see [autoMode] */ - val availableAutoModes: List - get() = characteristics.autoExposureModes - + val availableAutoModes: List by lazy { characteristics.autoExposureModes } /** * Gets auto exposure mode. @@ -575,9 +580,10 @@ class CameraSettings internal constructor( * @see [availableCompensationStep] * @see [compensation] */ - val availableCompensationRange: Range - get() = characteristics.exposureRange + val availableCompensationRange: Range by lazy { + characteristics.exposureRange ?: DEFAULT_COMPENSATION_RANGE + } /** * Gets current camera exposure compensation step. @@ -591,9 +597,9 @@ class CameraSettings internal constructor( * @see [availableCompensationRange] * @see [compensation] */ - val availableCompensationStep: Rational - get() = characteristics.exposureStep - ?: DEFAULT_COMPENSATION_STEP_RATIONAL + val availableCompensationStep: Rational by lazy { + characteristics.exposureStep ?: DEFAULT_COMPENSATION_STEP_RATIONAL + } /** * Gets exposure compensation. @@ -626,8 +632,7 @@ class CameraSettings internal constructor( /** * Get maximum number of available exposure metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfExposureMeteringRegions + val maxNumOfMeteringRegions by lazy { characteristics.maxNumberOfExposureMeteringRegions } /** * Gets the exposure metering regions. @@ -714,10 +719,11 @@ class CameraSettings internal constructor( private var persistentZoomRatio = 1f private var currentCropRect: Rect? = null - override val availableRatioRange: Range - get() = Range( + override val availableRatioRange: Range by lazy { + Range( DEFAULT_ZOOM_RATIO, characteristics.scalerMaxZoom ) + } override suspend fun getZoomRatio(): Float = mutex.withLock { persistentZoomRatio @@ -784,9 +790,9 @@ class CameraSettings internal constructor( cameraSettings: CameraSettings ) : Zoom(characteristics, cameraSettings) { - override val availableRatioRange: Range - get() = characteristics.zoomRatioRange - ?: DEFAULT_ZOOM_RATIO_RANGE + override val availableRatioRange: Range by lazy { + characteristics.zoomRatioRange ?: DEFAULT_ZOOM_RATIO_RANGE + } override suspend fun getZoomRatio(): Float { return cameraSettings.get(CaptureRequest.CONTROL_ZOOM_RATIO) @@ -834,8 +840,7 @@ class CameraSettings internal constructor( * * @see [autoMode] */ - val availableAutoModes: List - get() = characteristics.autoFocusModes + val availableAutoModes: List by lazy { characteristics.autoFocusModes } /** * Gets the auto focus mode. @@ -868,8 +873,9 @@ class CameraSettings internal constructor( * * @see [lensDistance] */ - val availableLensDistanceRange: Range - get() = characteristics.lensDistanceRange + val availableLensDistanceRange: Range by lazy { + characteristics.lensDistanceRange + } /** * Gets the lens focus distance. @@ -902,9 +908,9 @@ class CameraSettings internal constructor( /** * Get maximum number of available focus metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfFocusMeteringRegions - ?: DEFAULT_MAX_NUM_OF_METERING_REGION + val maxNumOfMeteringRegions: Int by lazy { + characteristics.maxNumberOfFocusMeteringRegions ?: DEFAULT_MAX_NUM_OF_METERING_REGION + } /** * Gets the focus metering regions. @@ -977,8 +983,9 @@ class CameraSettings internal constructor( * * @see [isEnableOptical] */ - val isOpticalAvailable: Boolean - get() = characteristics.isOpticalStabilizationAvailable + val isOpticalAvailable: Boolean by lazy { + characteristics.isOpticalStabilizationAvailable + } /** From a387abe404cfe25069bcf736924dd513a5cb6769 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 15:41:29 +0100 Subject: [PATCH 23/72] refactor(*): add listener for camera zoom --- .../sources/video/camera/CameraSettings.kt | 42 ++++++++++++++- .../streampack/app/ui/main/PreviewFragment.kt | 11 ++-- .../app/ui/main/PreviewViewModel.kt | 2 +- .../streampack/ui/views/PreviewView.kt | 53 ++++++++++++++----- 4 files changed, 88 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index c0026d17e..d81e54fb4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -73,6 +73,7 @@ import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.CancellationException +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.atomic.AtomicLong @@ -684,6 +685,8 @@ class CameraSettings internal constructor( protected val characteristics: CameraCharacteristics, protected val cameraSettings: CameraSettings ) { + private val listeners: CopyOnWriteArrayList = CopyOnWriteArrayList() + abstract val availableRatioRange: Range internal abstract suspend fun getCropSensorRegion(): Rect @@ -709,6 +712,20 @@ class CameraSettings internal constructor( } } + fun addListener(listener: OnZoomChangedListener) { + listeners.add(listener) + } + + fun removeListener(listener: OnZoomChangedListener) { + listeners.remove(listener) + } + + protected fun notifyZoomListeners(zoomRatio: Float) { + listeners.forEach { + it.onZoomChanged(zoomRatio) + } + } + class CropScalerRegionZoom internal constructor( characteristics: CameraCharacteristics, cameraSettings: CameraSettings @@ -732,6 +749,11 @@ class CameraSettings internal constructor( override suspend fun setZoomRatio(zoomRatio: Float) { mutex.withLock { val clampedValue = zoomRatio.clamp(availableRatioRange) + if (clampedValue == persistentZoomRatio) { + return@withLock + } + persistentZoomRatio = clampedValue + currentCropRect = getCropRegion( characteristics, clampedValue @@ -739,8 +761,8 @@ class CameraSettings internal constructor( cameraSettings.set( CaptureRequest.SCALER_CROP_REGION, currentCropRect ) - cameraSettings.applyRepeatingSession() - persistentZoomRatio = clampedValue + cameraSettings.applyRepeatingSessionSync() + notifyZoomListeners(clampedValue) } } @@ -800,10 +822,14 @@ class CameraSettings internal constructor( } override suspend fun setZoomRatio(zoomRatio: Float) { + if (zoomRatio == getZoomRatio()) { + return + } cameraSettings.set( CaptureRequest.CONTROL_ZOOM_RATIO, zoomRatio.clamp(availableRatioRange) ) cameraSettings.applyRepeatingSession() + notifyZoomListeners(zoomRatio) } override suspend fun getCropSensorRegion(): Rect { @@ -826,6 +852,18 @@ class CameraSettings internal constructor( } } } + + /** + * Listener for zoom change. + */ + interface OnZoomChangedListener { + /** + * Called when the zoom ratio changes. + * + * @param zoomRatio the zoom ratio + */ + fun onZoomChanged(zoomRatio: Float) + } } diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt index acd8f2a14..a7d2d9aca 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt @@ -34,6 +34,7 @@ import io.github.thibaultbee.streampack.app.R import io.github.thibaultbee.streampack.app.databinding.MainFragmentBinding import io.github.thibaultbee.streampack.app.utils.DialogUtils import io.github.thibaultbee.streampack.app.utils.PermissionManager +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource import io.github.thibaultbee.streampack.core.streamers.lifecycle.StreamerViewModelLifeCycleObserver @@ -182,15 +183,17 @@ class PreviewFragment : Fragment(R.layout.main_fragment) { private fun inflateStreamerPreview(streamer: IWithVideoSource) { val preview = binding.preview // Set camera settings button when camera is started - preview.listener = object : PreviewView.Listener { + preview.listener = object : PreviewView.PreviewListener { override fun onPreviewStarted() { Log.i(TAG, "Preview started") } + } - override fun onZoomRationOnPinchChanged(zoomRatio: Float) { - previewViewModel.onZoomRationOnPinchChanged() + preview.setZoomListener(object : CameraSettings.Zoom.OnZoomChangedListener { + override fun onZoomChanged(zoomRatio: Float) { + previewViewModel.onZoomChanged() } - } + }) // Wait till streamer exists to set it to the SurfaceView. lifecycleScope.launch { diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index bf57eed0c..cf15308d4 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -247,7 +247,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } - fun onZoomRationOnPinchChanged() { + fun onZoomChanged() { notifyPropertyChanged(BR.zoomRatio) } diff --git a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt index b9f2f4be3..c2837621d 100644 --- a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt @@ -35,6 +35,7 @@ import androidx.camera.viewfinder.core.ScaleType import androidx.camera.viewfinder.core.ViewfinderSurfaceRequest import androidx.camera.viewfinder.core.populateFromCharacteristics import io.github.thibaultbee.streampack.core.elements.sources.video.IPreviewableSource +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings.FocusMetering.Companion.DEFAULT_AUTO_CANCEL_DURATION_MS import io.github.thibaultbee.streampack.core.elements.sources.video.camera.ICameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.getCameraCharacteristics @@ -108,9 +109,11 @@ class PreviewView @JvmOverloads constructor( } /** - * The [Listener] to listen to specific view events. + * The [PreviewListener] to listen to specific view events. */ - var listener: Listener? = null + var listener: PreviewListener? = null + + private var zoomListener: CameraSettings.Zoom.OnZoomChangedListener? = null private var touchUpEvent: MotionEvent? = null @@ -176,12 +179,46 @@ class PreviewView @JvmOverloads constructor( } if (newVideoSource is IPreviewableSource) { attachToStreamerIfReady(true) + zoomListener?.let { + registerZoomListener(it) + } } } } } } + /** + * Sets the [CameraSettings.Zoom.OnZoomChangedListener] to listen to zoom changes. + * + * @param listener the [CameraSettings.Zoom.OnZoomChangedListener] to listen to zoom changes. + */ + fun setZoomListener(listener: CameraSettings.Zoom.OnZoomChangedListener?) { + if (listener == null) { + unregisterZoomListener() + } else { + registerZoomListener(listener) + } + zoomListener = listener + } + + private fun registerZoomListener(listener: CameraSettings.Zoom.OnZoomChangedListener) { + val source = streamer?.videoInput?.sourceFlow?.value + if (source is ICameraSource) { + source.settings.zoom.addListener(listener) + } + } + + private fun unregisterZoomListener() { + zoomListener?.let { + val source = streamer?.videoInput?.sourceFlow?.value + if (source is ICameraSource) { + source.settings.zoom.removeListener(it) + } + } + } + + /** * Sets the [IWithVideoSource] to preview. * @@ -511,10 +548,6 @@ class PreviewView @JvmOverloads constructor( mutex.withLock { val zoom = source.settings.zoom zoom.onPinch(scaleFactor) - val newZoomRatio = zoom.getZoomRatio() - post { - listener?.onZoomRationOnPinchChanged(newZoomRatio) - } } } return true @@ -528,7 +561,7 @@ class PreviewView @JvmOverloads constructor( /** * A listener for the [PreviewView]. */ - interface Listener { + interface PreviewListener { /** * Called when the preview is started. */ @@ -538,12 +571,6 @@ class PreviewView @JvmOverloads constructor( * Called when the preview failed to start. */ fun onPreviewFailed(t: Throwable) {} - - /** - * Called when the zoom ratio is changed. - * @param zoomRatio the new zoom ratio - */ - fun onZoomRationOnPinchChanged(zoomRatio: Float) {} } /** From bf905ee93913972c2751f422b481cdcec1a01664 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 15:50:13 +0100 Subject: [PATCH 24/72] fix(core): camera: prefix torch strength level range with available --- .../core/elements/sources/video/camera/CameraSettings.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index d81e54fb4..91fc4f7b9 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -328,7 +328,7 @@ class CameraSettings internal constructor( * Use the range to call [setStrengthLevel] */ @delegate:RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) - val strengthLevelRange: Range by lazy { + val availableStrengthLevelRange: Range by lazy { Range( 1, characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] ?: 1 @@ -348,7 +348,7 @@ class CameraSettings internal constructor( * Sets the flash strength. * * @param level flash strength. Range is from [1-x]. - * @see [strengthLevelRange] + * @see [availableStrengthLevelRange] */ @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) suspend fun setStrengthLevel(level: Int) { From 4e61275fc91f52a5312fc819ac6cf5e21cc429a6 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 4 Feb 2026 10:44:36 +0100 Subject: [PATCH 25/72] feat(core): make SingleStreamer inputs non nullable --- .../core/pipelines/StreamerPipelineTest.kt | 4 +- .../file/CameraSingleStreamerFileTest.kt | 1 + .../CameraSingleStreamerMultiEndpointTest.kt | 1 + .../state/CameraSingleStreamerStateTest.kt | 1 + .../single/state/SingleStreamerStateTest.kt | 1 + .../core/streamer/utils/StreamerUtils.kt | 4 +- .../streampack/core/interfaces/ISource.kt | 25 +- .../core/pipelines/StreamerPipeline.kt | 6 +- .../single/AudioOnlySingleStreamer.kt | 31 +- .../core/streamers/single/ISingleStreamer.kt | 27 +- .../core/streamers/single/SingleStreamer.kt | 330 +++++------------- .../streamers/single/SingleStreamerImpl.kt | 278 +++++++++++++++ .../single/VideoOnlySingleStreamer.kt | 56 ++- .../app/ui/main/PreviewViewModel.kt | 53 ++- .../ui/main/usecases/BuildStreamerUseCase.kt | 41 ++- .../streampack/app/utils/Extensions.kt | 4 +- .../services/MediaProjectionService.kt | 2 +- .../services/utils/StreamerFactory.kt | 22 +- .../streampack/ui/views/PreviewView.kt | 6 +- .../streampack/ui/views/StreamerExtensions.kt | 4 +- 20 files changed, 560 insertions(+), 337 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt index 8de63b080..d8591b2a3 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt @@ -83,8 +83,8 @@ class StreamerPipelineTest { ) streamerPipeline.addOutput(output) - val audioSource = streamerPipeline.audioInput?.sourceFlow?.value as IAudioSourceInternal - val videoSource = streamerPipeline.videoInput?.sourceFlow?.value as IVideoSourceInternal + val audioSource = streamerPipeline.audioInput.sourceFlow.value as IAudioSourceInternal + val videoSource = streamerPipeline.videoInput.sourceFlow.value as IVideoSourceInternal streamerPipeline.startStream() assertTrue(streamerPipeline.isStreamingFlow.first { it }) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt index 548b4e053..607d4849e 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt @@ -28,6 +28,7 @@ import io.github.thibaultbee.streampack.core.streamer.single.utils.SingleStreame import io.github.thibaultbee.streampack.core.streamer.utils.StreamerUtils import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt index 9a1f8a278..34da1422f 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt @@ -27,6 +27,7 @@ import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt index 0f5fcebd7..ee9232fe6 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt @@ -30,6 +30,7 @@ import io.github.thibaultbee.streampack.core.streamer.surface.SurfaceViewTestAct import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt index bfe9a2e05..81387d36b 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt @@ -22,6 +22,7 @@ import io.github.thibaultbee.streampack.core.interfaces.startStream import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import kotlinx.coroutines.test.runTest import org.junit.After diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt index 6571955c4..d4835c14e 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt @@ -4,7 +4,7 @@ import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.Media import io.github.thibaultbee.streampack.core.interfaces.ICloseableStreamer import io.github.thibaultbee.streampack.core.interfaces.startStream import io.github.thibaultbee.streampack.core.streamers.dual.DualStreamer -import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.ISingleStreamer import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.withContext @@ -14,7 +14,7 @@ import kotlin.time.Duration.Companion.seconds object StreamerUtils { suspend fun runSingleStream( - streamer: SingleStreamer, + streamer: ISingleStreamer, descriptor: MediaDescriptor, duration: Duration, pollDuration: Duration = 1.seconds diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt index 60a476805..80ae9b9e4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt @@ -36,7 +36,7 @@ interface IWithAudioSource { /** * The audio input to access to advanced settings. */ - val audioInput: IAudioInput? + val audioInput: IAudioInput /** * Sets a new audio source. @@ -44,8 +44,7 @@ interface IWithAudioSource { * @param audioSourceFactory The new audio source factory. */ suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) { - val audio = requireNotNull(audioInput) { "Audio input is not available" } - audio.setSource(audioSourceFactory) + audioInput.setSource(audioSourceFactory) } } @@ -63,9 +62,9 @@ interface IWithVideoRotation { */ interface IWithVideoSource { /** - * The audio input to access to advanced settings. + * The video input to access to advanced settings. */ - val videoInput: IVideoInput? + val videoInput: IVideoInput /** * Sets the video source. @@ -73,8 +72,7 @@ interface IWithVideoSource { * The previous video source will be released unless its preview is still running. */ suspend fun setVideoSource(videoSourceFactory: IVideoSourceInternal.Factory) { - val video = requireNotNull(videoInput) { "Video input is not available" } - video.setSource(videoSourceFactory) + videoInput.setSource(videoSourceFactory) } } @@ -87,8 +85,7 @@ interface IWithVideoSource { */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.setCameraId(cameraId: String) { - val video = requireNotNull(videoInput) { "Video input is not available" } - video.setSource(CameraSourceFactory(cameraId)) + videoInput.setSource(CameraSourceFactory(cameraId)) } @@ -96,7 +93,7 @@ suspend fun IWithVideoSource.setCameraId(cameraId: String) { * Whether the video source has a preview. */ val IWithVideoSource.isPreviewable: Boolean - get() = videoInput?.sourceFlow?.value is IPreviewableSource + get() = videoInput.sourceFlow?.value is IPreviewableSource /** * Sets the preview surface. @@ -105,7 +102,7 @@ val IWithVideoSource.isPreviewable: Boolean * @throws [IllegalStateException] if the video source is not previewable */ suspend fun IWithVideoSource.setPreview(surface: Surface) { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.setPreview(surface) + (videoInput.sourceFlow.value as? IPreviewableSource)?.setPreview(surface) ?: throw IllegalStateException("Video source is not previewable") } @@ -142,7 +139,7 @@ suspend fun IWithVideoSource.setPreview(textureView: TextureView) = * @throws [IllegalStateException] if the video source is not previewable */ suspend fun IWithVideoSource.startPreview() { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.startPreview() + (videoInput.sourceFlow.value as? IPreviewableSource)?.startPreview() ?: throw IllegalStateException("Video source is not previewable") } @@ -154,7 +151,7 @@ suspend fun IWithVideoSource.startPreview() { * @see [IWithVideoSource.stopPreview] */ suspend fun IWithVideoSource.startPreview(surface: Surface) { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.startPreview(surface) + (videoInput.sourceFlow.value as? IPreviewableSource)?.startPreview(surface) ?: throw IllegalStateException("Video source is not previewable") } @@ -195,5 +192,5 @@ suspend fun IWithVideoSource.startPreview(textureView: TextureView) = * Stops video preview. */ suspend fun IWithVideoSource.stopPreview() { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.stopPreview() + (videoInput.sourceFlow.value as? IPreviewableSource)?.stopPreview() } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt index 26d417892..28e53f39e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt @@ -116,7 +116,8 @@ open class StreamerPipeline( } else { null } - override val audioInput: IAudioInput? = _audioInput + override val audioInput: IAudioInput + get() = requireNotNull(_audioInput) { "Audio input is not available" } private val _videoInput = if (withVideo) { VideoInput(context, surfaceProcessorFactory, dispatcherProvider) { @@ -126,7 +127,8 @@ open class StreamerPipeline( null } - override val videoInput: IVideoInput? = _videoInput + override val videoInput: IVideoInput + get() = requireNotNull(_videoInput) { "Video input is not available" } private val _isStreamingFlow = MutableStateFlow(false) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt index d8043700c..2408dda37 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt @@ -25,8 +25,9 @@ import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSourceFactory +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput -import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo /** @@ -35,15 +36,18 @@ import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo * @param context the application context * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If parameter is null, no audio source are set. It can be set later with [AudioOnlySingleStreamer.setAudioSource]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun AudioOnlySingleStreamer( context: Context, audioSourceFactory: IAudioSourceInternal.Factory = MicrophoneSourceFactory(), - endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory() + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): AudioOnlySingleStreamer { val streamer = AudioOnlySingleStreamer( context = context, endpointFactory = endpointFactory, + dispatcherProvider = dispatcherProvider ) streamer.setAudioSource(audioSourceFactory) return streamer @@ -54,16 +58,19 @@ suspend fun AudioOnlySingleStreamer( * * @param context the application context * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class AudioOnlySingleStreamer( context: Context, - endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory() + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : ISingleStreamer, IAudioSingleStreamer { - private val streamer = SingleStreamer( + private val streamer = SingleStreamerImpl( context = context, - endpointFactory = endpointFactory, withAudio = true, - withVideo = false + withVideo = false, + endpointFactory = endpointFactory, + dispatcherProvider = dispatcherProvider ) override val throwableFlow = streamer.throwableFlow override val isOpenFlow = streamer.isOpenFlow @@ -74,8 +81,8 @@ class AudioOnlySingleStreamer( get() = streamer.info override val audioConfigFlow = streamer.audioConfigFlow - override val audioInput: IAudioInput = streamer.audioInput!! - + override val audioInput: IAudioInput = streamer.audioInput + override val audioEncoder: IEncoder? get() = streamer.audioEncoder @@ -97,12 +104,4 @@ class AudioOnlySingleStreamer( override suspend fun stopStream() = streamer.stopStream() override suspend fun release() = streamer.release() - - override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) { - throw UnsupportedOperationException("Audio single streamer does not support bitrate regulator controller") - } - - override fun removeBitrateRegulatorController() { - // Do nothing - } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt index d19ffbfc3..f0d14fe8c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt @@ -55,22 +55,25 @@ interface ISingleStreamer : IOpenableStreamer { * Gets configuration information */ fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo +} +val ISingleStreamer.withAudio: Boolean /** - * Adds a bitrate regulator controller to the streamer. + * Whether the streamer has an audio source. */ - fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) + get() = this is IAudioSingleStreamer +val ISingleStreamer.withVideo: Boolean /** - * Removes the bitrate regulator controller from the streamer. + * Whether the streamer has a video source. */ - fun removeBitrateRegulatorController() -} + get() = this is IVideoSingleStreamer + /** * An audio single Streamer */ -interface IAudioSingleStreamer : IAudioStreamer { +interface IAudioSingleStreamer : IAudioStreamer, ISingleStreamer { /** * The audio configuration flow. */ @@ -85,7 +88,7 @@ interface IAudioSingleStreamer : IAudioStreamer { /** * A video single streamer. */ -interface IVideoSingleStreamer : IVideoStreamer { +interface IVideoSingleStreamer : IVideoStreamer, ISingleStreamer { /** * The video configuration flow. */ @@ -95,5 +98,15 @@ interface IVideoSingleStreamer : IVideoStreamer { * Advanced settings for the video encoder. */ val videoEncoder: IEncoder? + + /** + * Adds a bitrate regulator controller to the streamer. + */ + fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) + + /** + * Removes the bitrate regulator controller from the streamer. + */ + fun removeBitrateRegulatorController() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt index d7c1138da..6b3caa9eb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt @@ -25,7 +25,6 @@ import androidx.annotation.RequiresApi import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder -import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal @@ -35,7 +34,6 @@ import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MediaProjectionAudioSourceFactory import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSourceFactory import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue @@ -43,19 +41,10 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRo import io.github.thibaultbee.streampack.core.interfaces.setCameraId import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController -import io.github.thibaultbee.streampack.core.streamers.infos.CameraStreamerConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo -import io.github.thibaultbee.streampack.core.streamers.infos.StreamerConfigurationInfo -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.cancel -import kotlinx.coroutines.flow.SharingStarted -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.merge -import kotlinx.coroutines.flow.stateIn -import kotlinx.coroutines.runBlocking /** @@ -66,6 +55,8 @@ import kotlinx.coroutines.runBlocking * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If set to null, you will have to set it later explicitly. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraSingleStreamer( @@ -73,10 +64,16 @@ suspend fun cameraSingleStreamer( cameraId: String = context.defaultCameraId, audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( - context, withAudio = true, withVideo = true, endpointFactory, defaultRotation + context = context, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) if (audioSourceFactory != null) { @@ -92,20 +89,24 @@ suspend fun cameraSingleStreamer( * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresApi(Build.VERSION_CODES.Q) suspend fun audioVideoMediaProjectionSingleStreamer( context: Context, mediaProjection: MediaProjection, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, endpointFactory = endpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) @@ -121,20 +122,24 @@ suspend fun audioVideoMediaProjectionSingleStreamer( * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If set to null, you will have to set it later explicitly. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionSingleStreamer( context: Context, mediaProjection: MediaProjection, audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, endpointFactory = endpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) @@ -152,6 +157,8 @@ suspend fun videoMediaProjectionSingleStreamer( * @param videoSourceFactory the video source factory. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun SingleStreamer( context: Context, @@ -159,13 +166,15 @@ suspend fun SingleStreamer( videoSourceFactory: IVideoSourceInternal.Factory, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, - withAudio = true, - withVideo = true, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setAudioSource(audioSourceFactory) streamer.setVideoSource(videoSourceFactory) @@ -173,249 +182,102 @@ suspend fun SingleStreamer( } /** - * A [ISingleStreamer] implementation for audio and video. + * A [ISingleStreamer] implementation for both audio and video. * * @param context the application context - * @param withAudio `true` to capture audio. It can't be changed after instantiation. - * @param withVideo `true` to capture video. It can't be changed after instantiation. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ -open class SingleStreamer( - protected val context: Context, - val withAudio: Boolean = true, - val withVideo: Boolean = true, +class SingleStreamer( + context: Context, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), dispatcherProvider: IDispatcherProvider = DispatcherProvider(), ) : ISingleStreamer, IAudioSingleStreamer, IVideoSingleStreamer { - private val coroutineScope: CoroutineScope = CoroutineScope(dispatcherProvider.default) - - private val pipeline = StreamerPipeline( - context, - withAudio, - withVideo, - audioOutputMode = StreamerPipeline.AudioOutputMode.CALLBACK, - surfaceProcessorFactory, - dispatcherProvider + private val streamer = SingleStreamerImpl( + context = context, + withAudio = true, + withVideo = true, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - private val pipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, - withVideo, - endpointFactory, - defaultRotation - ) as IEncodingPipelineOutputInternal - } - - override val throwableFlow: StateFlow = - merge(pipeline.throwableFlow, pipelineOutput.throwableFlow).stateIn( - coroutineScope, - SharingStarted.Eagerly, - null - ) - - override val isOpenFlow: StateFlow - get() = pipelineOutput.isOpenFlow - override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + override val throwableFlow = streamer.throwableFlow + override val isOpenFlow = streamer.isOpenFlow + override val isStreamingFlow = streamer.isStreamingFlow - // AUDIO - /** - * The audio input. - * It allows advanced audio source settings. - */ - override val audioInput = pipeline.audioInput + override val endpoint: IEndpoint + get() = streamer.endpoint + override val info: IConfigurationInfo + get() = streamer.info + override val audioConfigFlow = streamer.audioConfigFlow override val audioEncoder: IEncoder? - get() = pipelineOutput.audioEncoder - - override suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) = - pipeline.setAudioSource(audioSourceFactory) - - // VIDEO - /** - * The video input. - * It allows advanced video source settings. - */ - override val videoInput = pipeline.videoInput + get() = streamer.audioEncoder + override val audioInput: IAudioInput = streamer.audioInput!! + override val videoConfigFlow = streamer.videoConfigFlow override val videoEncoder: IEncoder? - get() = pipelineOutput.videoEncoder - - // ENDPOINT - override val endpoint: IEndpoint - get() = pipelineOutput.endpoint + get() = streamer.videoEncoder + override val videoInput: IVideoInput = streamer.videoInput!! /** * Sets the target rotation. * * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) */ - override suspend fun setTargetRotation(@RotationValue rotation: Int) { - pipeline.setTargetRotation(rotation) - } - - /** - * Gets configuration information. - * - * Could throw an exception if the endpoint needs to infer the configuration from the - * [MediaDescriptor]. - * In this case, prefer using [getInfo] with the [MediaDescriptor] used in [open]. - */ - override val info: IConfigurationInfo - get() = if (videoInput?.sourceFlow?.value is CameraSource) { - CameraStreamerConfigurationInfo(endpoint.info) - } else { - StreamerConfigurationInfo(endpoint.info) - } + override suspend fun setTargetRotation(@RotationValue rotation: Int) = + streamer.setTargetRotation(rotation) - /** - * Gets configuration information from [MediaDescriptor]. - * - * If the endpoint is not [DynamicEndpoint], [descriptor] is unused as the endpoint type is - * already known. - * - * @param descriptor the media descriptor - */ - override fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo { - val endpointInfo = try { - endpoint.info - } catch (_: Throwable) { - endpoint.getInfo(descriptor) - } - return if (videoInput?.sourceFlow?.value is CameraSource) { - CameraStreamerConfigurationInfo(endpointInfo) - } else { - StreamerConfigurationInfo(endpointInfo) - } - } - - // CONFIGURATION - /** - * The audio configuration flow. - */ - override val audioConfigFlow: StateFlow = pipelineOutput.audioCodecConfigFlow - - /** - * Configures audio settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and audio capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * @param audioConfig Audio configuration to set - * - * @throws [Throwable] if configuration can not be applied. - */ @RequiresPermission(Manifest.permission.RECORD_AUDIO) - override suspend fun setAudioConfig(audioConfig: AudioConfig) { - pipelineOutput.setAudioCodecConfig(audioConfig) - } + override suspend fun setAudioConfig(audioConfig: AudioConfig) = + streamer.setAudioConfig(audioConfig) - /** - * The video configuration flow. - */ - override val videoConfigFlow: StateFlow = pipelineOutput.videoCodecConfigFlow - - /** - * Configures video settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - */ - override suspend fun setVideoConfig(videoConfig: VideoConfig) { - pipelineOutput.setVideoCodecConfig(videoConfig) - } + override suspend fun setVideoConfig(videoConfig: VideoConfig) = + streamer.setVideoConfig(videoConfig) - /** - * Configures both video and audio settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and audio and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param audioConfig Audio configuration to set - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - * @see [IStreamer.release] - */ - @RequiresPermission(Manifest.permission.RECORD_AUDIO) - suspend fun setConfig(audioConfig: AudioConfig, videoConfig: VideoConfig) { - setAudioConfig(audioConfig) - setVideoConfig(videoConfig) - } + override fun getInfo(descriptor: MediaDescriptor) = streamer.getInfo(descriptor) - /** - * Opens the streamer endpoint. - * - * @param descriptor Media descriptor to open - */ - override suspend fun open(descriptor: MediaDescriptor) = pipelineOutput.open(descriptor) + override suspend fun open(descriptor: MediaDescriptor) = streamer.open(descriptor) - /** - * Closes the streamer endpoint. - */ - override suspend fun close() = pipelineOutput.close() + override suspend fun close() = streamer.close() - /** - * Starts audio/video stream. - * Stream depends of the endpoint: Audio/video could be write to a file or send to a remote - * device. - * To avoid creating an unresponsive UI, do not call on main thread. - * - * @see [stopStream] - */ - override suspend fun startStream() = pipelineOutput.startStream() + override suspend fun startStream() = streamer.startStream() - /** - * Stops audio/video stream. - * - * Internally, it resets audio and video recorders and encoders to get them ready for another - * [startStream] session. It explains why preview could be restarted. - * - * @see [startStream] - */ - override suspend fun stopStream() = pipeline.stopStream() + override suspend fun stopStream() = streamer.stopStream() - /** - * Releases the streamer. - */ - override suspend fun release() { - pipeline.release() - coroutineScope.cancel() - } + override suspend fun release() = streamer.release() - /** - * Adds a bitrate regulator controller. - * - * Limitation: it is only available for SRT for now. - */ override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) = - pipelineOutput.addBitrateRegulatorController(controllerFactory) + streamer.addBitrateRegulatorController(controllerFactory) - /** - * Removes the bitrate regulator controller. - */ - override fun removeBitrateRegulatorController() = - pipelineOutput.removeBitrateRegulatorController() + override fun removeBitrateRegulatorController() = streamer.removeBitrateRegulatorController() +} - companion object { - const val TAG = "SingleStreamer" - } + +/** + * Configures both video and audio settings. + * It is the first method to call after a [SingleStreamer] instantiation. + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [SingleStreamer.release] + */ +@RequiresPermission(Manifest.permission.RECORD_AUDIO) +suspend fun SingleStreamer.setConfig(audioConfig: AudioConfig, videoConfig: VideoConfig) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt new file mode 100644 index 000000000..e334ca746 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt @@ -0,0 +1,278 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.streamers.single + +import android.Manifest +import android.content.Context +import android.view.Surface +import androidx.annotation.RequiresPermission +import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor +import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSource +import io.github.thibaultbee.streampack.core.elements.utils.RotationValue +import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController +import io.github.thibaultbee.streampack.core.streamers.infos.CameraStreamerConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.infos.StreamerConfigurationInfo +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking + +/** + * A [ISingleStreamer] implementation for audio and video. + * + * @param context the application context + * @param withAudio `true` to capture audio. It can't be changed after instantiation. + * @param withVideo `true` to capture video. It can't be changed after instantiation. + * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + */ +internal class SingleStreamerImpl( + private val context: Context, + withAudio: Boolean, + withVideo: Boolean, + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider(), +) : ISingleStreamer, IAudioSingleStreamer, IVideoSingleStreamer { + private val coroutineScope: CoroutineScope = CoroutineScope(dispatcherProvider.default) + + private val pipeline = StreamerPipeline( + context, + withAudio, + withVideo, + audioOutputMode = StreamerPipeline.AudioOutputMode.CALLBACK, + surfaceProcessorFactory, + dispatcherProvider + ) + private val pipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, + withVideo, + endpointFactory, + defaultRotation + ) as IEncodingPipelineOutputInternal + } + + override val throwableFlow: StateFlow = + merge(pipeline.throwableFlow, pipelineOutput.throwableFlow).stateIn( + coroutineScope, + SharingStarted.Eagerly, + null + ) + + override val isOpenFlow: StateFlow + get() = pipelineOutput.isOpenFlow + + override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + + // AUDIO + /** + * The audio input. + * It allows advanced audio source settings. + */ + override val audioInput: IAudioInput + get() = pipeline.audioInput + + override val audioEncoder: IEncoder? + get() = pipelineOutput.audioEncoder + + override suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) = + pipeline.setAudioSource(audioSourceFactory) + + // VIDEO + /** + * The video input. + * It allows advanced video source settings. + */ + override val videoInput: IVideoInput + get() = pipeline.videoInput + + override val videoEncoder: IEncoder? + get() = pipelineOutput.videoEncoder + + // ENDPOINT + override val endpoint: IEndpoint + get() = pipelineOutput.endpoint + + /** + * Sets the target rotation. + * + * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) + */ + override suspend fun setTargetRotation(@RotationValue rotation: Int) { + pipeline.setTargetRotation(rotation) + } + + /** + * Gets configuration information. + * + * Could throw an exception if the endpoint needs to infer the configuration from the + * [MediaDescriptor]. + * In this case, prefer using [getInfo] with the [MediaDescriptor] used in [open]. + */ + override val info: IConfigurationInfo + get() = if (videoInput.sourceFlow.value is CameraSource) { + CameraStreamerConfigurationInfo(endpoint.info) + } else { + StreamerConfigurationInfo(endpoint.info) + } + + /** + * Gets configuration information from [MediaDescriptor]. + * + * If the endpoint is not [DynamicEndpoint], [descriptor] is unused as the endpoint type is + * already known. + * + * @param descriptor the media descriptor + */ + override fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo { + val endpointInfo = try { + endpoint.info + } catch (_: Throwable) { + endpoint.getInfo(descriptor) + } + return if (videoInput.sourceFlow.value is CameraSource) { + CameraStreamerConfigurationInfo(endpointInfo) + } else { + StreamerConfigurationInfo(endpointInfo) + } + } + + // CONFIGURATION + /** + * The audio configuration flow. + */ + override val audioConfigFlow: StateFlow = pipelineOutput.audioCodecConfigFlow + + /** + * Configures audio settings. + * It is the first method to call after a [SingleStreamerImpl] instantiation. + * It must be call when both stream and audio capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * @param audioConfig Audio configuration to set + * + * @throws [Throwable] if configuration can not be applied. + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + override suspend fun setAudioConfig(audioConfig: AudioConfig) { + pipelineOutput.setAudioCodecConfig(audioConfig) + } + + /** + * The video configuration flow. + */ + override val videoConfigFlow: StateFlow = pipelineOutput.videoCodecConfigFlow + + /** + * Configures video settings. + * It is the first method to call after a [SingleStreamerImpl] instantiation. + * It must be call when both stream and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + */ + override suspend fun setVideoConfig(videoConfig: VideoConfig) { + pipelineOutput.setVideoCodecConfig(videoConfig) + } + + /** + * Opens the streamer endpoint. + * + * @param descriptor Media descriptor to open + */ + override suspend fun open(descriptor: MediaDescriptor) = pipelineOutput.open(descriptor) + + /** + * Closes the streamer endpoint. + */ + override suspend fun close() = pipelineOutput.close() + + /** + * Starts audio/video stream. + * Stream depends of the endpoint: Audio/video could be write to a file or send to a remote + * device. + * To avoid creating an unresponsive UI, do not call on main thread. + * + * @see [stopStream] + */ + override suspend fun startStream() = pipelineOutput.startStream() + + /** + * Stops audio/video stream. + * + * Internally, it resets audio and video recorders and encoders to get them ready for another + * [startStream] session. It explains why preview could be restarted. + * + * @see [startStream] + */ + override suspend fun stopStream() = pipeline.stopStream() + + /** + * Releases the streamer. + */ + override suspend fun release() { + pipeline.release() + coroutineScope.cancel() + } + + /** + * Adds a bitrate regulator controller. + * + * Limitation: it is only available for SRT for now. + */ + override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) = + pipelineOutput.addBitrateRegulatorController(controllerFactory) + + /** + * Removes the bitrate regulator controller. + */ + override fun removeBitrateRegulatorController() = + pipelineOutput.removeBitrateRegulatorController() + + companion object Companion { + const val TAG = "SingleStreamer" + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt index fa8071750..c224ba146 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt @@ -26,12 +26,16 @@ import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.setCameraId +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo @@ -44,39 +48,53 @@ import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo * @param cameraId the camera id to use. By default, it is the default camera. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraVideoOnlySingleStreamer( context: Context, cameraId: String = context.defaultCameraId, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( - context, endpointFactory, defaultRotation + context = context, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) return streamer } /** - * Creates a [SingleStreamer] with the screen as video source and no audio source. + * Creates a [VideoOnlySingleStreamer] with the screen as video source and no audio source. * * @param context the application context * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. * @param endpointFactory the [IEndpointInternal.Factory] implementation * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionVideoOnlySingleStreamer( context: Context, mediaProjection: MediaProjection, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( context = context, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) return streamer @@ -89,17 +107,23 @@ suspend fun videoMediaProjectionVideoOnlySingleStreamer( * @param videoSourceFactory the video source factory. If parameter is null, no audio source are set. It can be set later with [VideoOnlySingleStreamer.setVideoSource]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun VideoOnlySingleStreamer( context: Context, videoSourceFactory: IVideoSourceInternal.Factory, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( context = context, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(videoSourceFactory) return streamer @@ -111,19 +135,27 @@ suspend fun VideoOnlySingleStreamer( * @param context the application context * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class VideoOnlySingleStreamer( context: Context, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : ISingleStreamer, IVideoSingleStreamer { - private val streamer = SingleStreamer( + private val streamer = SingleStreamerImpl( context = context, - endpointFactory = endpointFactory, withAudio = false, withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + endpointFactory = endpointFactory, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + override val throwableFlow = streamer.throwableFlow override val isOpenFlow = streamer.isOpenFlow override val isStreamingFlow = streamer.isStreamingFlow @@ -136,7 +168,7 @@ class VideoOnlySingleStreamer( override val videoConfigFlow = streamer.videoConfigFlow override val videoEncoder: IEncoder? get() = streamer.videoEncoder - override val videoInput: IVideoInput = streamer.videoInput!! + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index cf15308d4..9dcb6c070 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -55,7 +55,12 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking import io.github.thibaultbee.streampack.core.interfaces.startStream +import io.github.thibaultbee.streampack.core.streamers.single.IAudioSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.IVideoSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.withAudio +import io.github.thibaultbee.streampack.core.streamers.single.withVideo import io.github.thibaultbee.streampack.core.utils.extensions.isClosedException import io.github.thibaultbee.streampack.ext.srt.regulator.controllers.DefaultSrtBitrateRegulatorController import kotlinx.coroutines.CancellationException @@ -79,16 +84,18 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod private val defaultDispatcher = Dispatchers.IO - private val buildStreamerUseCase = BuildStreamerUseCase(application, storageRepository) + private val buildStreamerUseCase = BuildStreamerUseCase(application) private val streamerFlow = - MutableStateFlow( - SingleStreamer( - application, - runBlocking { storageRepository.isAudioEnableFlow.first() }) // TODO avoid runBlocking + MutableStateFlow( + buildFirstStreamer(runBlocking { storageRepository.isAudioEnableFlow.first() }) ) - private val streamer: SingleStreamer + + private val streamer: IVideoSingleStreamer get() = streamerFlow.value + private val audioStreamer: IAudioSingleStreamer? + get() = streamer as? IAudioSingleStreamer? + val streamerLiveData = streamerFlow.asLiveData() /** @@ -111,10 +118,10 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod val requiredPermissions: List get() { val permissions = mutableListOf() - if (streamer.videoInput?.sourceFlow is ICameraSource) { + if (streamer.videoInput.sourceFlow is ICameraSource) { permissions.add(Manifest.permission.CAMERA) } - if (streamer.audioInput?.sourceFlow?.value is IAudioRecordSource) { + if (audioStreamer?.audioInput?.sourceFlow?.value is IAudioRecordSource) { permissions.add(Manifest.permission.RECORD_AUDIO) } storageRepository.endpointDescriptorFlow.asLiveData().value?.let { @@ -149,7 +156,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod // Set audio source and video source if (streamer.withAudio) { Log.i(TAG, "Audio source is enabled. Setting audio source") - streamer.setAudioSource(MicrophoneSourceFactory()) + audioStreamer!!.setAudioSource(MicrophoneSourceFactory()) } else { Log.i(TAG, "Audio source is disabled") } @@ -167,7 +174,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod // TODO: cancel jobs linked to previous streamer viewModelScope.launch { - streamer.videoInput?.sourceFlow?.collect { + streamer.videoInput.sourceFlow.collect { notifySourceChanged() } } @@ -226,7 +233,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod ) == PackageManager.PERMISSION_GRANTED ) { try { - streamer.setAudioConfig(config) + audioStreamer!!.setAudioConfig(config) } catch (t: Throwable) { Log.e(TAG, "setAudioConfig failed", t) _streamerErrorLiveData.postValue("setAudioConfig: ${t.message ?: "Unknown error"}") @@ -247,6 +254,14 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } + private fun buildFirstStreamer(isAudioEnable: Boolean): IVideoSingleStreamer { + return if (isAudioEnable) { + SingleStreamer(application) + } else { + VideoOnlySingleStreamer(application) + } + } + fun onZoomChanged() { notifyPropertyChanged(BR.zoomRatio) } @@ -255,7 +270,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun configureAudio() { viewModelScope.launch { try { - storageRepository.audioConfigFlow.first()?.let { streamer.setAudioConfig(it) } + storageRepository.audioConfigFlow.first()?.let { audioStreamer?.setAudioConfig(it) } ?: Log.i( TAG, "Audio is disabled" @@ -271,7 +286,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun initializeVideoSource() { viewModelScope.launch { videoSourceMutex.withLock { - if (streamer.videoInput?.sourceFlow?.value == null) { + if (streamer.videoInput.sourceFlow.value == null) { streamer.setVideoSource(CameraSourceFactory(defaultCameraId)) } else { Log.i(TAG, "Camera source already set") @@ -327,7 +342,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } fun setMute(isMuted: Boolean) { - streamer.audioInput?.isMuted = isMuted + audioStreamer?.audioInput?.isMuted = isMuted } @RequiresPermission(Manifest.permission.CAMERA) @@ -337,7 +352,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod * exception instead of crashing. You can either catch the exception or check if the * configuration is valid for the new camera with [Context.isFpsSupported]. */ - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is ICameraSource) { viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { @@ -357,7 +372,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod */ viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is ICameraSource) { streamer.setNextCameraId(application) } @@ -369,7 +384,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun toggleVideoSource() { viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value val nextSource = when (videoSource) { is ICameraSource -> { BitmapSourceFactory(testBitmap) @@ -390,7 +405,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } - val isCameraSource = streamer.videoInput?.sourceFlow?.map { it is ICameraSource }?.asLiveData() + val isCameraSource = streamer.videoInput.sourceFlow?.map { it is ICameraSource }?.asLiveData() val isFlashAvailable = MutableLiveData(false) fun toggleFlash() { @@ -514,7 +529,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } private fun notifySourceChanged() { - val videoSource = streamer.videoInput?.sourceFlow?.value ?: return + val videoSource = streamer.videoInput.sourceFlow.value ?: return if (videoSource is ICameraSource) { notifyCameraChanged(videoSource) } else { diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt index 417f6614c..beef3a8f9 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt @@ -4,15 +4,17 @@ import android.Manifest import android.content.Context import android.content.pm.PackageManager import androidx.core.app.ActivityCompat -import io.github.thibaultbee.streampack.app.data.storage.DataStoreRepository +import io.github.thibaultbee.streampack.core.streamers.single.IAudioSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.IVideoSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.withAudio class BuildStreamerUseCase( - private val context: Context, - private val dataStoreRepository: DataStoreRepository + private val context: Context ) { /** - * Build a new [SingleStreamer] based on audio and video preferences. + * Build a new [IVideoSingleStreamer] based on audio and video preferences. * * Only create a new streamer if the previous one is not the same type. * @@ -21,28 +23,37 @@ class BuildStreamerUseCase( * @param previousStreamer Previous streamer to check if we need to create a new one. */ suspend operator fun invoke( - previousStreamer: SingleStreamer, + previousStreamer: IVideoSingleStreamer, isAudioEnable: Boolean - ): SingleStreamer { + ): IVideoSingleStreamer { if (previousStreamer.withAudio != isAudioEnable) { - return SingleStreamer(context, isAudioEnable).apply { - // Get previous streamer config if any + previousStreamer.release() + + val streamer = if (isAudioEnable) { + SingleStreamer(context) + } else { + VideoOnlySingleStreamer(context) + } + + // Get previous streamer config if any + if ((previousStreamer is IAudioSingleStreamer) && (streamer is IAudioSingleStreamer)) { val audioConfig = previousStreamer.audioConfigFlow.value - val videoConfig = previousStreamer.videoConfigFlow.value - if ((audioConfig != null && isAudioEnable)) { + if (audioConfig != null) { if (ActivityCompat.checkSelfPermission( context, Manifest.permission.RECORD_AUDIO ) == PackageManager.PERMISSION_GRANTED ) { - setAudioConfig(audioConfig) + streamer.setAudioConfig(audioConfig) } } - if (videoConfig != null) { - setVideoConfig(videoConfig) - } - previousStreamer.release() } + + val videoConfig = previousStreamer.videoConfigFlow.value + if (videoConfig != null) { + streamer.setVideoConfig(videoConfig) + } + return streamer } return previousStreamer } diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt index ec45c6943..bbe54ba7d 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt @@ -37,7 +37,7 @@ import io.github.thibaultbee.streampack.core.interfaces.setCameraId @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.setNextCameraId(context: Context) { val cameras = context.cameraManager.cameras - val videoSource = videoInput?.sourceFlow?.value + val videoSource = videoInput.sourceFlow.value val newCameraId = if (videoSource is ICameraSource) { val currentCameraIndex = cameras.indexOf(videoSource.cameraId) @@ -52,7 +52,7 @@ suspend fun IWithVideoSource.setNextCameraId(context: Context) { @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.toggleBackToFront(context: Context) { val cameraManager = context.cameraManager - val videoSource = videoInput?.sourceFlow?.value + val videoSource = videoInput.sourceFlow.value val cameras = if (videoSource is ICameraSource) { if (cameraManager.isBackCamera(videoSource.cameraId)) { cameraManager.frontCameras diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt index 0b740e6b5..5f5888bfe 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt @@ -163,7 +163,7 @@ abstract class MediaProjectionService( } if (streamer is IWithVideoSource) { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is IMediaProjectionSource) { streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) } else if (videoSource == null) { diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt index 61f5bc703..78e08fbab 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt @@ -16,13 +16,16 @@ package io.github.thibaultbee.streampack.services.utils import android.content.Context +import android.view.Surface import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.streamers.dual.DualStreamer import io.github.thibaultbee.streampack.core.streamers.dual.IDualStreamer +import io.github.thibaultbee.streampack.core.streamers.single.AudioOnlySingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.ISingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer import io.github.thibaultbee.streampack.services.StreamerService @@ -53,12 +56,19 @@ open class SingleStreamerFactory( ) : StreamerFactory { override fun create(context: Context): ISingleStreamer { - return SingleStreamer( - context, - withAudio, - withVideo, - defaultRotation = defaultRotation ?: context.displayRotation - ) + return if (withAudio && withVideo) { + SingleStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else if (withAudio) { + AudioOnlySingleStreamer(context) + } else { + VideoOnlySingleStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } } } diff --git a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt index c2837621d..50da71a1b 100644 --- a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt @@ -163,8 +163,8 @@ class PreviewView @JvmOverloads constructor( streamer: IWithVideoSource ) { sourceJob += defaultScope.launch { - streamer.videoInput?.sourceFlow?.runningHistoryNotNull() - ?.collect { (previousVideoSource, newVideoSource) -> + streamer.videoInput.sourceFlow.runningHistoryNotNull() + .collect { (previousVideoSource, newVideoSource) -> if (previousVideoSource == newVideoSource) { Logger.w(TAG, "No change in video source") } else { @@ -455,7 +455,7 @@ class PreviewView @JvmOverloads constructor( private suspend fun stopPreview() { streamer?.let { - val videoSource = it.videoInput?.sourceFlow?.value + val videoSource = it.videoInput.sourceFlow.value if (videoSource is IPreviewableSource) { Logger.d(TAG, "Stopping preview") videoSource.previewMutex.withLock { diff --git a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt index 7134c77cd..68a4efd41 100644 --- a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt @@ -18,7 +18,7 @@ suspend fun IWithVideoSource.setPreview( viewfinder: CameraViewfinder, previewSize: Size ): ViewfinderSurfaceRequest { - val videoSource = videoInput?.sourceFlow?.value as? IPreviewableSource + val videoSource = videoInput.sourceFlow.value as? IPreviewableSource ?: throw IllegalStateException("Video source is not previewable") return videoSource.setPreview(viewfinder, previewSize) } @@ -35,7 +35,7 @@ suspend fun IWithVideoSource.startPreview( viewfinder: CameraViewfinder, previewSize: Size ): ViewfinderSurfaceRequest { - val videoSource = videoInput?.sourceFlow?.value as? IPreviewableSource + val videoSource = videoInput.sourceFlow.value as? IPreviewableSource ?: throw IllegalStateException("Video source is not previewable") return videoSource.startPreview(viewfinder, previewSize) } From ac9ce6da3c4a3028ec192a418128101eae3c9297 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 4 Feb 2026 11:09:41 +0100 Subject: [PATCH 26/72] feat(core): make DualStreamer inputs non nullable --- .../dual/file/CameraDualStreamerFileTest.kt | 1 + .../core/streamers/dual/DualStreamer.kt | 333 ++++++------------ .../core/streamers/dual/DualStreamerImpl.kt | 283 +++++++++++++++ .../core/streamers/dual/IDualStreamer.kt | 4 +- .../streamers/dual/VideoOnlyDualStreamer.kt | 72 +++- .../services/utils/StreamerFactory.kt | 20 +- 6 files changed, 457 insertions(+), 256 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt index f8ce74d43..a148b8c03 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt @@ -28,6 +28,7 @@ import io.github.thibaultbee.streampack.core.streamer.dual.utils.DualStreamerCon import io.github.thibaultbee.streampack.core.streamer.utils.StreamerUtils import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.dual.cameraDualStreamer +import io.github.thibaultbee.streampack.core.streamers.dual.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt index 72c92ffd0..e889d3aeb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt @@ -36,26 +36,15 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation -import io.github.thibaultbee.streampack.core.elements.utils.extensions.isCompatibleWith import io.github.thibaultbee.streampack.core.interfaces.setCameraId import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioOutputMode -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioVideoEncodingPipelineOutput -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal -import io.github.thibaultbee.streampack.core.pipelines.utils.MultiThrowable +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.cancel -import kotlinx.coroutines.flow.SharingStarted -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.combineTransform -import kotlinx.coroutines.flow.merge -import kotlinx.coroutines.flow.stateIn -import kotlinx.coroutines.runBlocking /** * Creates a [DualStreamer] with a default audio source. @@ -66,6 +55,8 @@ import kotlinx.coroutines.runBlocking * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraDualStreamer( @@ -74,19 +65,22 @@ suspend fun cameraDualStreamer( audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( - context, - withAudio = true, - withVideo = true, - firstEndpointFactory, - secondEndpointFactory, - defaultRotation + context = context, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setCameraId(cameraId) if (audioSourceFactory != null) { - streamer.audioInput!!.setSource(audioSourceFactory) + streamer.setAudioSource(audioSourceFactory) } return streamer } @@ -99,6 +93,8 @@ suspend fun cameraDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresApi(Build.VERSION_CODES.Q) suspend fun audioVideoMediaProjectionDualStreamer( @@ -106,18 +102,20 @@ suspend fun audioVideoMediaProjectionDualStreamer( mediaProjection: MediaProjection, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - streamer.videoInput!!.setSource(MediaProjectionVideoSourceFactory(mediaProjection)) + streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) streamer.setAudioSource(MediaProjectionAudioSourceFactory(mediaProjection)) return streamer } @@ -131,6 +129,8 @@ suspend fun audioVideoMediaProjectionDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionDualStreamer( context: Context, @@ -138,16 +138,19 @@ suspend fun videoMediaProjectionDualStreamer( audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) if (audioSourceFactory != null) { streamer.setAudioSource(audioSourceFactory) @@ -164,6 +167,8 @@ suspend fun videoMediaProjectionDualStreamer( * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun DualStreamer( context: Context, @@ -171,16 +176,19 @@ suspend fun DualStreamer( videoSourceFactory: IVideoSourceInternal.Factory, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, - withAudio = true, - withVideo = true, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setAudioSource(audioSourceFactory) streamer.setVideoSource(videoSourceFactory) return streamer @@ -192,238 +200,93 @@ suspend fun DualStreamer( * For example, you can use it to live stream and record simultaneously. * * @param context the application context - * @param withAudio `true` to capture audio. It can't be changed after instantiation. - * @param withVideo `true` to capture video. It can't be changed after instantiation. * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ -open class DualStreamer( - protected val context: Context, - val withAudio: Boolean = true, - val withVideo: Boolean = true, +class DualStreamer( + context: Context, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), - dispatcherProvider: IDispatcherProvider = DispatcherProvider(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : IDualStreamer, IAudioDualStreamer, IVideoDualStreamer { - private val coroutineScope = CoroutineScope(dispatcherProvider.default) - - private val pipeline = StreamerPipeline( - context, - withAudio, - withVideo, - AudioOutputMode.PUSH, - surfaceProcessorFactory, - dispatcherProvider - ) - - private val firstPipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, withVideo, firstEndpointFactory, defaultRotation - ) as IEncodingPipelineOutputInternal - } - - - /** - * First output of the streamer. - */ - override val first = firstPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput - - private val secondPipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, withVideo, secondEndpointFactory, defaultRotation - ) as IEncodingPipelineOutputInternal - } - - /** - * Second output of the streamer. - */ - override val second = secondPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput - - override val throwableFlow: StateFlow = merge( - pipeline.throwableFlow, - firstPipelineOutput.throwableFlow, - secondPipelineOutput.throwableFlow - ).stateIn( - coroutineScope, - SharingStarted.Eagerly, - null - ) - - /** - * Whether any of the output is opening. - */ - override val isOpenFlow: StateFlow = combineTransform( - firstPipelineOutput.isOpenFlow, secondPipelineOutput.isOpenFlow - ) { isOpens -> - emit(isOpens.any { it }) - }.stateIn( - coroutineScope, - SharingStarted.Eagerly, - false + private val streamer = DualStreamerImpl( + context = context, + withAudio = true, + withVideo = true, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - /** - * Whether any of the output is streaming. - */ - override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow - - /** - * Closes the outputs. - * Same as calling [first.close] and [second.close]. - */ - override suspend fun close() { - firstPipelineOutput.close() - secondPipelineOutput.close() - } + override val first = streamer.first as IConfigurableVideoEncodingPipelineOutput + override val second = streamer.second as IConfigurableVideoEncodingPipelineOutput - // SOURCES - override val audioInput = pipeline.audioInput + override val throwableFlow = streamer.throwableFlow + override val isOpenFlow = streamer.isOpenFlow + override val isStreamingFlow = streamer.isStreamingFlow - // PROCESSORS - override val videoInput = pipeline.videoInput + override val audioInput: IAudioInput = streamer.audioInput + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. * * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) */ - override suspend fun setTargetRotation(@RotationValue rotation: Int) { - pipeline.setTargetRotation(rotation) - } + override suspend fun setTargetRotation(@RotationValue rotation: Int) = + streamer.setTargetRotation(rotation) - /** - * Sets audio configuration. - * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig]. - * - * @param audioConfig the audio configuration to set - */ @RequiresPermission(Manifest.permission.RECORD_AUDIO) - override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) { - val throwables = mutableListOf() + override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) = + streamer.setAudioConfig(audioConfig) - val firstAudioCodecConfig = firstPipelineOutput.audioCodecConfigFlow.value - if ((firstAudioCodecConfig != null) && (!firstAudioCodecConfig.isCompatibleWith(audioConfig.firstAudioConfig))) { - firstPipelineOutput.invalidateAudioCodecConfig() - } - val secondAudioCodecConfig = secondPipelineOutput.audioCodecConfigFlow.value - if ((secondAudioCodecConfig != null) && (!secondAudioCodecConfig.isCompatibleWith( - audioConfig.secondAudioConfig - )) - ) { - secondPipelineOutput.invalidateAudioCodecConfig() - } + override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) = + streamer.setVideoConfig(videoConfig) - try { - firstPipelineOutput.setAudioCodecConfig(audioConfig.firstAudioConfig) - } catch (t: Throwable) { - throwables += t - } - try { - audioConfig.secondAudioConfig.let { secondPipelineOutput.setAudioCodecConfig(it) } - } catch (t: Throwable) { - throwables += t - } - if (throwables.isNotEmpty()) { - if (throwables.size == 1) { - throw throwables.first() - } else { - throw MultiThrowable(throwables) - } - } - } + override suspend fun close() = streamer.close() - /** - * Sets video configuration. - * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig]. - * To only set video configuration for a specific output, use [first.setVideoCodecConfig] or - * [second.setVideoCodecConfig] outputs. - * In that case, you call [first.setVideoCodecConfig] or [second.setVideoCodecConfig] explicitly, - * make sure that the frame rate for both configurations is the same. - * - * @param videoConfig the video configuration to set - */ - override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) { - val throwables = mutableListOf() - - val firstVideoCodecConfig = firstPipelineOutput.videoCodecConfigFlow.value - if ((firstVideoCodecConfig != null) && (!firstVideoCodecConfig.isCompatibleWith(videoConfig.firstVideoConfig))) { - firstPipelineOutput.invalidateVideoCodecConfig() - } + override suspend fun startStream() = streamer.startStream() - val secondVideoCodecConfig = secondPipelineOutput.videoCodecConfigFlow.value - if ((secondVideoCodecConfig != null) && (!secondVideoCodecConfig.isCompatibleWith( - videoConfig.secondVideoConfig - )) - ) { - secondPipelineOutput.invalidateVideoCodecConfig() - } - - try { - firstPipelineOutput.setVideoCodecConfig(videoConfig.firstVideoConfig) - } catch (t: Throwable) { - throwables += t - } - try { - secondPipelineOutput.setVideoCodecConfig(videoConfig.secondVideoConfig) - } catch (t: Throwable) { - throwables += t - } - if (throwables.isNotEmpty()) { - if (throwables.size == 1) { - throw throwables.first() - } else { - throw MultiThrowable(throwables) - } - } - } + override suspend fun stopStream() = streamer.stopStream() - /** - * Configures both video and audio settings. - * - * It must be call when both stream and audio and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param audioConfig Audio configuration to set - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - * @see [DualStreamer.release] - */ - @RequiresPermission(Manifest.permission.RECORD_AUDIO) - suspend fun setConfig( - audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig - ) { - setAudioConfig(audioConfig) - setVideoConfig(videoConfig) - } - - - override suspend fun startStream() = pipeline.startStream() - - override suspend fun stopStream() = pipeline.stopStream() + override suspend fun release() = streamer.release() +} - override suspend fun release() { - pipeline.release() - coroutineScope.cancel() - } +/** + * Configures both video and audio settings. + * + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [DualStreamer.release] + */ +@RequiresPermission(Manifest.permission.RECORD_AUDIO) +suspend fun DualStreamer.setConfig( + audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig +) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) } /** * Sets audio configuration. * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig] when both + * It is a shortcut for [DualStreamer.setVideoConfig] when both * outputs use the same audio configuration. * * @param audioConfig the audio configuration to set @@ -436,7 +299,7 @@ suspend fun DualStreamer.setAudioConfig(audioConfig: AudioConfig) { /** * Sets video configuration. * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig] when both + * It is a shortcut for [DualStreamer.setVideoConfig] when both * outputs use the same video configuration. * * @param videoConfig the video configuration to set diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt new file mode 100644 index 000000000..9455abd3f --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt @@ -0,0 +1,283 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.streamers.dual + +import android.Manifest +import android.content.Context +import android.view.Surface +import androidx.annotation.RequiresPermission +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal.Factory +import io.github.thibaultbee.streampack.core.elements.utils.RotationValue +import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation +import io.github.thibaultbee.streampack.core.elements.utils.extensions.isCompatibleWith +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioOutputMode +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioVideoEncodingPipelineOutput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.pipelines.utils.MultiThrowable +import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.combineTransform +import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking + + +/** + * A class that handles 2 outputs. + * + * For example, you can use it to live stream and record simultaneously. + * + * @param context the application context + * @param withAudio `true` to capture audio. It can't be changed after instantiation. + * @param withVideo `true` to capture video. It can't be changed after instantiation. + * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. + * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. + */ +internal class DualStreamerImpl( + private val context: Context, + withAudio: Boolean = true, + withVideo: Boolean = true, + firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider(), +) : IDualStreamer, IAudioDualStreamer, IVideoDualStreamer { + private val coroutineScope = CoroutineScope(dispatcherProvider.default) + + private val pipeline = StreamerPipeline( + context, + withAudio, + withVideo, + AudioOutputMode.PUSH, + surfaceProcessorFactory, + dispatcherProvider + ) + + private val firstPipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, withVideo, firstEndpointFactory, defaultRotation + ) as IEncodingPipelineOutputInternal + } + + + /** + * First output of the streamer. + */ + override val first = firstPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput + + private val secondPipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, withVideo, secondEndpointFactory, defaultRotation + ) as IEncodingPipelineOutputInternal + } + + /** + * Second output of the streamer. + */ + override val second = secondPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput + + override val throwableFlow: StateFlow = merge( + pipeline.throwableFlow, + firstPipelineOutput.throwableFlow, + secondPipelineOutput.throwableFlow + ).stateIn( + coroutineScope, + SharingStarted.Eagerly, + null + ) + + /** + * Whether any of the output is opening. + */ + override val isOpenFlow: StateFlow = combineTransform( + firstPipelineOutput.isOpenFlow, secondPipelineOutput.isOpenFlow + ) { isOpens -> + emit(isOpens.any { it }) + }.stateIn( + coroutineScope, + SharingStarted.Eagerly, + false + ) + + /** + * Whether any of the output is streaming. + */ + override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + + /** + * Closes the outputs. + * Same as calling [first.close] and [second.close]. + */ + override suspend fun close() { + firstPipelineOutput.close() + secondPipelineOutput.close() + } + + // SOURCES + override val audioInput = pipeline.audioInput + + // PROCESSORS + override val videoInput = pipeline.videoInput + + /** + * Sets the target rotation. + * + * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) + */ + override suspend fun setTargetRotation(@RotationValue rotation: Int) { + pipeline.setTargetRotation(rotation) + } + + /** + * Sets audio configuration. + * + * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig]. + * + * @param audioConfig the audio configuration to set + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) { + val throwables = mutableListOf() + + val firstAudioCodecConfig = firstPipelineOutput.audioCodecConfigFlow.value + if ((firstAudioCodecConfig != null) && (!firstAudioCodecConfig.isCompatibleWith(audioConfig.firstAudioConfig))) { + firstPipelineOutput.invalidateAudioCodecConfig() + } + val secondAudioCodecConfig = secondPipelineOutput.audioCodecConfigFlow.value + if ((secondAudioCodecConfig != null) && (!secondAudioCodecConfig.isCompatibleWith( + audioConfig.secondAudioConfig + )) + ) { + secondPipelineOutput.invalidateAudioCodecConfig() + } + + try { + firstPipelineOutput.setAudioCodecConfig(audioConfig.firstAudioConfig) + } catch (t: Throwable) { + throwables += t + } + try { + audioConfig.secondAudioConfig.let { secondPipelineOutput.setAudioCodecConfig(it) } + } catch (t: Throwable) { + throwables += t + } + if (throwables.isNotEmpty()) { + if (throwables.size == 1) { + throw throwables.first() + } else { + throw MultiThrowable(throwables) + } + } + } + + /** + * Sets video configuration. + * + * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig]. + * To only set video configuration for a specific output, use [first.setVideoCodecConfig] or + * [second.setVideoCodecConfig] outputs. + * In that case, you call [first.setVideoCodecConfig] or [second.setVideoCodecConfig] explicitly, + * make sure that the frame rate for both configurations is the same. + * + * @param videoConfig the video configuration to set + */ + override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) { + val throwables = mutableListOf() + + val firstVideoCodecConfig = firstPipelineOutput.videoCodecConfigFlow.value + if ((firstVideoCodecConfig != null) && (!firstVideoCodecConfig.isCompatibleWith(videoConfig.firstVideoConfig))) { + firstPipelineOutput.invalidateVideoCodecConfig() + } + + val secondVideoCodecConfig = secondPipelineOutput.videoCodecConfigFlow.value + if ((secondVideoCodecConfig != null) && (!secondVideoCodecConfig.isCompatibleWith( + videoConfig.secondVideoConfig + )) + ) { + secondPipelineOutput.invalidateVideoCodecConfig() + } + + try { + firstPipelineOutput.setVideoCodecConfig(videoConfig.firstVideoConfig) + } catch (t: Throwable) { + throwables += t + } + try { + secondPipelineOutput.setVideoCodecConfig(videoConfig.secondVideoConfig) + } catch (t: Throwable) { + throwables += t + } + if (throwables.isNotEmpty()) { + if (throwables.size == 1) { + throw throwables.first() + } else { + throw MultiThrowable(throwables) + } + } + } + + /** + * Configures both video and audio settings. + * + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [DualStreamer.release] + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + suspend fun setConfig( + audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig + ) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) + } + + + override suspend fun startStream() = pipeline.startStream() + + override suspend fun stopStream() = pipeline.stopStream() + + override suspend fun release() { + pipeline.release() + coroutineScope.cancel() + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt index 979ffe9a8..b98472b80 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt @@ -243,9 +243,9 @@ internal constructor( val dynamicRangeProfile = firstVideoConfig.dynamicRangeProfile } -interface IAudioDualStreamer : IAudioStreamer +interface IAudioDualStreamer : IAudioStreamer, IDualStreamer -interface IVideoDualStreamer : IVideoStreamer +interface IVideoDualStreamer : IVideoStreamer, IDualStreamer interface IDualStreamer : ICloseableStreamer { /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt index 31b89d58d..b4664f6b7 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt @@ -23,14 +23,20 @@ import android.view.Surface import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal.Factory import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.setCameraId +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput +import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig /** @@ -41,6 +47,8 @@ import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigu * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraVideoOnlyDualStreamer( @@ -48,10 +56,17 @@ suspend fun cameraVideoOnlyDualStreamer( cameraId: String = context.defaultCameraId, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( - context, firstEndpointFactory, secondEndpointFactory, defaultRotation + context = context, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) return streamer @@ -65,19 +80,25 @@ suspend fun cameraVideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionVideoOnlyDualStreamer( context: Context, mediaProjection: MediaProjection, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) return streamer @@ -91,19 +112,25 @@ suspend fun videoMediaProjectionVideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun VideoOnlyDualStreamer( context: Context, videoSourceFactory: IVideoSourceInternal.Factory, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(videoSourceFactory) return streamer @@ -116,20 +143,26 @@ suspend fun VideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class VideoOnlyDualStreamer( context: Context, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : IDualStreamer, IVideoDualStreamer { - private val streamer = DualStreamer( + private val streamer = DualStreamerImpl( context = context, - firstEndpointFactory = firstEndpointFactory, - secondEndpointFactory = secondEndpointFactory, withAudio = false, withVideo = true, - defaultRotation = defaultRotation + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) override val first = streamer.first as IConfigurableVideoEncodingPipelineOutput @@ -139,7 +172,7 @@ class VideoOnlyDualStreamer( override val isOpenFlow = streamer.isOpenFlow override val isStreamingFlow = streamer.isStreamingFlow - override val videoInput: IVideoInput = streamer.videoInput!! + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. @@ -159,4 +192,17 @@ class VideoOnlyDualStreamer( override suspend fun stopStream() = streamer.stopStream() override suspend fun release() = streamer.release() -} \ No newline at end of file +} + +/** + * Sets video configuration. + * + * It is a shortcut for [VideoOnlyDualStreamer.setVideoConfig] when both + * outputs use the same video configuration. + * + * @param videoConfig the video configuration to set + */ +suspend fun VideoOnlyDualStreamer.setVideoConfig(videoConfig: VideoConfig) { + setVideoConfig(DualStreamerVideoConfig(videoConfig)) +} + diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt index 78e08fbab..d9866e2e4 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt @@ -22,6 +22,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRo import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.streamers.dual.DualStreamer import io.github.thibaultbee.streampack.core.streamers.dual.IDualStreamer +import io.github.thibaultbee.streampack.core.streamers.dual.VideoOnlyDualStreamer import io.github.thibaultbee.streampack.core.streamers.single.AudioOnlySingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.ISingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer @@ -86,11 +87,18 @@ open class DualStreamerFactory( ) : StreamerFactory { override fun create(context: Context): IDualStreamer { - return DualStreamer( - context, - withAudio, - withVideo, - defaultRotation = defaultRotation ?: context.displayRotation - ) + return if (withAudio && withVideo) { + DualStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else if (withVideo) { + VideoOnlyDualStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else { + throw IllegalArgumentException("DualStreamer audio only is not supported yet") + } } } \ No newline at end of file From 24f5d10e55dfb11e4bbc33b04f5703d772b54a61 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 4 Feb 2026 14:30:46 +0100 Subject: [PATCH 27/72] chore(version):bump to 3.2.0 snapshot --- build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index f5a6323b9..da27d7574 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,8 +5,8 @@ plugins { } allprojects { - val versionCode by extra { 3_001_002 } - val versionName by extra { "3.1.2" } + val versionCode by extra { 3_002_000 } + val versionName by extra { "3.2.0" } group = "io.github.thibaultbee.streampack" version = versionName From 2efcc08712f3062dee89bef4525e85a4e227b5f6 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 9 Feb 2026 09:45:41 +0100 Subject: [PATCH 28/72] fix(ui): fix former comments on startPreview --- .../github/thibaultbee/streampack/ui/views/PreviewView.kt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt index 50da71a1b..40bd147a9 100644 --- a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt @@ -373,9 +373,7 @@ class PreviewView @JvmOverloads constructor( } /** - * Requests a [Surface] for the size and the current streamer video source. - * - * The [Surface] is emit to the [surfaceFlow]. + * Starts the preview for the given [size]. */ private fun startPreview(size: Size) { Logger.d(TAG, "Requesting surface for $size") @@ -388,9 +386,7 @@ class PreviewView @JvmOverloads constructor( } /** - * Requests a [Surface] for the size and the [videoSource]. - * - * The [Surface] is emit to the [surfaceFlow]. + * Starts the preview for the given [size] and [videoSource]. */ private fun startPreview( size: Size, From fd0f88b7267cec5cf46cbf13bfc26bb813169290 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 9 Feb 2026 09:46:07 +0100 Subject: [PATCH 29/72] refactor(compose): rename PreviewView and add a zoom callback --- .../{PreviewView.kt => SourcePreview.kt} | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) rename ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/{PreviewView.kt => SourcePreview.kt} (82%) diff --git a/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/SourcePreview.kt similarity index 82% rename from ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt rename to ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/SourcePreview.kt index 379649bb7..aa2e1035d 100644 --- a/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt +++ b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/SourcePreview.kt @@ -26,6 +26,7 @@ import androidx.compose.ui.viewinterop.AndroidView import io.github.thibaultbee.streampack.compose.utils.BitmapUtils import io.github.thibaultbee.streampack.core.elements.sources.video.IPreviewableSource import io.github.thibaultbee.streampack.core.elements.sources.video.bitmap.BitmapSourceFactory +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings.FocusMetering.Companion.DEFAULT_AUTO_CANCEL_DURATION_MS import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource import io.github.thibaultbee.streampack.core.logger.Logger @@ -34,7 +35,7 @@ import io.github.thibaultbee.streampack.ui.views.PreviewView import kotlinx.coroutines.launch import kotlinx.coroutines.sync.withLock -private const val TAG = "ComposePreviewView" +private const val TAG = "ComposeSourcePreview" /** * Displays the preview of a [IWithVideoSource]. @@ -48,9 +49,10 @@ private const val TAG = "ComposePreviewView" * @param onTapToFocusTimeoutMs the duration in milliseconds after which the focus area set by tap-to-focus is cleared */ @Composable -fun PreviewScreen( +fun SourcePreview( videoSource: IWithVideoSource, modifier: Modifier = Modifier, + onZoomChanged: ((zoomRatio: Float) -> Unit)? = null, enableZoomOnPinch: Boolean = true, enableTapToFocus: Boolean = true, onTapToFocusTimeoutMs: Long = DEFAULT_AUTO_CANCEL_DURATION_MS @@ -63,6 +65,15 @@ fun PreviewScreen( this.enableZoomOnPinch = enableZoomOnPinch this.enableTapToFocus = enableTapToFocus this.onTapToFocusTimeoutMs = onTapToFocusTimeoutMs + onZoomChanged?.let { + val onZoomChangedListener = object : CameraSettings.Zoom.OnZoomChangedListener { + override fun onZoomChanged(zoomRatio: Float) { + onZoomChanged(zoomRatio) + } + } + this.setZoomListener(onZoomChangedListener) + } + scope.launch { try { @@ -76,7 +87,7 @@ fun PreviewScreen( modifier = modifier, onRelease = { scope.launch { - val source = videoSource.videoInput?.sourceFlow?.value as? IPreviewableSource + val source = videoSource.videoInput.sourceFlow.value as? IPreviewableSource source?.previewMutex?.withLock { source.stopPreview() source.resetPreview() @@ -87,7 +98,7 @@ fun PreviewScreen( @Preview @Composable -fun PreviewScreenPreview() { +fun PreviewScreenSourcePreview() { val context = LocalContext.current val streamer = SingleStreamer(context) LaunchedEffect(Unit) { @@ -101,5 +112,5 @@ fun PreviewScreenPreview() { ) } - PreviewScreen(streamer, modifier = Modifier.fillMaxSize()) + SourcePreview(streamer, modifier = Modifier.fillMaxSize()) } \ No newline at end of file From 97965c8d2e5af4e5edae5916a8c0bb43f4ec9b8e Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 9 Feb 2026 14:33:59 +0100 Subject: [PATCH 30/72] docs(README.md): add compose package info --- README.md | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index fe37c3e69..21ddf2a2a 100644 --- a/README.md +++ b/README.md @@ -17,15 +17,17 @@ Get StreamPack core latest artifacts on Maven Central: ```groovy dependencies { - implementation 'io.github.thibaultbee.streampack:streampack-core:3.1.2' - // For UI (incl. PreviewView) - implementation 'io.github.thibaultbee.streampack:streampack-ui:3.1.2' + implementation 'io.github.thibaultbee.streampack:streampack-core:3.2.0' + // For xml UI (incl. PreviewView) + implementation 'io.github.thibaultbee.streampack:streampack-ui:3.2.0' + // Or compose UI (incl. SourcePreview) + implementation 'io.github.thibaultbee.streampack:streampack-compose:3.2.0' // For services (incl. screen capture/media projection service) - implementation 'io.github.thibaultbee.streampack:streampack-services:3.1.2' + implementation 'io.github.thibaultbee.streampack:streampack-services:3.2.0' // For RTMP - implementation 'io.github.thibaultbee.streampack:streampack-rtmp:3.1.2' + implementation 'io.github.thibaultbee.streampack:streampack-rtmp:3.2.0' // For SRT - implementation 'io.github.thibaultbee.streampack:streampack-srt:3.1.2' + implementation 'io.github.thibaultbee.streampack:streampack-srt:3.2.0' } ``` @@ -159,6 +161,8 @@ minutes, you will be able to stream live video to your server. 5. Inflates the preview with the streamer + Either `xml` UI + ```kotlin val streamer = cameraSingleStreamer(context = requireContext()) // Already instantiated streamer val preview = findViewById(R.id.preview) // Already inflated preview @@ -175,6 +179,16 @@ minutes, you will be able to stream live video to your server. streamer.startPreview(preview) ``` + Or Compose UI + + ```kotlin + val streamer = cameraSingleStreamer(context = requireContext()) // Already instantiated streamer + + Box { + SourcePreview(streamer, modifier = Modifier.fillMaxSize()) + } + ``` + 6. Sets the device orientation ```kotlin From 601cff3fc8c21ca9fd332ad06a2492f67d972c3a Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 10 Feb 2026 15:58:10 +0100 Subject: [PATCH 31/72] feat(core): camera: in settings, add `totalCaptureResultFlow` instead of `applyRepeatingSession` --- .../sources/video/camera/CameraSettings.kt | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 91fc4f7b9..90ccdb8ee 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -69,6 +69,7 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.flow.conflate import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -93,11 +94,18 @@ class CameraSettings internal constructor( */ val cameraId = cameraController.cameraId - @RequiresApi(Build.VERSION_CODES.Q) - private fun getPhysicalCameraIdCallbackFlow() = callbackFlow { + /** + * Current physical camera id. + */ + val physicalCameraIdFlow: Flow + @RequiresApi(Build.VERSION_CODES.Q) + get() = getTotalCaptureResultCallbackFlow().map { it[CaptureResult.LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_ID]!! } + .distinctUntilChanged() + + private fun getTotalCaptureResultCallbackFlow() = callbackFlow { val captureCallback = object : CaptureResultListener { override fun onCaptureResult(result: TotalCaptureResult): Boolean { - trySend(result.get(CaptureResult.LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_ID)!!) + trySend(result) return false } } @@ -107,14 +115,13 @@ class CameraSettings internal constructor( cameraController.removeCaptureCallbackListener(captureCallback) } } - }.conflate().distinctUntilChanged() + }.conflate() /** - * Current physical camera id. + * The total capture result flow. */ - val physicalCameraIdFlow: Flow - @RequiresApi(Build.VERSION_CODES.Q) - get() = getPhysicalCameraIdCallbackFlow() + val totalCaptureResultFlow: Flow + get() = getTotalCaptureResultCallbackFlow() /** * Whether the camera is available. @@ -212,7 +219,7 @@ class CameraSettings internal constructor( * * @param onCaptureResult the capture result callback. Return `true` to stop the callback. */ - suspend fun applyRepeatingSession(onCaptureResult: CaptureResultListener) { + private suspend fun applyRepeatingSession(onCaptureResult: CaptureResultListener) { val tag = TagBundle.Factory.default.create() val captureCallback = object : CaptureResultListener { override fun onCaptureResult(result: TotalCaptureResult): Boolean { From 540f13b7b8fbe0242f1f7c7ee4c9f62a8b84d46f Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 10 Feb 2026 16:33:21 +0100 Subject: [PATCH 32/72] fix(core): camera: in settings, make the coroutine scope not accessible --- .../core/elements/sources/video/camera/CameraSettings.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 90ccdb8ee..02a093db9 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -85,7 +85,7 @@ import java.util.concurrent.atomic.AtomicLong * @param characteristics Camera characteristics of the current camera. */ class CameraSettings internal constructor( - val coroutineScope: CoroutineScope, + coroutineScope: CoroutineScope, val characteristics: CameraCharacteristics, private val cameraController: CameraController ) { From 6d3ada180a0095ed0890fe1c6796b24a9fc29d3d Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 10 Feb 2026 17:04:45 +0100 Subject: [PATCH 33/72] fix(core): camera: in settings, call awaitClose from the coroutine scope --- .../elements/sources/video/camera/CameraSettings.kt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 02a093db9..516dadbc6 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -70,7 +70,7 @@ import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.flow.conflate import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.map -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.CancellationException @@ -85,7 +85,7 @@ import java.util.concurrent.atomic.AtomicLong * @param characteristics Camera characteristics of the current camera. */ class CameraSettings internal constructor( - coroutineScope: CoroutineScope, + private val coroutineScope: CoroutineScope, val characteristics: CameraCharacteristics, private val cameraController: CameraController ) { @@ -111,8 +111,12 @@ class CameraSettings internal constructor( } cameraController.addCaptureCallbackListener(captureCallback) awaitClose { - runBlocking { - cameraController.removeCaptureCallbackListener(captureCallback) + coroutineScope.launch { + try { + cameraController.removeCaptureCallbackListener(captureCallback) + } catch (e: Exception) { + Logger.w(TAG, "Failed to remove capture callback listener on close", e) + } } } }.conflate() From 224ff5d6a3118f86cbcd130b9b8e4e9726e5e947 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 10 Feb 2026 19:42:47 +0100 Subject: [PATCH 34/72] refactor(core): camera: in settings, use a shared flow to listen to callback flow --- .../sources/video/camera/CameraSettings.kt | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 516dadbc6..be8e3f567 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -66,10 +66,11 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.channels.awaitClose import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.callbackFlow -import kotlinx.coroutines.flow.conflate import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.shareIn import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -97,35 +98,37 @@ class CameraSettings internal constructor( /** * Current physical camera id. */ - val physicalCameraIdFlow: Flow - @RequiresApi(Build.VERSION_CODES.Q) - get() = getTotalCaptureResultCallbackFlow().map { it[CaptureResult.LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_ID]!! } + @delegate:RequiresApi(Build.VERSION_CODES.Q) + val physicalCameraIdFlow: Flow by lazy { + captureResultSharedFlow.map { it[CaptureResult.LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_ID]!! } .distinctUntilChanged() + } - private fun getTotalCaptureResultCallbackFlow() = callbackFlow { - val captureCallback = object : CaptureResultListener { - override fun onCaptureResult(result: TotalCaptureResult): Boolean { - trySend(result) - return false + private val captureResultSharedFlow by lazy { + callbackFlow { + val captureCallback = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + trySend(result) + return false + } } - } - cameraController.addCaptureCallbackListener(captureCallback) - awaitClose { - coroutineScope.launch { - try { - cameraController.removeCaptureCallbackListener(captureCallback) - } catch (e: Exception) { - Logger.w(TAG, "Failed to remove capture callback listener on close", e) + cameraController.addCaptureCallbackListener(captureCallback) + awaitClose { + coroutineScope.launch { + try { + cameraController.removeCaptureCallbackListener(captureCallback) + } catch (e: Exception) { + Logger.w(TAG, "Failed to remove capture callback listener on close", e) + } } } - } - }.conflate() + }.shareIn(coroutineScope, SharingStarted.WhileSubscribed(), 1) + } /** * The total capture result flow. */ - val totalCaptureResultFlow: Flow - get() = getTotalCaptureResultCallbackFlow() + val captureResultFlow: Flow by lazy { captureResultSharedFlow } /** * Whether the camera is available. From 1be661df72343e4185be3507eb686cb14c5171bc Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 11 Feb 2026 14:03:08 +0100 Subject: [PATCH 35/72] refactor(core): camera: in settings, add `isLockAvailable` API --- .../core/elements/sources/video/camera/CameraSettings.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index be8e3f567..b9343d545 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -668,6 +668,10 @@ class CameraSettings internal constructor( cameraSettings.applyRepeatingSession() } + + @delegate:RequiresApi(Build.VERSION_CODES.M) + val isLockAvailable: Boolean by lazy { characteristics[CameraCharacteristics.CONTROL_AE_LOCK_AVAILABLE] == true } + /** * Gets auto exposure lock. * From a630e48749399ae4b77f8076eedf1b1fac68b05f Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 11 Feb 2026 15:11:11 +0100 Subject: [PATCH 36/72] feat(core): camera: in settings, add `defaultStrengthLevel` API --- .../sources/video/camera/CameraSettings.kt | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index b9343d545..78f4f97e5 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -345,10 +345,22 @@ class CameraSettings internal constructor( val availableStrengthLevelRange: Range by lazy { Range( 1, - characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] ?: 1 + characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] ?: DEFAULT_STRENGTH_LEVEL ) } + /** + * Gets the default flash strength. + * Range is from [availableStrengthLevelRange]. + * + * Use the range to call [setStrengthLevel] + */ + @delegate:RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + val defaultStrengthLevel: Int by lazy { + characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_DEFAULT_LEVEL] + ?: DEFAULT_STRENGTH_LEVEL + } + val strengthLevel: Int /** * Gets the flash strength. @@ -356,7 +368,8 @@ class CameraSettings internal constructor( * @return the flash strength */ @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) - get() = cameraSettings.get(CaptureRequest.FLASH_STRENGTH_LEVEL) ?: 1 + get() = cameraSettings.get(CaptureRequest.FLASH_STRENGTH_LEVEL) + ?: DEFAULT_STRENGTH_LEVEL /** * Sets the flash strength. @@ -369,6 +382,10 @@ class CameraSettings internal constructor( cameraSettings.set(CaptureRequest.FLASH_STRENGTH_LEVEL, level) cameraSettings.applyRepeatingSession() } + + companion object { + private const val DEFAULT_STRENGTH_LEVEL = 1 + } } class WhiteBalance internal constructor( From 02bb4918288752e2c693eccbe5d1c921e95ec42d Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Thu, 19 Feb 2026 13:35:39 +0100 Subject: [PATCH 37/72] refactor(core): move snapshot API to a specific interface --- .../core/elements/interfaces/Snapshotable.kt | 79 ++++++++++++ .../video/DefaultSurfaceProcessor.kt | 27 ++++- .../processing/video/ISurfaceProcessor.kt | 2 - .../core/pipelines/inputs/VideoInput.kt | 112 ++++-------------- 4 files changed, 130 insertions(+), 90 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/interfaces/Snapshotable.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/interfaces/Snapshotable.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/interfaces/Snapshotable.kt new file mode 100644 index 000000000..168efe8f5 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/interfaces/Snapshotable.kt @@ -0,0 +1,79 @@ +package io.github.thibaultbee.streampack.core.elements.interfaces + +import android.graphics.Bitmap +import androidx.annotation.IntRange +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.io.FileOutputStream +import java.io.OutputStream + +/** + * An interface to take a snapshot of the current video frame. + */ +interface ISnapshotable { + /** + * Takes a snapshot of the current video frame. + * + * The snapshot is returned as a [Bitmap]. + * + * @param rotationDegrees The rotation to apply to the snapshot, in degrees. 0 means no rotation. + * @return The snapshot as a [Bitmap]. + */ + suspend fun takeSnapshot(@IntRange(from = 0, to = 359) rotationDegrees: Int = 0): Bitmap +} + +/** + * Takes a JPEG snapshot of the current video frame. + * + * The snapshot is saved to the specified file. + * + * @param filePathString The path of the file to save the snapshot to. + * @param quality The quality of the JPEG, from 0 to 100. + * @param rotationDegrees The rotation to apply to the snapshot, in degrees. + */ +suspend fun ISnapshotable.takeJpegSnapshot( + filePathString: String, + @IntRange(from = 0, to = 100) quality: Int = 100, + @IntRange(from = 0, to = 359) rotationDegrees: Int = 0, + dispatcher: CoroutineDispatcher = Dispatchers.IO +) = takeJpegSnapshot(withContext(dispatcher) { + FileOutputStream(filePathString) +}, quality, rotationDegrees) + + +/** + * Takes a JPEG snapshot of the current video frame. + * + * The snapshot is saved to the specified file. + * + * @param file The file to save the snapshot to. + * @param quality The quality of the JPEG, from 0 to 100. + * @param rotationDegrees The rotation to apply to the snapshot, in degrees. + */ +suspend fun ISnapshotable.takeJpegSnapshot( + file: File, + @IntRange(from = 0, to = 100) quality: Int = 100, + @IntRange(from = 0, to = 359) rotationDegrees: Int = 0, + dispatcher: CoroutineDispatcher = Dispatchers.IO +) = takeJpegSnapshot(withContext(dispatcher) { + FileOutputStream(file) +}, quality, rotationDegrees) + +/** + * Takes a snapshot of the current video frame. + * + * The snapshot is saved as a JPEG to the specified output stream. + * @param outputStream The output stream to save the snapshot to. + * @param quality The quality of the JPEG, from 0 to 100. + * @param rotationDegrees The rotation to apply to the snapshot, in degrees. + */ +suspend fun ISnapshotable.takeJpegSnapshot( + outputStream: OutputStream, + @IntRange(from = 0, to = 100) quality: Int = 100, + @IntRange(from = 0, to = 359) rotationDegrees: Int = 0 +) { + val bitmap = takeSnapshot(rotationDegrees) + bitmap.compress(Bitmap.CompressFormat.JPEG, quality, outputStream) +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt index 5766ad546..392b3b764 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt @@ -23,6 +23,7 @@ import android.view.Surface import androidx.annotation.IntRange import androidx.concurrent.futures.CallbackToFutureAdapter import com.google.common.util.concurrent.ListenableFuture +import io.github.thibaultbee.streampack.core.elements.interfaces.ISnapshotable import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils @@ -38,12 +39,14 @@ import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Compan import io.github.thibaultbee.streampack.core.pipelines.IVideoDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.utils.HandlerThreadExecutor import java.util.concurrent.atomic.AtomicBoolean +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine private class DefaultSurfaceProcessor( private val dynamicRangeProfile: DynamicRangeProfile, private val glThread: HandlerThreadExecutor, -) : ISurfaceProcessorInternal, SurfaceTexture.OnFrameAvailableListener { +) : ISurfaceProcessorInternal, SurfaceTexture.OnFrameAvailableListener, ISnapshotable { private val renderer = OpenGlRenderer() private val glHandler = glThread.handler @@ -225,7 +228,27 @@ private class DefaultSurfaceProcessor( } } - override fun snapshot( + /** + * Takes a snapshot of the current video frame. + * + * The snapshot is returned as a [Bitmap]. + * + * @param rotationDegrees The rotation to apply to the snapshot, in degrees. 0 means no rotation. + * @return The snapshot as a [Bitmap]. + */ + override suspend fun takeSnapshot(rotationDegrees: Int): Bitmap { + return suspendCoroutine { continuation -> + val listener = snapshot(rotationDegrees) + try { + val bitmap = listener.get() + continuation.resume(bitmap) + } catch (e: Exception) { + continuation.resumeWith(Result.failure(e)) + } + } + } + + private fun snapshot( @IntRange(from = 0, to = 359) rotationDegrees: Int ): ListenableFuture { if (isReleaseRequested.get()) { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/ISurfaceProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/ISurfaceProcessor.kt index f017e1fa6..35d1c8fec 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/ISurfaceProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/ISurfaceProcessor.kt @@ -51,8 +51,6 @@ interface ISurfaceProcessorInternal : ISurfaceProcessor, Releasable { fun removeAllOutputSurfaces() - fun snapshot(@IntRange(from = 0, to = 359) rotationDegrees: Int): ListenableFuture - /** * Factory interface for creating instances of [ISurfaceProcessorInternal]. */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt index 54e7bfa3b..ff85574f4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt @@ -19,6 +19,7 @@ import android.content.Context import android.graphics.Bitmap import android.view.Surface import androidx.annotation.IntRange +import io.github.thibaultbee.streampack.core.elements.interfaces.ISnapshotable import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput @@ -45,18 +46,13 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.io.File -import java.io.FileOutputStream -import java.io.OutputStream import java.util.concurrent.atomic.AtomicBoolean -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine /** * The public interface for the video input. * It provides access to the video source, the video processor, and the streaming state. */ -interface IVideoInput { +interface IVideoInput : ISnapshotable { /** * Whether the video input is streaming. @@ -85,64 +81,6 @@ interface IVideoInput { * The video processor for adding effects to the video frames. */ val processor: ISurfaceProcessorInternal - - /** - * Takes a snapshot of the current video frame. - * - * The snapshot is returned as a [Bitmap]. - * - * @param rotationDegrees The rotation to apply to the snapshot, in degrees. 0 means no rotation. - * @return The snapshot as a [Bitmap]. - */ - suspend fun takeSnapshot(@IntRange(from = 0, to = 359) rotationDegrees: Int = 0): Bitmap -} - -/** - * Takes a JPEG snapshot of the current video frame. - * - * The snapshot is saved to the specified file. - * - * @param filePathString The path of the file to save the snapshot to. - * @param quality The quality of the JPEG, from 0 to 100. - * @param rotationDegrees The rotation to apply to the snapshot, in degrees. - */ -suspend fun IVideoInput.takeJpegSnapshot( - filePathString: String, - @IntRange(from = 0, to = 100) quality: Int = 100, - @IntRange(from = 0, to = 359) rotationDegrees: Int = 0 -) = takeJpegSnapshot(FileOutputStream(filePathString), quality, rotationDegrees) - - -/** - * Takes a JPEG snapshot of the current video frame. - * - * The snapshot is saved to the specified file. - * - * @param file The file to save the snapshot to. - * @param quality The quality of the JPEG, from 0 to 100. - * @param rotationDegrees The rotation to apply to the snapshot, in degrees. - */ -suspend fun IVideoInput.takeJpegSnapshot( - file: File, - @IntRange(from = 0, to = 100) quality: Int = 100, - @IntRange(from = 0, to = 359) rotationDegrees: Int = 0 -) = takeJpegSnapshot(FileOutputStream(file), quality, rotationDegrees) - -/** - * Takes a snapshot of the current video frame. - * - * The snapshot is saved as a JPEG to the specified output stream. - * @param outputStream The output stream to save the snapshot to. - * @param quality The quality of the JPEG, from 0 to 100. - * @param rotationDegrees The rotation to apply to the snapshot, in degrees. - */ -suspend fun IVideoInput.takeJpegSnapshot( - outputStream: OutputStream, - @IntRange(from = 0, to = 100) quality: Int = 100, - @IntRange(from = 0, to = 359) rotationDegrees: Int = 0 -) { - val bitmap = takeSnapshot(rotationDegrees) - bitmap.compress(Bitmap.CompressFormat.JPEG, quality, outputStream) } /** @@ -436,6 +374,13 @@ internal class VideoInput( return newSurfaceProcessor } + /** + * Takes a snapshot of the current video frame. + * + * It starts the video source if needed. + * + * The snapshot is returned as a [Bitmap]. + */ override suspend fun takeSnapshot( @IntRange( from = 0, @@ -445,14 +390,13 @@ internal class VideoInput( if (isReleaseRequested.get()) { throw IllegalStateException("Input is released") } - return startStreamForBlock { - suspendCoroutine { continuation -> - val listener = processor.snapshot(rotationDegrees) - try { - val bitmap = listener.get() - continuation.resume(bitmap) - } catch (e: Exception) { - continuation.resumeWith(Result.failure(e)) + return withContext(dispatcherProvider.default) { + sourceMutex.withLock { + val processor = processor as? ISnapshotable? + ?: throw IllegalStateException("Processor is not a snapshotable") + + startStreamForBlockUnsafe { + processor.takeSnapshot(rotationDegrees) } } } @@ -605,23 +549,19 @@ internal class VideoInput( * * If the stream was already running, it will not be stopped after the block. */ - private suspend fun startStreamForBlock(block: suspend () -> T): T { + private suspend fun startStreamForBlockUnsafe(block: suspend () -> T): T { if (isReleaseRequested.get()) { throw IllegalStateException("Input is released") } - return withContext(dispatcherProvider.default) { - sourceMutex.withLock { - val wasStreaming = isStreamingFlow.value - if (!wasStreaming) { - startStreamUnsafe() - } - try { - block() - } finally { - if (!wasStreaming) { - stopStreamUnsafe() - } - } + val wasStreaming = isStreamingFlow.value + if (!wasStreaming) { + startStreamUnsafe() + } + return try { + block() + } finally { + if (!wasStreaming) { + stopStreamUnsafe() } } } From 5cba6db43e4e6fb9419fd3020f8cbc79983f299a Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sat, 21 Feb 2026 14:33:06 +0100 Subject: [PATCH 38/72] docs(README.md): improve README.md header --- README.md | 96 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 21ddf2a2a..ab4a765c2 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,34 @@ -# StreamPack: RTMP and SRT live streaming SDK for Android +[![Maven Central](https://img.shields.io/maven-central/v/io.github.thibaultbee.streampack/streampack-core)](https://central.sonatype.com/artifact/io.github.thibaultbee.streampack/streampack-core) +[![License](https://img.shields.io/github/license/ThibaultBee/StreamPack)](LICENSE.md) +[![API](https://img.shields.io/badge/API-21%2B-brightgreen.svg)](https://android-arsenal.com/api?level=21) +[![Stars](https://img.shields.io/github/stars/ThibaultBee/StreamPack?style=social)](https://github.com/ThibaultBee/StreamPack/stargazers) -StreamPack is a flexible live streaming library for Android made for both demanding video -broadcasters and new video enthusiasts. +# StreamPack — Android Live Streaming SDK (RTMP, SRT, Camera2, Kotlin) -## Hop On Board! 🚀 +StreamPack is a modular, high-performance Android library designed for low-latency live streaming. +It empowers developers to build professional broadcasting applications with support for SRT, RTMP, +and RTMPS protocols. Leveraging the latest Android Camera2 and MediaCodec APIs, it is fully +extensible, allowing for the integration of custom protocols and custom audio/video sources. -⭐ If you like this project, don’t forget to star it! +## 🏗️ 5-minute Quick Start / Boilerplate -💖 Want to support its development? Consider becoming a sponsor. +If you want to create a new application, you should use the +template [StreamPack boilerplate](https://github.com/ThibaultBee/StreamPack-boilerplate). In 5 +minutes, you will be able to stream live video to your server. -🛠️ Contributions are welcome—feel free to open issues or submit pull requests! +## Table of Contents -## Setup +- [Setup](#-setup) +- [Features](#-features) +- [Quick Start](#-quick-start) +- [Permissions](#-permissions) +- [Documentation](#-documentations) +- [Demos](#-demos) +- [Contributing](#-contributing--support) +- [Tips](#-tips) +- [License](#-license) + +## 📦 Setup Get StreamPack core latest artifacts on Maven Central: @@ -31,7 +48,7 @@ dependencies { } ``` -## Features +## ✨ Features * Video: * Source: Cameras, Screen recorder @@ -61,20 +78,14 @@ dependencies { * Ultra low-latency based on [SRT](https://github.com/Haivision/srt) * Network adaptive bitrate mechanism for [SRT](https://github.com/Haivision/srt) -## Quick start - -If you want to create a new application, you should use the -template [StreamPack boilerplate](https://github.com/ThibaultBee/StreamPack-boilerplate). In 5 -minutes, you will be able to stream live video to your server. - -## Getting started +## 🚀 Quick Start -### Getting started for a camera stream +### 📷 Camera Stream 1. Request the required permissions in your Activity/Fragment. See the [Permissions](#permissions) section for more information. -2. Creates a `View` to display the preview in your layout +2. Create a `View` to display the preview in your layout As a camera preview, you can also use a `SurfaceView`, a `TextureView` or any `View` where that can provide a `Surface`. @@ -92,7 +103,7 @@ minutes, you will be able to stream live video to your server. `app:enableZoomOnPinch` is a boolean to enable zoom on pinch gesture. -3. Instantiates the streamer (main live streaming class) +3. Instantiate the streamer (main live streaming class) A `Streamer` is a class that represents a whole streaming pipeline from capture to endpoint ( incl. encoding, muxing, sending). @@ -119,7 +130,7 @@ minutes, you will be able to stream live video to your server. */ val streamer = cameraSingleStreamer(context = requireContext()) - + /** * To have multiple independent outputs (like for live and record), use a `cameraDualStreamer` or even the `StreamerPipeline`. * @@ -129,12 +140,11 @@ minutes, you will be able to stream live video to your server. * streamer.setVideoSource(CameraSourceFactory()) // Same as streamer.setCameraId(context.defaultCameraId) * streamer.setAudioSource(MicrophoneSourceFactory()) */ - ``` For more information, check the [Streamers](docs/Streamers.md) documentation. -4. Configures audio and video settings +4. Configure audio and video settings ```kotlin val streamer = cameraSingleStreamer(context = requireContext()) // Already instantiated streamer @@ -159,7 +169,7 @@ minutes, you will be able to stream live video to your server. } ``` -5. Inflates the preview with the streamer +5. Inflate the preview with the streamer Either `xml` UI @@ -189,7 +199,7 @@ minutes, you will be able to stream live video to your server. } ``` -6. Sets the device orientation +6. Set the device orientation ```kotlin // Already instantiated streamer @@ -237,7 +247,7 @@ minutes, you will be able to stream live video to your server. You can also create your own `targetRotation` provider. -7. Starts the live streaming +7. Start the live streaming ```kotlin // Already instantiated streamer @@ -256,7 +266,7 @@ minutes, you will be able to stream live video to your server. // streamer.startStream("rtmp://serverip:1935/s/streamKey") // For RTMP/RTMPS ``` -8. Stops and releases the streamer +8. Stop and release the streamer ```kotlin // Already instantiated streamer @@ -272,7 +282,7 @@ the [documentation](#documentations). For a complete example, check out the [demos/camera](demos/camera) directory. -### Getting started for a screen recorder stream +### 🖥️ Screen recorder stream 1. Add the `streampack-services` dependency in your `build.gradle` file: @@ -282,17 +292,17 @@ For a complete example, check out the [demos/camera](demos/camera) directory. } ``` -2. Requests the required permissions in your Activity/Fragment. See the +2. Request the required permissions in your Activity/Fragment. See the [Permissions](#permissions) section for more information. -3. Creates a `MyService` that extends `MediaProjectionService` (so you can customize +3. Create a `MyService` that extends `MediaProjectionService` (so you can customize notifications among other things). -4. Creates a screen record `Intent` and requests the activity result +4. Create a screen record `Intent` and requests the activity result ```kotlin MediaProjectionUtils.createScreenCaptureIntent(context = requireContext()) ``` -5. Starts the service +5. Start the service ```kotlin MediaProjectionService.bindService( @@ -313,7 +323,7 @@ For a complete example, check out the [demos/camera](demos/camera) directory. For a complete example, check out the [demos/screenrecorder](demos/screenrecorder) directory . -## Permissions +## 🔑 Permissions You need to add the following permissions in your `AndroidManifest.xml`: @@ -330,7 +340,7 @@ You need to add the following permissions in your `AndroidManifest.xml`: To record locally, you also need to request the following dangerous permission: `android.permission.WRITE_EXTERNAL_STORAGE`. -### Permissions for a camera stream +### Camera stream To use the camera, you need to request the following permission: @@ -355,7 +365,7 @@ For the PlayStore, your application might declare this in its `AndroidManifest.x ``` -### Permissions for a screen recorder stream +### Screen recorder stream To use the screen recorder, you need to request the following permission: @@ -381,7 +391,7 @@ You will also have to declare the `Service`, ``` -## Documentations +## 📖 Documentations - [StreamPack API guide](https://thibaultbee.github.io/StreamPack) - Additional documentations are available in the `docs` directory: @@ -389,7 +399,7 @@ You will also have to declare the `Service`, - [Streamers](docs/Streamers.md) - [Streamer elements](docs/AdvancedStreamer.md) -## Demos +## 🎬 Demos ### Camera and audio demo @@ -415,7 +425,7 @@ Tells FFplay to listen on IP `0.0.0.0` and port `1935`. ffplay -listen 1 -i 'rtmp://0.0.0.0:1935/s/streamKey' ``` -On StreamPack sample app settings, set `Endpoint` -> `Type` to `Stream to a remove RTMP device`, +On StreamPack sample app settings, set `Endpoint` -> `Type` to `Stream to a remote RTMP device`, then set the server `URL` to `rtmp://serverip:1935/s/streamKey`. At this point, StreamPack sample app should successfully sends audio and video frames. On FFplay side, you should be able to watch this live stream. @@ -432,7 +442,15 @@ On StreamPack sample app settings, set the server `IP` to your server IP and ser At this point, StreamPack sample app should successfully sends audio and video frames. On FFplay side, you should be able to watch this live stream. -## Tips +## 🤝 Contributing & Support + +⭐ If you like this project, don’t forget to star it! + +💖 Want to support its development? Consider becoming a sponsor. + +🛠️ Contributions are welcome—feel free to open issues or submit pull requests! + +## 💡 Tips ### RTMP or SRT @@ -503,7 +521,7 @@ Even if StreamPack sdk supports a `minSdkVersion` 21. I strongly recommend to se `minSdkVersion` of your application to a higher version (the highest is the best!) for better performance. -## Licence +## 📄 License Copyright 2021 Thibault B. From 394d7b73ff729f78dbea042b705d46788e79dc93 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 2 Mar 2026 11:20:04 +0100 Subject: [PATCH 39/72] refactor(core): make mutable frame and raw frame private to force user to use a pool --- .../encoding/EncodingPipelineOutputTest.kt | 6 +- .../streampack/core/utils/FakeRawFramey.kt | 28 ++++++ .../streampack/core/elements/data/Frame.kt | 85 +------------------ .../endpoints/composites/muxers/ts/TsMuxer.kt | 18 ++-- .../core/elements/utils/pool/FramePool.kt | 72 ++++++++++++++-- .../pool/{ObjectPool.kt => ObjectPoolImpl.kt} | 17 ++-- .../core/elements/utils/pool/RawFramePool.kt | 49 +++++++++-- .../elements/endpoints/DynamicEndpointTest.kt | 4 +- .../composites/muxers/ts/TsMuxerTest.kt | 14 +-- .../composites/muxers/ts/packets/PesTest.kt | 8 +- .../{FakeFrames.kt => FakeFrameFactory.kt} | 18 +++- 11 files changed, 189 insertions(+), 130 deletions(-) create mode 100644 core/src/androidTest/java/io/github/thibaultbee/streampack/core/utils/FakeRawFramey.kt rename core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/{ObjectPool.kt => ObjectPoolImpl.kt} (87%) rename core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/{FakeFrames.kt => FakeFrameFactory.kt} (82%) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt index b2f1d4daf..ac31423dd 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt @@ -20,7 +20,6 @@ import android.util.Log import androidx.core.net.toFile import androidx.test.platform.app.InstrumentationRegistry import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.MutableRawFrame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.DummyEndpoint @@ -37,6 +36,7 @@ import io.github.thibaultbee.streampack.core.pipelines.outputs.IConfigurableVide import io.github.thibaultbee.streampack.core.pipelines.outputs.IPipelineEventOutputInternal import io.github.thibaultbee.streampack.core.pipelines.outputs.isStreaming import io.github.thibaultbee.streampack.core.pipelines.outputs.releaseBlocking +import io.github.thibaultbee.streampack.core.utils.FakeRawFrame import junit.framework.TestCase.assertTrue import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first @@ -317,7 +317,7 @@ class EncodingPipelineOutputTest { try { output.queueAudioFrame( - MutableRawFrame( + FakeRawFrame( ByteBuffer.allocateDirect(16384), Random.nextLong() ) @@ -330,7 +330,7 @@ class EncodingPipelineOutputTest { output.startStream(descriptor) output.queueAudioFrame( - MutableRawFrame( + FakeRawFrame( ByteBuffer.allocateDirect(16384), Random.nextLong() ) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/utils/FakeRawFramey.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/utils/FakeRawFramey.kt new file mode 100644 index 000000000..2edb8a4fe --- /dev/null +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/utils/FakeRawFramey.kt @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.utils + +import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import java.nio.ByteBuffer + +data class FakeRawFrame( + override val rawBuffer: ByteBuffer, + override val timestampInUs: Long +) : RawFrame { + override fun close() { + // Nothing to do + } +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index ab615b1ad..9c408c632 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -75,35 +75,6 @@ fun RawFrame.copy( }) } -/** - * A mutable [RawFrame] internal representation. - * - * The purpose is to get reusable [RawFrame] - */ -data class MutableRawFrame( - /** - * Contains an audio or video frame data. - */ - override var rawBuffer: ByteBuffer, - - /** - * Presentation timestamp in µs - */ - override var timestampInUs: Long, - /** - * A callback to call when frame is closed. - */ - override var onClosed: (MutableRawFrame) -> Unit = {} -) : RawFrame, WithClosable { - override fun close() { - try { - onClosed(this) - } catch (_: Throwable) { - // Nothing to do - } - } -} - /** * Encoded frame representation */ @@ -169,60 +140,6 @@ fun Frame.copy( }) } - -/** - * A mutable [Frame] internal representation. - * - * The purpose is to get reusable [Frame] - */ -data class MutableFrame( - /** - * Contains an audio or video frame data. - */ - override var rawBuffer: ByteBuffer, - - /** - * Presentation timestamp in µs - */ - override var ptsInUs: Long, - - /** - * Decoded timestamp in µs (not used). - */ - override var dtsInUs: Long?, - - /** - * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) - */ - override var isKeyFrame: Boolean, - - /** - * Contains csd buffers for key frames and audio frames only. - * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. - * ESDS for AAC frames,... - */ - override var extra: Extra?, - - /** - * Contains frame format.. - * TODO: to remove - */ - override var format: MediaFormat, - - /** - * A callback to call when frame is closed. - */ - override var onClosed: (MutableFrame) -> Unit = {} -) : Frame, WithClosable { - override fun close() { - try { - onClosed(this) - } catch (_: Throwable) { - // Nothing to do - } - } -} - /** * Ensures that extra are not used at the same time. * @@ -231,7 +148,7 @@ data class MutableFrame( class Extra(private val extraBuffers: List) { private val lock = Any() - val _length by lazy { extraBuffers.sumOf { it.remaining() } } + private val _length by lazy { extraBuffers.sumOf { it.remaining() } } fun getLength(): Int { return synchronized(lock) { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index 127662e43..1376bc07e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -40,7 +40,7 @@ import java.util.MissingFormatArgumentException import kotlin.random.Random class TsMuxer : IMuxerInternal { - private val byteBufferPool = ByteBufferPool(true) + private val tsBufferPool = ByteBufferPool(true) override val info by lazy { TSMuxerInfo } private val tsServices = mutableListOf() @@ -60,10 +60,10 @@ class TsMuxer : IMuxerInternal { private val tsId = Random.nextInt(Byte.MIN_VALUE.toInt(), Byte.MAX_VALUE.toInt()).toShort() private var pat = Pat( - byteBufferPool, listener, tsServices, tsId, packetCount = 0 + tsBufferPool, listener, tsServices, tsId, packetCount = 0 ) private var sdt = Sdt( - byteBufferPool, listener, tsServices, tsId, packetCount = 0 + tsBufferPool, listener, tsServices, tsId, packetCount = 0 ) override val streamConfigs: List @@ -78,7 +78,7 @@ class TsMuxer : IMuxerInternal { override fun write( frame: Frame, streamPid: Int ) { - try { + frame.use { frame -> val pes = getPes(streamPid.toShort()) val mimeType = pes.stream.config.mimeType val newFrame = when { @@ -169,8 +169,6 @@ class TsMuxer : IMuxerInternal { newFrame.close() } } - } finally { - frame.close() } } @@ -361,12 +359,12 @@ class TsMuxer : IMuxerInternal { service.pmt = service.pmt?.apply { versionNumber = (versionNumber + 1).toByte() streams = service.streams - } ?: Pmt(byteBufferPool, listener, service, service.streams, getNewPid()) + } ?: Pmt(tsBufferPool, listener, service, service.streams, getNewPid()) // Init PES newStreams.forEach { Pes( - byteBufferPool, + tsBufferPool, listener, it, service.pcrPid == it.pid, @@ -442,12 +440,12 @@ class TsMuxer : IMuxerInternal { tsServices.forEach { removeStreams(it) } - byteBufferPool.clear() + tsBufferPool.clear() } override fun release() { tsServices.clear() - byteBufferPool.close() + tsBufferPool.close() } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt index c1e824e97..3c94714f5 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -17,14 +17,17 @@ package io.github.thibaultbee.streampack.core.elements.utils.pool import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Extra -import io.github.thibaultbee.streampack.core.elements.data.MutableFrame +import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.WithClosable import java.nio.ByteBuffer /** * A pool of [MutableFrame]. */ -internal class FramePool() : ObjectPool() { +internal class FramePool : IClearableObjectPool { + private val pool = ObjectPoolImpl() + fun get( rawBuffer: ByteBuffer, ptsInUs: Long, @@ -32,13 +35,13 @@ internal class FramePool() : ObjectPool() { isKeyFrame: Boolean, extra: Extra?, format: MediaFormat, - onClosed: (MutableFrame) -> Unit - ): MutableFrame { - val frame = get() + onClosed: (Frame) -> Unit = {} + ): Frame { + val frame = pool.get() val onClosedHook = { frame: MutableFrame -> onClosed(frame) - put(frame) + pool.put(frame) } return if (frame != null) { @@ -63,6 +66,63 @@ internal class FramePool() : ObjectPool() { } } + override fun clear() = pool.clear() + + override fun close() = pool.close() + + /** + * A mutable [Frame] internal representation. + * + * The purpose is to get reusable [Frame] + */ + private data class MutableFrame( + /** + * Contains an audio or video frame data. + */ + override var rawBuffer: ByteBuffer, + + /** + * Presentation timestamp in µs + */ + override var ptsInUs: Long, + + /** + * Decoded timestamp in µs (not used). + */ + override var dtsInUs: Long?, + + /** + * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) + */ + override var isKeyFrame: Boolean, + + /** + * Contains csd buffers for key frames and audio frames only. + * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. + * ESDS for AAC frames,... + */ + override var extra: Extra?, + + /** + * Contains frame format.. + * TODO: to remove + */ + override var format: MediaFormat, + + /** + * A callback to call when frame is closed. + */ + override var onClosed: (MutableFrame) -> Unit = {} + ) : Frame, WithClosable { + override fun close() { + try { + onClosed(this) + } catch (_: Throwable) { + // Nothing to do + } + } + } + companion object { /** * The default frame pool. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPoolImpl.kt similarity index 87% rename from core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPoolImpl.kt index 73bc0b337..7817192dd 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPoolImpl.kt @@ -15,11 +15,18 @@ */ package io.github.thibaultbee.streampack.core.elements.utils.pool -import io.github.thibaultbee.streampack.core.logger.Logger import java.io.Closeable import java.util.ArrayDeque import java.util.concurrent.atomic.AtomicBoolean +interface IClearableObjectPool : Closeable { + fun clear() +} + +interface IObjectPool : IClearableObjectPool { + fun put(any: T) +} + /** * A pool of objects. * @@ -29,12 +36,12 @@ import java.util.concurrent.atomic.AtomicBoolean * * @param T the type of object to pool */ -internal sealed class ObjectPool() : Closeable { +internal open class ObjectPoolImpl : IObjectPool { private val pool = ArrayDeque() private val isClosed = AtomicBoolean(false) - protected fun get(): T? { + fun get(): T? { if (isClosed.get()) { throw IllegalStateException("ObjectPool is closed") } @@ -53,7 +60,7 @@ internal sealed class ObjectPool() : Closeable { * * @param any the object to put */ - fun put(any: T) { + override fun put(any: T) { if (isClosed.get()) { throw IllegalStateException("ObjectPool is closed") } @@ -65,7 +72,7 @@ internal sealed class ObjectPool() : Closeable { /** * Clears the pool. */ - fun clear() { + override fun clear() { if (isClosed.get()) { return } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt index 0c7b4e6a0..87e09b2e8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt @@ -15,24 +15,27 @@ */ package io.github.thibaultbee.streampack.core.elements.utils.pool -import io.github.thibaultbee.streampack.core.elements.data.MutableRawFrame +import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.WithClosable import java.nio.ByteBuffer /** * A pool of [MutableRawFrame]. */ -internal class RawFramePool() : ObjectPool() { +internal class RawFramePool : IClearableObjectPool { + private val pool = ObjectPoolImpl() + fun get( rawBuffer: ByteBuffer, timestampInUs: Long, - onClosed: (MutableRawFrame) -> Unit = {} - ): MutableRawFrame { - val frame = get() + onClosed: (RawFrame) -> Unit = {} + ): RawFrame { + val frame = pool.get() val onClosedHook = { frame: MutableRawFrame -> onClosed(frame) - put(frame) + pool.put(frame) } return if (frame != null) { @@ -49,6 +52,40 @@ internal class RawFramePool() : ObjectPool() { } } + override fun clear() = pool.clear() + + override fun close() = pool.close() + + /** + * A mutable [RawFrame] internal representation. + * + * The purpose is to get reusable [RawFrame] + */ + private data class MutableRawFrame( + /** + * Contains an audio or video frame data. + */ + override var rawBuffer: ByteBuffer, + + /** + * Presentation timestamp in µs + */ + override var timestampInUs: Long, + /** + * A callback to call when frame is closed. + */ + override var onClosed: (MutableRawFrame) -> Unit = {} + ) : RawFrame, WithClosable { + override fun close() { + try { + onClosed(this) + } catch (_: Throwable) { + // Nothing to do + } + } + } + + companion object { /** * The default frame pool. diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt index 3b96e15f9..44475954a 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt @@ -6,7 +6,7 @@ import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor import io.github.thibaultbee.streampack.core.elements.utils.DescriptorUtils -import io.github.thibaultbee.streampack.core.elements.utils.FakeFrames +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrameFactory import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.test.runTest import org.junit.Assert.assertFalse @@ -85,7 +85,7 @@ class DynamicEndpointTest { val dynamicEndpoint = DynamicEndpoint(context, Dispatchers.Default, Dispatchers.IO) try { dynamicEndpoint.write( - FakeFrames.create(MediaFormat.MIMETYPE_AUDIO_AAC), + FakeFrameFactory.create(MediaFormat.MIMETYPE_AUDIO_AAC), 0 ) fail("Throwable expected") diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt index b3fa4dd23..4eaacf8db 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt @@ -21,7 +21,7 @@ import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.TSConst import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.Utils.createFakeServiceInfo -import io.github.thibaultbee.streampack.core.elements.utils.FakeFrames +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrameFactory import io.github.thibaultbee.streampack.core.elements.utils.MockUtils import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -193,7 +193,7 @@ class TsMuxerTest { val tsMux = TsMuxer() try { tsMux.write( - FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), -1 ) fail() @@ -206,7 +206,7 @@ class TsMuxerTest { val tsMux = TsMuxer() try { tsMux.write( - FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), -1 ) fail() @@ -231,11 +231,11 @@ class TsMuxerTest { tsMux.addStreams(service, listOf(config))[config]!! tsMux.write( - FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid ) tsMux.write( - FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid ) } @@ -251,10 +251,10 @@ class TsMuxerTest { tsMux.addStreams(createFakeServiceInfo(), listOf(config))[config]!! tsMux.write( - FakeFrames.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid ) tsMux.write( - FakeFrames.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid ) } } \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/packets/PesTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/packets/PesTest.kt index 389504af9..361cea1dd 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/packets/PesTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/packets/PesTest.kt @@ -21,7 +21,7 @@ import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.data.Stream import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.AssertEqualsBuffersMockMuxerListener -import io.github.thibaultbee.streampack.core.elements.utils.FakeFrames +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrameFactory import io.github.thibaultbee.streampack.core.elements.utils.MockUtils import io.github.thibaultbee.streampack.core.elements.utils.ResourcesUtils import io.github.thibaultbee.streampack.core.elements.utils.pool.ByteBufferPool @@ -54,7 +54,7 @@ class PesTest { MockUtils.mockTimeUtils(1433034) val rawData = ResourcesUtils.readByteBuffer(TEST_SAMPLES_DIR + "pes-video1/raw") - val frame = FakeFrames.create( + val frame = FakeFrameFactory.create( buffer = rawData, pts = 1433334, dts = 1400000, @@ -84,7 +84,7 @@ class PesTest { MockUtils.mockTimeUtils(700000) val rawData = ResourcesUtils.readByteBuffer(TEST_SAMPLES_DIR + "pes-audio1/raw.aac") - val frame = FakeFrames.create( + val frame = FakeFrameFactory.create( buffer = rawData, pts = 1400000, dts = null, @@ -108,7 +108,7 @@ class PesTest { MockUtils.mockTimeUtils(700000) val rawData = ResourcesUtils.readByteBuffer(TEST_SAMPLES_DIR + "pes-audio2/raw.aac") - val frame = FakeFrames.create( + val frame = FakeFrameFactory.create( buffer = rawData, pts = 1400000, dts = null, diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrameFactory.kt similarity index 82% rename from core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt rename to core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrameFactory.kt index 542ef0a8e..78888f82d 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrameFactory.kt @@ -18,11 +18,10 @@ package io.github.thibaultbee.streampack.core.elements.utils import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Extra import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer import kotlin.random.Random -object FakeFrames { +object FakeFrameFactory { fun create( mimeType: String, buffer: ByteBuffer = ByteBuffer.wrap(Random.nextBytes(1024)), @@ -52,7 +51,7 @@ object FakeFrames { ) ) } - return MutableFrame( + return FakeFrame( buffer, pts, dts, @@ -67,4 +66,17 @@ object FakeFrames { format = format ) } + + data class FakeFrame( + override val rawBuffer: ByteBuffer, + override val ptsInUs: Long, + override val dtsInUs: Long?, + override val isKeyFrame: Boolean, + override val extra: Extra?, + override val format: MediaFormat, + ) : Frame { + override fun close() { + // Nothing to do + } + } } From 1e4546ffefcffa908a558b1b47772d9e8e9c8b90 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 2 Mar 2026 16:46:45 +0100 Subject: [PATCH 40/72] refactor(core): camera: pass a coroutine dispatcher instead of a coroutine scope --- .../camera/controllers/CameraController.kt | 2 +- .../controllers/CameraSessionController.kt | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index a2fca02b0..f5cf17a7b 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -163,7 +163,7 @@ internal class CameraController( return if (sessionController == null) { val deviceController = getDeviceController() CameraSessionController.create( - coroutineScope, + defaultDispatcher, deviceController, sessionCallback, sessionCompat, diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index b42274eca..10b1d4533 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -28,7 +28,7 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder import io.github.thibaultbee.streampack.core.logger.Logger -import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -39,7 +39,7 @@ import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext internal class CameraSessionController private constructor( - private val coroutineScope: CoroutineScope, + private val coroutineDispatcher: CoroutineDispatcher, private val captureSession: CameraCaptureSession, private val captureRequestBuilder: CaptureRequestWithTargetsBuilder, private val sessionCallback: CameraSessionCallback, @@ -71,7 +71,7 @@ internal class CameraSessionController private constructor( private val captureCallbacks = setOf(captureCallback, sessionCallback) - suspend fun isEmpty() = withContext(coroutineScope.coroutineContext) { + suspend fun isEmpty() = withContext(coroutineDispatcher) { requestTargetMutex.withLock { captureRequestBuilder.isEmpty() } } @@ -81,7 +81,7 @@ internal class CameraSessionController private constructor( * @param surface The target to check * @return true if the target is in the current capture request, false otherwise */ - suspend fun hasTarget(surface: Surface) = withContext(coroutineScope.coroutineContext) { + suspend fun hasTarget(surface: Surface) = withContext(coroutineDispatcher) { requestTargetMutex.withLock { captureRequestBuilder.hasTarget(surface) } @@ -94,7 +94,7 @@ internal class CameraSessionController private constructor( * @return true if the target is in the current capture request, false otherwise */ suspend fun hasTarget(cameraSurface: CameraSurface) = - withContext(coroutineScope.coroutineContext) { + withContext(coroutineDispatcher) { requestTargetMutex.withLock { captureRequestBuilder.hasTarget(cameraSurface) } @@ -110,7 +110,7 @@ internal class CameraSessionController private constructor( require(targets.all { it.surface.isValid }) { "All targets must be valid" } require(targets.all { outputs.contains(it) }) { "Targets must be in the current capture session: $targets ($outputs)" } - val res = withContext(coroutineScope.coroutineContext) { + val res = withContext(coroutineDispatcher) { requestTargetMutex.withLock { val res = targets.map { captureRequestBuilder.addTarget(it) @@ -131,7 +131,7 @@ internal class CameraSessionController private constructor( suspend fun addTarget(name: String): Boolean { require(outputs.any { it.name == name }) { "Target type must be in the current capture session: $name ($outputs)" } - val res = withContext(coroutineScope.coroutineContext) { + val res = withContext(coroutineDispatcher) { requestTargetMutex.withLock { val target = outputs.first { it.name == name } val res = captureRequestBuilder.addTarget(target) @@ -151,7 +151,7 @@ internal class CameraSessionController private constructor( require(target.surface.isValid) { "Target must be valid: $target" } require(outputs.contains(target)) { "Target must be in the current capture session: $target ($outputs)" } - val res = withContext(coroutineScope.coroutineContext) { + val res = withContext(coroutineDispatcher) { requestTargetMutex.withLock { val res = captureRequestBuilder.addTarget(target) setRepeatingSession() @@ -167,7 +167,7 @@ internal class CameraSessionController private constructor( * @param targets The targets to remove */ suspend fun removeTargets(targets: List) { - withContext(coroutineScope.coroutineContext) { + withContext(coroutineDispatcher) { requestTargetMutex.withLock { targets.forEach { captureRequestBuilder.removeTarget(it) @@ -187,7 +187,7 @@ internal class CameraSessionController private constructor( * @param name The name of target to remove */ suspend fun removeTarget(name: String) { - withContext(coroutineScope.coroutineContext) { + withContext(coroutineDispatcher) { requestTargetMutex.withLock { val target = outputs.firstOrNull { it.name == name } target?.let { @@ -212,7 +212,7 @@ internal class CameraSessionController private constructor( * @param target The target to remove */ suspend fun removeTarget(target: CameraSurface) { - withContext(coroutineScope.coroutineContext) { + withContext(coroutineDispatcher) { requestTargetMutex.withLock { captureRequestBuilder.removeTarget(target) @@ -226,7 +226,7 @@ internal class CameraSessionController private constructor( } suspend fun close() { - withContext(coroutineScope.coroutineContext) { + withContext(coroutineDispatcher) { captureSessionMutex.withLock { if (isClosed) { Logger.w(TAG, "Session already closed") @@ -254,7 +254,7 @@ internal class CameraSessionController private constructor( Logger.w(TAG, "Capture request is empty") return } - withContext(coroutineScope.coroutineContext) { + withContext(coroutineDispatcher) { captureSessionMutex.withLock { if (isClosed) { Logger.w(TAG, "Camera session controller is released") @@ -273,7 +273,7 @@ internal class CameraSessionController private constructor( } private suspend fun stopRepeatingSession() { - withContext(coroutineScope.coroutineContext) { + withContext(coroutineDispatcher) { captureSessionMutex.withLock { if (isClosed) { Logger.w(TAG, "Camera session controller is released") @@ -316,7 +316,7 @@ internal class CameraSessionController private constructor( outputs: List, dynamicRange: Long, fpsRange: Range - ): CameraSessionController = withContext(coroutineScope.coroutineContext) { + ): CameraSessionController = withContext(coroutineDispatcher) { requestTargetMutex.withLock { require(outputs.isNotEmpty()) { "At least one output is required" } require(outputs.all { it.surface.isValid }) { "All outputs $outputs must be valid but ${outputs.filter { !it.surface.isValid }} is invalid" } @@ -349,7 +349,7 @@ internal class CameraSessionController private constructor( ) val controller = CameraSessionController( - coroutineScope, + coroutineDispatcher, newCaptureSession, captureRequestBuilder, sessionCallback, @@ -372,7 +372,7 @@ internal class CameraSessionController private constructor( private const val TAG = "CameraSessionController" suspend fun create( - coroutineScope: CoroutineScope, + coroutineDispatcher: CoroutineDispatcher, cameraDeviceController: CameraDeviceController, sessionCallback: CameraSessionCallback, sessionCompat: ICameraCaptureSessionCompat, @@ -402,7 +402,7 @@ internal class CameraSessionController private constructor( defaultRequestBuilder() } return CameraSessionController( - coroutineScope, + coroutineDispatcher, captureSession, captureRequestBuilder, sessionCallback, From beb361abf39504dc170b705b73cefd0b8728f0ea Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Mar 2026 15:05:29 +0100 Subject: [PATCH 41/72] refactor(core): camera: move capture request builder out of the session controller as it is not link to it --- .../sources/video/camera/CameraSettings.kt | 10 +- .../sources/video/camera/CameraSource.kt | 2 +- .../camera/controllers/CameraController.kt | 169 ++++++----- .../controllers/CameraDeviceController.kt | 9 +- .../controllers/CameraSessionController.kt | 275 +++--------------- .../utils/CaptureRequestWithTargetsBuilder.kt | 27 +- 6 files changed, 155 insertions(+), 337 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 78f4f97e5..cee9d7437 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -248,6 +248,13 @@ class CameraSettings internal constructor( */ suspend fun applyRepeatingSession() = cameraController.setRepeatingSession() + /** + * Resets all settings to default. + */ + suspend fun resetAll() { + + } + private class TagBundle(val keyId: Long) { private val tagMap = mutableMapOf().apply { put(TAG_KEY_ID, keyId) @@ -345,7 +352,8 @@ class CameraSettings internal constructor( val availableStrengthLevelRange: Range by lazy { Range( 1, - characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] ?: DEFAULT_STRENGTH_LEVEL + characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] + ?: DEFAULT_STRENGTH_LEVEL ) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt index 38c607749..4a9216483 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt @@ -66,7 +66,7 @@ internal class CameraSource( characteristics, dispatcherProvider, cameraId, - captureRequestBuilder = { + onCaptureRequestBuilder = { defaultCaptureRequest(this) } ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index f5cf17a7b..4bec131bb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -48,7 +48,7 @@ internal class CameraController( private val characteristics: CameraCharacteristics, dispatcherProvider: CameraDispatcherProvider, val cameraId: String, - val captureRequestBuilder: CaptureRequestWithTargetsBuilder.() -> Unit = {} + val onCaptureRequestBuilder: CaptureRequestWithTargetsBuilder.() -> Unit = {} ) { private val sessionCompat = CameraCaptureSessionCompatBuilder.build(dispatcherProvider) @@ -59,6 +59,7 @@ internal class CameraController( private var deviceController: CameraDeviceController? = null private var sessionController: CameraSessionController? = null + private var captureRequestBuilder: CaptureRequestWithTargetsBuilder? = null private val sessionCallback = CameraSessionCallback(coroutineScope) private val controllerMutex = Mutex() @@ -155,22 +156,35 @@ internal class CameraController( } } + @RequiresPermission(Manifest.permission.CAMERA) + private suspend fun getCaptureRequestBuilder(): CaptureRequestWithTargetsBuilder { + return if (captureRequestBuilder != null) { + captureRequestBuilder!! + } else { + getDeviceController().createCaptureRequestBuilder().apply { + captureRequestBuilder = this + + val minFrameDuration = 1_000_000_000 / fpsRange.upper.toLong() + set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange) + set(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) + onCaptureRequestBuilder() + } + } + } + /** * To be used under [controllerMutex] */ @RequiresPermission(Manifest.permission.CAMERA) private suspend fun getSessionController(): CameraSessionController { return if (sessionController == null) { - val deviceController = getDeviceController() CameraSessionController.create( - defaultDispatcher, - deviceController, + getDeviceController(), sessionCallback, sessionCompat, outputs.values.toList(), dynamicRange = dynamicRangeProfile.dynamicRange, - fpsRange = fpsRange, - captureRequestBuilder + defaultDispatcher ).apply { applySessionController(this) Logger.d(TAG, "Session controller created") @@ -180,12 +194,10 @@ internal class CameraController( sessionController!! } else { try { - val deviceController = getDeviceController() sessionController!!.recreate( - deviceController, + getDeviceController(), outputs.values.toList(), - dynamicRange = dynamicRangeProfile.dynamicRange, - fpsRange = fpsRange + dynamicRange = dynamicRangeProfile.dynamicRange ).apply { applySessionController(this) Logger.d(TAG, "Session controller recreated") @@ -198,7 +210,9 @@ internal class CameraController( } } - private fun applySessionController(sessionController: CameraSessionController) { + private suspend fun applySessionController( + sessionController: CameraSessionController + ) { this.sessionController = sessionController isActiveJob = coroutineScope.launch { @@ -206,6 +220,19 @@ internal class CameraController( _isActiveFlow.emit(!it) } } + + // Re-add targets that are in the new outputs (identified by their name) + captureRequestBuilder?.let { + val targets = outputs.values.filter { output -> + it.hasTarget(output) + } + it.clearTargets() + if (targets.isNotEmpty()) { + it.addTargets(targets) + } + } + + setRepeatingSessionUnsafe() } /** @@ -227,8 +254,7 @@ internal class CameraController( sessionController.recreate( getDeviceController(), outputs.values.toList(), - dynamicRange = dynamicRangeProfile.dynamicRange, - fpsRange = fpsRange + dynamicRange = dynamicRangeProfile.dynamicRange ).apply { applySessionController(this) Logger.d(TAG, "Session controller restarted") @@ -244,53 +270,18 @@ internal class CameraController( @RequiresPermission(Manifest.permission.CAMERA) suspend fun addTarget(name: String) = withContext(defaultDispatcher) { controllerMutex.withLock { - val sessionController = getSessionController() - sessionController.addTarget(name) - } - } + val target = outputs[name] ?: throw IllegalArgumentException( + "Output $name not found in outputs ${outputs.keys.joinToString(", ")}" + ) - /** - * Adds a target to the current capture session. - * - * @param target The target to add - * @return true if the target has been added, false otherwise - */ - @RequiresPermission(Manifest.permission.CAMERA) - suspend fun addTarget(target: CameraSurface) = withContext(defaultDispatcher) { - controllerMutex.withLock { - val sessionController = getSessionController() - sessionController.addTarget(target) - } - } - - /** - * Adds targets to the current capture session. - * - * @param targets The targets to add - * @return true if the targets have been added, false otherwise - */ - @RequiresPermission(Manifest.permission.CAMERA) - suspend fun addTargets(targets: List) = withContext(defaultDispatcher) { - controllerMutex.withLock { - val sessionController = getSessionController() - sessionController.addTargets(targets) - } - } - - /** - * Removes a target from the current capture session. - * - * @param target The target to remove - */ - @RequiresPermission(Manifest.permission.CAMERA) - suspend fun removeTarget(target: CameraSurface) { - withContext(defaultDispatcher) { - controllerMutex.withLock { - val sessionController = getSessionController() - sessionController.removeTarget(target) - if (sessionController.isEmpty()) { - closeControllers() + val wasAdded = getCaptureRequestBuilder().addTarget(target) + if (wasAdded) { + try { + setRepeatingSessionUnsafe() + } catch (t: Throwable) { + Logger.e(TAG, "Error to add target $name", t) } + Logger.d(TAG, "Target $name added") } } } @@ -304,11 +295,18 @@ internal class CameraController( suspend fun removeTarget(name: String) { withContext(defaultDispatcher) { controllerMutex.withLock { - val sessionController = getSessionController() - sessionController.removeTarget(name) - if (sessionController.isEmpty()) { - closeControllers() + val target = outputs[name] ?: return@withContext + + val wasRemoved = getCaptureRequestBuilder().removeTarget(target) + if (!wasRemoved) { + return@withLock } + try { + setRepeatingSessionUnsafe() + } catch (t: Throwable) { + Logger.e(TAG, "Error to remove target $name", t) + } + Logger.d(TAG, "Target $name removed") } } } @@ -317,8 +315,9 @@ internal class CameraController( * Gets a setting from the current capture request. */ fun getSetting(key: CaptureRequest.Key): T? { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - return sessionController.getSetting(key) + val captureRequestBuilder = + requireNotNull(captureRequestBuilder) { "CaptureRequestBuilder is null" } + return captureRequestBuilder.get(key) } /** @@ -330,8 +329,9 @@ internal class CameraController( * @param value The setting value */ fun setSetting(key: CaptureRequest.Key, value: T) { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setSetting(key, value) + val captureRequestBuilder = + requireNotNull(captureRequestBuilder) { "CaptureRequestBuilder is null" } + captureRequestBuilder.set(key, value) } /** @@ -339,6 +339,7 @@ internal class CameraController( * * @param fpsRange The fps range */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setFps(fps: Int) { withContext(defaultDispatcher) { controllerMutex.withLock { @@ -348,12 +349,15 @@ internal class CameraController( this@CameraController.fps = fps + val captureRequestBuilder = getCaptureRequestBuilder() + + val range = fpsRange + val minFrameDuration = 1_000_000_000 / range.upper.toLong() + captureRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range) + captureRequestBuilder.set(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) + if (isActiveFlow.value) { - val range = fpsRange - val minFrameDuration = 1_000_000_000 / range.upper.toLong() - setSetting(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range) - setSetting(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) - setRepeatingSession() + setRepeatingSessionUnsafe() } } } @@ -400,14 +404,32 @@ internal class CameraController( sessionCallback.removeListener(listener) } + /** + * Sets a repeating session with the current capture request. + * + * @param tag A tag to associate with the session. + */ + suspend fun setRepeatingSessionUnsafe(tag: Any? = null) { + val sessionController = getSessionController() + val captureRequestBuilder = getCaptureRequestBuilder() + tag?.let { + captureRequestBuilder.setTag(it) + } + sessionController.applyRepeatingSession(captureRequestBuilder) + } + + /** * Sets a repeating session with the current capture request. * * @param tag A tag to associate with the session. */ suspend fun setRepeatingSession(tag: Any? = null) { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setRepeatingSession(tag) + withContext(defaultDispatcher) { + controllerMutex.withLock { + setRepeatingSessionUnsafe(tag) + } + } } private suspend fun closeControllers() { @@ -420,6 +442,7 @@ internal class CameraController( withContext(defaultDispatcher) { controllerMutex.withLock { closeControllers() + captureRequestBuilder = null sessionController = null } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraDeviceController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraDeviceController.kt index 200a146ce..24dcf9220 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraDeviceController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraDeviceController.kt @@ -26,6 +26,7 @@ import android.view.Surface import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.ICameraCaptureSessionCompat import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -54,10 +55,9 @@ internal class CameraDeviceController private constructor( val id = cameraDevice.id - fun createCaptureRequest( - templateType: Int - ): CaptureRequest.Builder { - return cameraDevice.createCaptureRequest(templateType) + fun createCaptureRequestBuilder(template: Int = CameraDevice.TEMPLATE_RECORD): CaptureRequestWithTargetsBuilder { + val captureRequest = cameraDevice.createCaptureRequest(template) + return CaptureRequestWithTargetsBuilder(captureRequest) } fun createCaptureSession( @@ -66,7 +66,6 @@ internal class CameraDeviceController private constructor( sessionCompat.createCaptureSession(cameraDevice, targets, callback) } - fun createCaptureSessionByOutputConfiguration( outputConfigurations: List, callback: CameraCaptureSession.StateCallback diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index 10b1d4533..44f50ac90 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -20,7 +20,6 @@ import android.hardware.camera2.CameraCaptureSession.CaptureCallback import android.hardware.camera2.CaptureFailure import android.hardware.camera2.CaptureRequest import android.hardware.camera2.TotalCaptureResult -import android.util.Range import android.view.Surface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.ICameraCaptureSessionCompat import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback @@ -41,11 +40,8 @@ import kotlinx.coroutines.withContext internal class CameraSessionController private constructor( private val coroutineDispatcher: CoroutineDispatcher, private val captureSession: CameraCaptureSession, - private val captureRequestBuilder: CaptureRequestWithTargetsBuilder, private val sessionCallback: CameraSessionCallback, private val sessionCompat: ICameraCaptureSessionCompat, - private val outputs: List, - val dynamicRange: Long, val cameraIsClosedFlow: StateFlow, val isClosedFlow: StateFlow ) { @@ -54,8 +50,6 @@ internal class CameraSessionController private constructor( val isClosed: Boolean get() = isClosedFlow.value || cameraIsClosedFlow.value - private val requestTargetMutex = Mutex() - /** * A default capture callback that logs the failure reason. */ @@ -71,160 +65,6 @@ internal class CameraSessionController private constructor( private val captureCallbacks = setOf(captureCallback, sessionCallback) - suspend fun isEmpty() = withContext(coroutineDispatcher) { - requestTargetMutex.withLock { captureRequestBuilder.isEmpty() } - } - - /** - * Whether the current capture request has a target - * - * @param surface The target to check - * @return true if the target is in the current capture request, false otherwise - */ - suspend fun hasTarget(surface: Surface) = withContext(coroutineDispatcher) { - requestTargetMutex.withLock { - captureRequestBuilder.hasTarget(surface) - } - } - - /** - * Whether the current capture request has a target - * - * @param cameraSurface The target to check - * @return true if the target is in the current capture request, false otherwise - */ - suspend fun hasTarget(cameraSurface: CameraSurface) = - withContext(coroutineDispatcher) { - requestTargetMutex.withLock { - captureRequestBuilder.hasTarget(cameraSurface) - } - } - - /** - * Adds targets to the current capture session - * - * @param targets The targets to add - */ - suspend fun addTargets(targets: List): Boolean { - require(targets.isNotEmpty()) { "At least one target is required" } - require(targets.all { it.surface.isValid }) { "All targets must be valid" } - require(targets.all { outputs.contains(it) }) { "Targets must be in the current capture session: $targets ($outputs)" } - - val res = withContext(coroutineDispatcher) { - requestTargetMutex.withLock { - val res = targets.map { - captureRequestBuilder.addTarget(it) - }.all { it } - setRepeatingSession() - res - } - } - - return res - } - - /** - * Adds a target to the current capture session - * - * @param name The name of target to add - */ - suspend fun addTarget(name: String): Boolean { - require(outputs.any { it.name == name }) { "Target type must be in the current capture session: $name ($outputs)" } - - val res = withContext(coroutineDispatcher) { - requestTargetMutex.withLock { - val target = outputs.first { it.name == name } - val res = captureRequestBuilder.addTarget(target) - setRepeatingSession() - res - } - } - return res - } - - /** - * Adds a target to the current capture session - * - * @param target The target to add - */ - suspend fun addTarget(target: CameraSurface): Boolean { - require(target.surface.isValid) { "Target must be valid: $target" } - require(outputs.contains(target)) { "Target must be in the current capture session: $target ($outputs)" } - - val res = withContext(coroutineDispatcher) { - requestTargetMutex.withLock { - val res = captureRequestBuilder.addTarget(target) - setRepeatingSession() - res - } - } - return res - } - - /** - * Removes targets from the current capture session - * - * @param targets The targets to remove - */ - suspend fun removeTargets(targets: List) { - withContext(coroutineDispatcher) { - requestTargetMutex.withLock { - targets.forEach { - captureRequestBuilder.removeTarget(it) - } - if (captureRequestBuilder.isEmpty()) { - stopRepeatingSession() - } else { - setRepeatingSession() - } - } - } - } - - /** - * Removes a target from the current capture session - * - * @param name The name of target to remove - */ - suspend fun removeTarget(name: String) { - withContext(coroutineDispatcher) { - requestTargetMutex.withLock { - val target = outputs.firstOrNull { it.name == name } - target?.let { - captureRequestBuilder.removeTarget(it) - } ?: Logger.w( - TAG, - "Target type $name not found in current outputs $outputs" - ) - - if (captureRequestBuilder.isEmpty()) { - stopRepeatingSession() - } else { - setRepeatingSession() - } - } - } - } - - /** - * Removes a target from the current capture session - * - * @param target The target to remove - */ - suspend fun removeTarget(target: CameraSurface) { - withContext(coroutineDispatcher) { - requestTargetMutex.withLock { - captureRequestBuilder.removeTarget(target) - - if (captureRequestBuilder.isEmpty()) { - stopRepeatingSession() - } else { - setRepeatingSession() - } - } - } - } - suspend fun close() { withContext(coroutineDispatcher) { captureSessionMutex.withLock { @@ -244,12 +84,27 @@ internal class CameraSessionController private constructor( } } + /** + * Sets or stops a repeating session with the current capture request. + * + * If the [captureRequestBuilder] does not hold a [Surface], it will stop the repeating session. + * + * @param captureRequestBuilder The capture request builder to use + */ + suspend fun applyRepeatingSession(captureRequestBuilder: CaptureRequestWithTargetsBuilder) { + if (captureRequestBuilder.isEmpty()) { + stopRepeatingSession() + } else { + setRepeatingSession(captureRequestBuilder) + } + } + /** * Sets a repeating session with the current capture request. * - * @param tag A tag to associate with the session. + * @param captureRequestBuilder The capture request builder to use */ - suspend fun setRepeatingSession(tag: Any? = null) { + private suspend fun setRepeatingSession(captureRequestBuilder: CaptureRequestWithTargetsBuilder) { if (captureRequestBuilder.isEmpty()) { Logger.w(TAG, "Capture request is empty") return @@ -261,8 +116,6 @@ internal class CameraSessionController private constructor( return@withContext } - tag?.let { captureRequestBuilder.setTag(it) } - sessionCompat.setRepeatingSingleRequest( captureSession, captureRequestBuilder.build(), @@ -285,22 +138,6 @@ internal class CameraSessionController private constructor( } } - /** - * Gets a setting from the current capture request. - */ - fun getSetting(key: CaptureRequest.Key) = captureRequestBuilder.get(key) - - /** - * Sets a setting to the current capture request. - * - * Don't forget to call [setRepeatingSession] to apply the setting. - * - * @param key The setting key - * @param value The setting value - */ - fun setSetting(key: CaptureRequest.Key, value: T) = - captureRequestBuilder.set(key, value) - /** * Creates a new capture session with the given outputs. * @@ -315,71 +152,44 @@ internal class CameraSessionController private constructor( cameraDeviceController: CameraDeviceController, outputs: List, dynamicRange: Long, - fpsRange: Range ): CameraSessionController = withContext(coroutineDispatcher) { - requestTargetMutex.withLock { - require(outputs.isNotEmpty()) { "At least one output is required" } - require(outputs.all { it.surface.isValid }) { "All outputs $outputs must be valid but ${outputs.filter { !it.surface.isValid }} is invalid" } + require(outputs.isNotEmpty()) { "At least one output is required" } + require(outputs.all { it.surface.isValid }) { "All outputs $outputs must be valid but ${outputs.filter { !it.surface.isValid }} is invalid" } - if ((dynamicRange == this@CameraSessionController.dynamicRange) && (outputs == this@CameraSessionController.outputs) && !isClosed) { - Logger.w(TAG, "Same dynamic range and outputs, returning the same controller") - return@withContext this@CameraSessionController - } - - // Re-add targets that are in the new outputs (identified by their name) - val targets = outputs.filter { - captureRequestBuilder.hasTarget(it.name) - } - captureRequestBuilder.clearTargets() - captureRequestBuilder.addTargets(targets) - val minFrameDuration = 1_000_000_000 / fpsRange.upper.toLong() - captureRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange) - captureRequestBuilder.set(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) - - // Close current session - close() - - val isClosedFlow = MutableStateFlow(false) - val newCaptureSession = - CameraUtils.createCaptureSession( - cameraDeviceController, - outputs.map { it.surface }, - dynamicRange, - isClosedFlow - ) + // Close current session + close() - val controller = CameraSessionController( - coroutineDispatcher, - newCaptureSession, - captureRequestBuilder, - sessionCallback, - sessionCompat, - outputs, + val isClosedFlow = MutableStateFlow(false) + val newCaptureSession = + CameraUtils.createCaptureSession( + cameraDeviceController, + outputs.map { it.surface }, dynamicRange, - cameraDeviceController.isClosedFlow, - isClosedFlow.asStateFlow() + isClosedFlow ) - if (!captureRequestBuilder.isEmpty()) { - controller.setRepeatingSession() - } + val controller = CameraSessionController( + coroutineDispatcher, + newCaptureSession, + sessionCallback, + sessionCompat, + cameraDeviceController.isClosedFlow, + isClosedFlow.asStateFlow() + ) - controller - } + controller } companion object { private const val TAG = "CameraSessionController" suspend fun create( - coroutineDispatcher: CoroutineDispatcher, cameraDeviceController: CameraDeviceController, sessionCallback: CameraSessionCallback, sessionCompat: ICameraCaptureSessionCompat, outputs: List, dynamicRange: Long, - fpsRange: Range, - defaultRequestBuilder: CaptureRequestWithTargetsBuilder.() -> Unit = {} + coroutineDispatcher: CoroutineDispatcher ): CameraSessionController { require(outputs.isNotEmpty()) { "At least one output is required" } require(outputs.all { it.surface.isValid }) { "All outputs $outputs must be valid but ${outputs.filter { !it.surface.isValid }} is invalid" } @@ -393,22 +203,11 @@ internal class CameraSessionController private constructor( isClosedFlow ) - val captureRequestBuilder = CaptureRequestWithTargetsBuilder.create( - cameraDeviceController - ).apply { - val minFrameDuration = 1_000_000_000 / fpsRange.upper.toLong() - set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange) - set(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) - defaultRequestBuilder() - } return CameraSessionController( coroutineDispatcher, captureSession, - captureRequestBuilder, sessionCallback, sessionCompat, - outputs, - dynamicRange, cameraDeviceController.isClosedFlow, isClosedFlow.asStateFlow() ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt index 8104e437b..ff2fef991 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt @@ -15,20 +15,17 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils -import android.hardware.camera2.CameraDevice import android.hardware.camera2.CaptureRequest import android.hardware.camera2.CaptureRequest.Builder import android.view.Surface -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraDeviceController -import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock /** * A class to store for [CaptureRequest] with targets */ -internal class CaptureRequestWithTargetsBuilder private constructor( - private val captureRequestBuilder: Builder +internal class CaptureRequestWithTargetsBuilder( + private var captureRequestBuilder: Builder ) { private val mutableTargets = mutableSetOf() private val mutex = Mutex() @@ -64,12 +61,14 @@ internal class CaptureRequestWithTargetsBuilder private constructor( } /** - * Adds a target to the CaptureRequest + * Adds a target to the CaptureRequest from a [CameraSurface] * * @param cameraSurface The surface to add * @return true if the surface was added, false otherwise */ suspend fun addTarget(cameraSurface: CameraSurface): Boolean = mutex.withLock { + require(cameraSurface.surface.isValid) { "Target $cameraSurface must be valid" } + val wasAdded = mutableTargets.add(cameraSurface) if (wasAdded) { captureRequestBuilder.addTarget(cameraSurface.surface) @@ -84,6 +83,9 @@ internal class CaptureRequestWithTargetsBuilder private constructor( * @return true if the surface was added, false otherwise */ suspend fun addTargets(cameraSurfaces: List) = mutex.withLock { + require(cameraSurfaces.isNotEmpty()) { "At least one target is required" } + require(cameraSurfaces.all { it.surface.isValid }) { "All targets must be valid" } + cameraSurfaces.forEach { val wasAdded = mutableTargets.add(it) if (wasAdded) { @@ -139,17 +141,4 @@ internal class CaptureRequestWithTargetsBuilder private constructor( override fun toString(): String { return "$captureRequestBuilder with targets: $targets" } - - companion object { - /** - * Create a CaptureRequestBuilderWithTargets - * - * @param cameraDeviceController The camera device controller - * @param template The template to use - */ - fun create( - cameraDeviceController: CameraDeviceController, - template: Int = CameraDevice.TEMPLATE_RECORD, - ) = CaptureRequestWithTargetsBuilder(cameraDeviceController.createCaptureRequest(template)) - } } \ No newline at end of file From 8fcc52d5877ec76644b2da4ae394e00da58e8e01 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Mar 2026 15:57:01 +0100 Subject: [PATCH 42/72] feat(core): camera: add an API to reset all the camera settings --- .../sources/video/camera/CameraSettings.kt | 63 +++++++++++++++++-- .../sources/video/camera/CameraSource.kt | 3 +- .../camera/controllers/CameraController.kt | 49 +++++++++++---- .../utils/CaptureRequestWithTargetsBuilder.kt | 8 ++- .../app/ui/main/PreviewViewModel.kt | 10 +++ 5 files changed, 110 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index cee9d7437..e9b9fce1e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -15,6 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.video.camera +import android.Manifest import android.content.Context import android.graphics.PointF import android.graphics.Rect @@ -35,6 +36,7 @@ import android.util.Range import android.util.Rational import androidx.annotation.IntRange import androidx.annotation.RequiresApi +import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.is90or270 import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraController import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoExposureModes @@ -188,6 +190,7 @@ class CameraSettings internal constructor( * @param key the key to get * @return the value associated with the key */ + @RequiresPermission(Manifest.permission.CAMERA) fun get(key: CaptureRequest.Key) = cameraController.getSetting(key) /** @@ -198,7 +201,8 @@ class CameraSettings internal constructor( * @param key the key to set * @param value the value to set */ - fun set(key: CaptureRequest.Key, value: T) = + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun set(key: CaptureRequest.Key, value: T) = cameraController.setSetting(key, value) /** @@ -208,6 +212,7 @@ class CameraSettings internal constructor( * * @return the total capture result */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun applyRepeatingSessionSync(): TotalCaptureResult { val deferred = CompletableDeferred() @@ -226,6 +231,7 @@ class CameraSettings internal constructor( * * @param onCaptureResult the capture result callback. Return `true` to stop the callback. */ + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun applyRepeatingSession(onCaptureResult: CaptureResultListener) { val tag = TagBundle.Factory.default.create() val captureCallback = object : CaptureResultListener { @@ -240,19 +246,22 @@ class CameraSettings internal constructor( } cameraController.addCaptureCallbackListener(captureCallback) - cameraController.setRepeatingSession(tag) + cameraController.applyRepeatingSession(tag) } /** * Applies settings to the camera repeatedly. */ - suspend fun applyRepeatingSession() = cameraController.setRepeatingSession() + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun applyRepeatingSession() = cameraController.applyRepeatingSession() /** * Resets all settings to default. */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun resetAll() { - + cameraController.resetSettings() + applyRepeatingSession() } private class TagBundle(val keyId: Long) { @@ -312,8 +321,10 @@ class CameraSettings internal constructor( /** * @return `true` if flash is already on, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) get() = getFlash() == CaptureResult.FLASH_MODE_TORCH + @RequiresPermission(Manifest.permission.CAMERA) private fun getFlash(): Int = cameraSettings.get(CaptureRequest.FLASH_MODE) ?: CaptureResult.FLASH_MODE_OFF @@ -322,6 +333,7 @@ class CameraSettings internal constructor( * * @param isEnable `true` to enable flash, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setIsEnable(isEnable: Boolean) { if (isEnable) { setFlash(CaptureRequest.FLASH_MODE_TORCH) @@ -337,6 +349,7 @@ class CameraSettings internal constructor( * * @param mode flash mode */ + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun setFlash(mode: Int) { cameraSettings.set(CaptureRequest.FLASH_MODE, mode) cameraSettings.applyRepeatingSession() @@ -376,6 +389,7 @@ class CameraSettings internal constructor( * @return the flash strength */ @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.FLASH_STRENGTH_LEVEL) ?: DEFAULT_STRENGTH_LEVEL @@ -386,6 +400,7 @@ class CameraSettings internal constructor( * @see [availableStrengthLevelRange] */ @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setStrengthLevel(level: Int) { cameraSettings.set(CaptureRequest.FLASH_STRENGTH_LEVEL, level) cameraSettings.applyRepeatingSession() @@ -419,6 +434,7 @@ class CameraSettings internal constructor( * * @return current camera auto white balance mode */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AWB_MODE) ?: CaptureResult.CONTROL_AWB_MODE_OFF @@ -428,6 +444,7 @@ class CameraSettings internal constructor( * @param autoMode auto white balance mode * @see [availableAutoModes] */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setAutoMode(autoMode: Int) { cameraSettings.set(CaptureRequest.CONTROL_AWB_MODE, autoMode) cameraSettings.applyRepeatingSession() @@ -442,12 +459,14 @@ class CameraSettings internal constructor( * Gets the white balance metering regions. */ val meteringRegions: List + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AWB_REGIONS)?.toList() ?: emptyList() /** * Sets the white balance metering regions. */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setMeteringRegions(value: List) { cameraSettings.set( CaptureRequest.CONTROL_AWB_REGIONS, value.toTypedArray() @@ -459,6 +478,7 @@ class CameraSettings internal constructor( * Gets the auto white balance lock state. */ val isLocked: Boolean + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AWB_LOCK) ?: false /** @@ -466,6 +486,7 @@ class CameraSettings internal constructor( * * @param isLocked the lock state. `true` to lock auto white balance, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setIsLocked(isLocked: Boolean) { cameraSettings.set(CaptureRequest.CONTROL_AWB_LOCK, isLocked) cameraSettings.applyRepeatingSession() @@ -498,6 +519,7 @@ class CameraSettings internal constructor( * * @return the sensitivity */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.SENSOR_SENSITIVITY) ?: DEFAULT_SENSITIVITY @@ -507,6 +529,7 @@ class CameraSettings internal constructor( * * @param sensorSensitivity the sensitivity */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setSensorSensitivity(sensorSensitivity: Int) { cameraSettings.set( CaptureRequest.SENSOR_SENSITIVITY, @@ -543,6 +566,7 @@ class CameraSettings internal constructor( * @see [WhiteBalance.autoMode] */ val rggbChannelVector: RggbChannelVector? + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.COLOR_CORRECTION_GAINS) /** @@ -552,6 +576,7 @@ class CameraSettings internal constructor( * * @param rggbChannelVector the color correction gain */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setRggbChannelVector(rggbChannelVector: RggbChannelVector) { cameraSettings.set( CaptureRequest.CONTROL_AWB_MODE, @@ -599,6 +624,7 @@ class CameraSettings internal constructor( * * @return the auto exposure mode */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AE_MODE) ?: CaptureResult.CONTROL_AE_MODE_OFF @@ -607,6 +633,7 @@ class CameraSettings internal constructor( * * @param autoMode the exposure auto mode */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setAutoMode(autoMode: Int) { cameraSettings.set(CaptureRequest.CONTROL_AE_MODE, autoMode) cameraSettings.applyRepeatingSession() @@ -653,6 +680,7 @@ class CameraSettings internal constructor( * * @return the exposure compensation */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AE_EXPOSURE_COMPENSATION) ?: DEFAULT_COMPENSATION @@ -661,6 +689,7 @@ class CameraSettings internal constructor( * * @param compensation the exposure compensation */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setCompensation(compensation: Int) { cameraSettings.set( CaptureRequest.CONTROL_AE_EXPOSURE_COMPENSATION, @@ -678,6 +707,7 @@ class CameraSettings internal constructor( * Gets the exposure metering regions. */ val meteringRegions: List + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AE_REGIONS)?.toList() ?: emptyList() @@ -686,6 +716,7 @@ class CameraSettings internal constructor( * * @param meteringRegions the metering regions */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setMeteringRegions(meteringRegions: List) { cameraSettings.set( CaptureRequest.CONTROL_AE_REGIONS, meteringRegions.toTypedArray() @@ -703,6 +734,7 @@ class CameraSettings internal constructor( * @return the auto exposure lock state */ val isLocked: Boolean + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AE_LOCK) ?: false /** @@ -710,6 +742,7 @@ class CameraSettings internal constructor( * * @param isLocked the lock state. `true` to lock auto exposure, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setIsLocked(isLocked: Boolean) { cameraSettings.set( CaptureRequest.CONTROL_AE_LOCK, isLocked @@ -789,6 +822,7 @@ class CameraSettings internal constructor( persistentZoomRatio } + @RequiresPermission(Manifest.permission.CAMERA) override suspend fun setZoomRatio(zoomRatio: Float) { mutex.withLock { val clampedValue = zoomRatio.clamp(availableRatioRange) @@ -859,11 +893,13 @@ class CameraSettings internal constructor( characteristics.zoomRatioRange ?: DEFAULT_ZOOM_RATIO_RANGE } + @RequiresPermission(Manifest.permission.CAMERA) override suspend fun getZoomRatio(): Float { return cameraSettings.get(CaptureRequest.CONTROL_ZOOM_RATIO) ?: DEFAULT_ZOOM_RATIO } + @RequiresPermission(Manifest.permission.CAMERA) override suspend fun setZoomRatio(zoomRatio: Float) { if (zoomRatio == getZoomRatio()) { return @@ -934,6 +970,7 @@ class CameraSettings internal constructor( * * @return the auto focus mode */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AF_MODE) ?: CaptureResult.CONTROL_AF_MODE_OFF @@ -942,6 +979,7 @@ class CameraSettings internal constructor( * * @param autoMode the auto focus mode */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setAutoMode(autoMode: Int) { cameraSettings.set(CaptureRequest.CONTROL_AF_MODE, autoMode) cameraSettings.applyRepeatingSession() @@ -969,6 +1007,7 @@ class CameraSettings internal constructor( * * @return the lens focus distance */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.LENS_FOCUS_DISTANCE) ?: DEFAULT_LENS_DISTANCE @@ -979,6 +1018,7 @@ class CameraSettings internal constructor( * * @param lensDistance the lens focus distance */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setLensDistance(lensDistance: Float) { cameraSettings.set( CaptureRequest.LENS_FOCUS_DISTANCE, lensDistance.clamp(availableLensDistanceRange) @@ -997,6 +1037,7 @@ class CameraSettings internal constructor( * Gets the focus metering regions. */ val meteringRegions: List + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AF_REGIONS)?.toList() ?: emptyList() @@ -1005,6 +1046,7 @@ class CameraSettings internal constructor( * * @param meteringRegions the metering regions */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setMeteringRegions(meteringRegions: List) { cameraSettings.set( CaptureRequest.CONTROL_AF_REGIONS, meteringRegions.toTypedArray() @@ -1035,6 +1077,7 @@ class CameraSettings internal constructor( * * @return `true` if video stabilization is enabled, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE) == CaptureResult.CONTROL_VIDEO_STABILIZATION_MODE_ON /** @@ -1042,6 +1085,7 @@ class CameraSettings internal constructor( * * @param isEnableVideo `true` to enable video stabilization, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setIsEnableVideo(isEnableVideo: Boolean) { if (isEnableVideo) { cameraSettings.set( @@ -1082,6 +1126,7 @@ class CameraSettings internal constructor( * * @return `true` if optical video stabilization is enabled, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE) == CaptureResult.LENS_OPTICAL_STABILIZATION_MODE_ON /** @@ -1089,6 +1134,7 @@ class CameraSettings internal constructor( * * @param isEnableOptical `true` to enable optical video stabilization, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setIsEnableOptical(isEnableOptical: Boolean) { if (isEnableOptical) { cameraSettings.set( @@ -1117,6 +1163,7 @@ class CameraSettings internal constructor( private var autoCancelHandle: Job? = null @Suppress("UNCHECKED_CAST") + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun cancelAfAeTrigger() { // Cancel previous AF trigger cameraSettings.set( @@ -1134,7 +1181,8 @@ class CameraSettings internal constructor( } @Suppress("UNCHECKED_CAST") - private fun addFocusMetering( + @RequiresPermission(Manifest.permission.CAMERA) + private suspend fun addFocusMetering( afRects: List, aeRects: List, awbRects: List @@ -1166,6 +1214,7 @@ class CameraSettings internal constructor( } @Suppress("UNCHECKED_CAST") + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun triggerAf(overrideAeMode: Boolean) { cameraSettings.set( CaptureRequest.CONTROL_AF_TRIGGER, @@ -1208,6 +1257,7 @@ class CameraSettings internal constructor( deferred.await() } + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun executeMetering( afRectangles: List, aeRectangles: List, @@ -1251,6 +1301,7 @@ class CameraSettings internal constructor( } } + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun startFocusAndMetering( afPoints: List, aePoints: List, @@ -1408,6 +1459,7 @@ class CameraSettings internal constructor( * @param fovRotationDegree the orientation of the field of view * @param timeoutDurationMs duration in milliseconds after which the focus and metering will be cancelled automatically */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun onTap( context: Context, afPoints: List, @@ -1441,6 +1493,7 @@ class CameraSettings internal constructor( /** * Cancel the focus and metering. */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun cancelFocusAndMetering() { disableAutoCancel() diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt index 4a9216483..953dcd700 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt @@ -119,8 +119,7 @@ internal class CameraSource( private fun defaultCaptureRequest( captureRequest: CaptureRequestWithTargetsBuilder ) { - if (settings.focus.availableAutoModes.contains(CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO) - ) { + if (settings.focus.availableAutoModes.contains(CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO)) { captureRequest.set( CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index 4bec131bb..052e62d5d 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -210,6 +210,7 @@ internal class CameraController( } } + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun applySessionController( sessionController: CameraSessionController ) { @@ -232,7 +233,7 @@ internal class CameraController( } } - setRepeatingSessionUnsafe() + applyRepeatingSessionUnsafe() } /** @@ -277,7 +278,7 @@ internal class CameraController( val wasAdded = getCaptureRequestBuilder().addTarget(target) if (wasAdded) { try { - setRepeatingSessionUnsafe() + applyRepeatingSessionUnsafe() } catch (t: Throwable) { Logger.e(TAG, "Error to add target $name", t) } @@ -302,7 +303,7 @@ internal class CameraController( return@withLock } try { - setRepeatingSessionUnsafe() + applyRepeatingSessionUnsafe() } catch (t: Throwable) { Logger.e(TAG, "Error to remove target $name", t) } @@ -314,23 +315,43 @@ internal class CameraController( /** * Gets a setting from the current capture request. */ + @RequiresPermission(Manifest.permission.CAMERA) fun getSetting(key: CaptureRequest.Key): T? { val captureRequestBuilder = requireNotNull(captureRequestBuilder) { "CaptureRequestBuilder is null" } return captureRequestBuilder.get(key) } + /** + * Resets the current capture request to its default settings. + * + * Don't forget to call [applyRepeatingSession] to apply the setting. + */ + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun resetSettings() { + withContext(defaultDispatcher) { + controllerMutex.withLock { + captureRequestBuilder ?: return@withLock + val currentCaptureRequestBuilder = getCaptureRequestBuilder() + // Get targets of previous builder + val targets = currentCaptureRequestBuilder.getTargets() + captureRequestBuilder = null + getCaptureRequestBuilder().addTargets(targets) + } + } + } + /** * Sets a setting to the current capture request. * - * Don't forget to call [setRepeatingSession] to apply the setting. + * Don't forget to call [applyRepeatingSession] to apply the setting. * * @param key The setting key * @param value The setting value */ - fun setSetting(key: CaptureRequest.Key, value: T) { - val captureRequestBuilder = - requireNotNull(captureRequestBuilder) { "CaptureRequestBuilder is null" } + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun setSetting(key: CaptureRequest.Key, value: T) { + val captureRequestBuilder = getCaptureRequestBuilder() captureRequestBuilder.set(key, value) } @@ -357,7 +378,7 @@ internal class CameraController( captureRequestBuilder.set(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) if (isActiveFlow.value) { - setRepeatingSessionUnsafe() + applyRepeatingSessionUnsafe() } } } @@ -409,7 +430,8 @@ internal class CameraController( * * @param tag A tag to associate with the session. */ - suspend fun setRepeatingSessionUnsafe(tag: Any? = null) { + @RequiresPermission(Manifest.permission.CAMERA) + private suspend fun applyRepeatingSessionUnsafe(tag: Any? = null) { val sessionController = getSessionController() val captureRequestBuilder = getCaptureRequestBuilder() tag?.let { @@ -424,26 +446,27 @@ internal class CameraController( * * @param tag A tag to associate with the session. */ - suspend fun setRepeatingSession(tag: Any? = null) { + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun applyRepeatingSession(tag: Any? = null) { withContext(defaultDispatcher) { controllerMutex.withLock { - setRepeatingSessionUnsafe(tag) + applyRepeatingSessionUnsafe(tag) } } } private suspend fun closeControllers() { sessionController?.close() + sessionController = null deviceController?.close() deviceController = null + captureRequestBuilder = null } suspend fun release() { withContext(defaultDispatcher) { controllerMutex.withLock { closeControllers() - captureRequestBuilder = null - sessionController = null } } outputs.clear() diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt index ff2fef991..1bd541b3f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt @@ -31,9 +31,11 @@ internal class CaptureRequestWithTargetsBuilder( private val mutex = Mutex() /** - * The targets of the CaptureRequest + * Gets the targets from the CaptureRequest + * + * @return The targets */ - private val targets = mutableTargets.toList() + suspend fun getTargets() = mutex.withLock { mutableTargets.toList() } /** * Whether the CaptureRequest has no target @@ -139,6 +141,6 @@ internal class CaptureRequestWithTargetsBuilder( fun build() = captureRequestBuilder.build() override fun toString(): String { - return "$captureRequestBuilder with targets: $targets" + return "$captureRequestBuilder with targets: $mutableTargets" } } \ No newline at end of file diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index 9dcb6c070..13c97025b 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.app.ui.main import android.Manifest +import android.annotation.SuppressLint import android.app.Application import android.content.Context import android.content.pm.PackageManager @@ -408,6 +409,8 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod val isCameraSource = streamer.videoInput.sourceFlow?.map { it is ICameraSource }?.asLiveData() val isFlashAvailable = MutableLiveData(false) + + @SuppressLint("MissingPermission") fun toggleFlash() { cameraSettings?.let { viewModelScope.launch { @@ -417,6 +420,8 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } val isAutoWhiteBalanceAvailable = MutableLiveData(false) + + @SuppressLint("MissingPermission") fun toggleAutoWhiteBalanceMode() { cameraSettings?.let { settings -> val awbModes = settings.whiteBalance.availableAutoModes @@ -444,6 +449,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod 0f } } + @SuppressLint("MissingPermission") set(value) { cameraSettings?.let { settings -> settings.exposure.let { @@ -487,6 +493,8 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } val isAutoFocusModeAvailable = MutableLiveData(false) + + @SuppressLint("MissingPermission") fun toggleAutoFocusMode() { cameraSettings?.let { val afModes = it.focus.availableAutoModes @@ -515,6 +523,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod 0f } } + @SuppressLint("MissingPermission") set(value) { cameraSettings?.let { settings -> settings.focus.let { @@ -541,6 +550,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } + @SuppressLint("MissingPermission") private fun notifyCameraChanged(videoSource: ICameraSource) { val settings = videoSource.settings // Set optical stabilization first From 82845e6bd1b889833a52920abbc690fae136d735 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:31:07 +0100 Subject: [PATCH 43/72] fix(core): lifecycle: correctly use NonCancellable --- .../lifecycle/StreamerViewModelLifeCycleObserver.kt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerViewModelLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerViewModelLifeCycleObserver.kt index ad42887a8..12dd7b9c4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerViewModelLifeCycleObserver.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerViewModelLifeCycleObserver.kt @@ -22,6 +22,7 @@ import io.github.thibaultbee.streampack.core.interfaces.ICloseableStreamer import io.github.thibaultbee.streampack.core.interfaces.IStreamer import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext /** * A [DefaultLifecycleObserver] to control a streamer on [Activity] lifecycle in a ViewModel. @@ -37,10 +38,12 @@ open class StreamerViewModelLifeCycleObserver(protected val streamer: IStreamer) DefaultLifecycleObserver { override fun onPause(owner: LifecycleOwner) { - owner.lifecycleScope.launch(NonCancellable) { - streamer.stopStream() - if (streamer is ICloseableStreamer) { - streamer.close() + owner.lifecycleScope.launch { + withContext(NonCancellable) { + streamer.stopStream() + if (streamer is ICloseableStreamer) { + streamer.close() + } } } } From e8eca7ca433e91cd3d9c33de3d231f2f7b3a770d Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sun, 8 Mar 2026 13:34:25 +0100 Subject: [PATCH 44/72] fix(core): properly returns media sink type exception --- .../streampack/core/elements/endpoints/MediaSinkType.kt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt index f54b85d94..81328f3c6 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt @@ -22,11 +22,8 @@ enum class MediaSinkType(val schemes: Set) { if (isLocalFileScheme(scheme)) { return FILE } - try { - return entries.first { it.schemes.contains(scheme) } - } catch (t: Throwable) { - throw IllegalArgumentException("Unknown scheme: $scheme", t) - } + return entries.firstOrNull { it.schemes.contains(scheme) } + ?: throw IllegalArgumentException("Unknown scheme: $scheme") } private fun isLocalFileScheme(scheme: String?): Boolean { From be26c6251739ef5b366ddd2d6d37367319627fe9 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sun, 8 Mar 2026 13:35:52 +0100 Subject: [PATCH 45/72] fix(rtmp): fix few warnings in the media descriptor --- .../mediadescriptor/RtmpMediaDescriptor.kt | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/configuration/mediadescriptor/RtmpMediaDescriptor.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/configuration/mediadescriptor/RtmpMediaDescriptor.kt index 2815c09a8..45231f401 100644 --- a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/configuration/mediadescriptor/RtmpMediaDescriptor.kt +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/configuration/mediadescriptor/RtmpMediaDescriptor.kt @@ -26,7 +26,7 @@ import java.security.InvalidParameterException /** * Creates a RTMP connection descriptor from an [descriptor]. * If the descriptor is already a [RtmpMediaDescriptor], it will be returned as is. - * If the descriptor is an [UriMediaDescriptor], it will be converted to a [RtmpMediaDescriptor] with default [TSServiceInfo]. + * If the descriptor is an [UriMediaDescriptor], it will be converted to a [RtmpMediaDescriptor]. * Otherwise, an [InvalidParameterException] will be thrown. */ fun RtmpMediaDescriptor(descriptor: MediaDescriptor) = @@ -65,7 +65,7 @@ class RtmpMediaDescriptor( require(streamKey.isNotBlank()) { "Invalid streamKey $streamKey" } } - override val uri = Uri.Builder() + override val uri: Uri = Uri.Builder() .scheme(scheme) .encodedAuthority("$host:$port") .apply { @@ -109,12 +109,10 @@ class RtmpMediaDescriptor( val port = if (uri.port > 0) { uri.port } else { - if ((scheme == RTMPS_SCHEME) || (scheme == RTMPTS_SCHEME)) { - SSL_DEFAULT_PORT - } else if ((scheme == RTMPT_SCHEME) || (scheme == RTMPTE_SCHEME)) { - HTTP_DEFAULT_PORT - } else { - DEFAULT_PORT + when (scheme) { + RTMPS_SCHEME, RTMPTS_SCHEME -> SSL_DEFAULT_PORT + RTMPT_SCHEME, RTMPTE_SCHEME -> HTTP_DEFAULT_PORT + else -> DEFAULT_PORT } } if (uri.pathSegments.isEmpty()) { From 07ef6f56a729dd7ab0e98bd326c09c9c8a3932ee Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sun, 8 Mar 2026 20:35:36 +0100 Subject: [PATCH 46/72] fix(rtmp): to not push frame to the muxer if the connection is closed. --- .../endpoints/composites/muxer/utils/FlvTagBuilder.kt | 2 +- .../streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt index a450972d7..afb98d8f9 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt @@ -138,7 +138,7 @@ class FlvTagBuilder(val channel: ChannelWithCloseableData) { } } - companion object Companion { + companion object { private const val TAG = "FlvTagBuilder" private const val AUDIO_STREAM_PID = 0 diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt index eab0e0eb9..bee91e202 100644 --- a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt @@ -166,6 +166,10 @@ class RtmpEndpoint internal constructor( override suspend fun write( frame: Frame, streamPid: Int ) { + if (!_isOpenFlow.value) { + frame.close() + return + } val startUpTimestamp = getStartUpTimestamp(frame.ptsInUs) val ts = (frame.ptsInUs - startUpTimestamp) / 1000 flvTagBuilder.write(frame, ts.toInt(), streamPid) From e6c0fcc505203a4e9e979ab7e84fde04f453527a Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:13:33 +0100 Subject: [PATCH 47/72] feat(core): add a specific error message when scheme or extension are empty --- .../streampack/core/elements/endpoints/MediaContainerType.kt | 3 +++ .../streampack/core/elements/endpoints/MediaSinkType.kt | 3 +++ 2 files changed, 6 insertions(+) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaContainerType.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaContainerType.kt index b9821d2ab..9284f961c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaContainerType.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaContainerType.kt @@ -15,6 +15,9 @@ enum class MediaContainerType(val values: Set) { companion object { private fun inferFromExtension(extension: String): MediaContainerType { + if (extension.isEmpty()) { + throw IllegalArgumentException("Extension cannot be empty") + } val type = entries.firstOrNull { it.values.contains(extension) } return type ?: throw IllegalArgumentException("Unsupported extension: $extension") } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt index 81328f3c6..939a6af6d 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt @@ -22,6 +22,9 @@ enum class MediaSinkType(val schemes: Set) { if (isLocalFileScheme(scheme)) { return FILE } + if (scheme.isNullOrEmpty()) { + throw IllegalArgumentException("Scheme cannot be empty") + } return entries.firstOrNull { it.schemes.contains(scheme) } ?: throw IllegalArgumentException("Unknown scheme: $scheme") } From eef110642fc38eaf21b06683d607eb634aa99003 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 11 Mar 2026 15:51:48 +0100 Subject: [PATCH 48/72] refactor(core): lifecycle: merge life cycle observer in a single class --- .../StreamerActivityLifeCycleObserver.kt | 39 ------------------- ...server.kt => StreamerLifeCycleObserver.kt} | 21 +++++++--- .../streampack/app/ui/main/PreviewFragment.kt | 6 +-- 3 files changed, 18 insertions(+), 48 deletions(-) delete mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerActivityLifeCycleObserver.kt rename core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/{StreamerViewModelLifeCycleObserver.kt => StreamerLifeCycleObserver.kt} (69%) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerActivityLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerActivityLifeCycleObserver.kt deleted file mode 100644 index abab6d9d0..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerActivityLifeCycleObserver.kt +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2022 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.streamers.lifecycle - -import androidx.lifecycle.DefaultLifecycleObserver -import androidx.lifecycle.LifecycleOwner -import io.github.thibaultbee.streampack.core.interfaces.IStreamer -import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking - -/** - * A [DefaultLifecycleObserver] to control a streamer on [Activity] lifecycle. - * - * It stops streamer when application goes to background and release it when application is destroyed. - * - * To use it, call: - * - `lifeCycle.addObserver(StreamerActivityLifeCycleObserver(streamer))` - * - * @param streamer The streamer to control - */ -open class StreamerActivityLifeCycleObserver(streamer: IStreamer) : - StreamerViewModelLifeCycleObserver(streamer) { - - override fun onDestroy(owner: LifecycleOwner) { - streamer.releaseBlocking() - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerViewModelLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt similarity index 69% rename from core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerViewModelLifeCycleObserver.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt index 12dd7b9c4..c779c909b 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerViewModelLifeCycleObserver.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Thibault B. + * Copyright (C) 2022 Thibault B. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,23 +20,26 @@ import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.lifecycleScope import io.github.thibaultbee.streampack.core.interfaces.ICloseableStreamer import io.github.thibaultbee.streampack.core.interfaces.IStreamer +import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.launch import kotlinx.coroutines.withContext /** - * A [DefaultLifecycleObserver] to control a streamer on [Activity] lifecycle in a ViewModel. + * A [DefaultLifecycleObserver] to control a streamer on [Activity] lifecycle. * - * It stops streamer when application goes to background. + * It stops streamer when application goes to background and release it when application is destroyed. * * To use it, call: * - `lifeCycle.addObserver(StreamerActivityLifeCycleObserver(streamer))` * * @param streamer The streamer to control + * @param releaseOnDestroy Whether to release the streamer when application is destroyed. If a view model is used, this should be set to false. */ -open class StreamerViewModelLifeCycleObserver(protected val streamer: IStreamer) : - DefaultLifecycleObserver { - +open class StreamerLifeCycleObserver( + private val streamer: IStreamer, + private val releaseOnDestroy: Boolean = false +) : DefaultLifecycleObserver { override fun onPause(owner: LifecycleOwner) { owner.lifecycleScope.launch { withContext(NonCancellable) { @@ -47,4 +50,10 @@ open class StreamerViewModelLifeCycleObserver(protected val streamer: IStreamer) } } } + + override fun onDestroy(owner: LifecycleOwner) { + if (releaseOnDestroy) { + streamer.releaseBlocking() + } + } } \ No newline at end of file diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt index a7d2d9aca..3f4296514 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt @@ -37,13 +37,13 @@ import io.github.thibaultbee.streampack.app.utils.PermissionManager import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource -import io.github.thibaultbee.streampack.core.streamers.lifecycle.StreamerViewModelLifeCycleObserver +import io.github.thibaultbee.streampack.core.streamers.lifecycle.StreamerLifeCycleObserver import io.github.thibaultbee.streampack.ui.views.PreviewView import kotlinx.coroutines.launch class PreviewFragment : Fragment(R.layout.main_fragment) { private lateinit var binding: MainFragmentBinding - private var previousStreamerLifecycleObserver: StreamerViewModelLifeCycleObserver? = null + private var previousStreamerLifecycleObserver: StreamerLifeCycleObserver? = null private val previewViewModel: PreviewViewModel by viewModels { PreviewViewModelFactory(requireActivity().application) @@ -111,7 +111,7 @@ class PreviewFragment : Fragment(R.layout.main_fragment) { if (previousStreamerLifecycleObserver != null) { lifecycle.removeObserver(previousStreamerLifecycleObserver!!) } - val newObserver = StreamerViewModelLifeCycleObserver(streamer).apply { + val newObserver = StreamerLifeCycleObserver(streamer).apply { previousStreamerLifecycleObserver = this } lifecycle.addObserver(newObserver) From d5cdb7de0c82be454a5ef477185b6c9212fd634f Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 11 Mar 2026 15:48:29 +0100 Subject: [PATCH 49/72] feat(core): audioInput: add a specific start capture to run the audio without the stream like to display a vumeter --- .../elements/processing/RawFramePullPush.kt | 64 +++++----- .../core/pipelines/inputs/AudioInput.kt | 115 ++++++++++++++++-- 2 files changed, 141 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt index aa5bfedd5..4aeb1a004 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt @@ -68,40 +68,48 @@ class RawFramePullPush( } } + private suspend fun CoroutineScope.runProcessing() { + while (isActive) { + val rawFrame = mutex.withLock { + val unwrapSource = source ?: return@withLock null + try { + val buffer = bufferPool.get(unwrapSource.minBufferSize) + val timestampInUs = unwrapSource.fillAudioFrame(buffer) + pool.get(buffer, timestampInUs) + } catch (t: Throwable) { + Logger.e(TAG, "Failed to get frame: ${t.message}") + null + } + } + if (rawFrame == null) { + continue + } + + // Process buffer with effects + val processedFrame = try { + frameProcessor.process(rawFrame) + } catch (t: Throwable) { + Logger.e(TAG, "Failed to pre-process frame: ${t.message}") + continue + } + + // Store for outputs + onFrame(processedFrame) + } + Logger.e(TAG, "Processing loop ended") + } + fun startStream() { if (isReleaseRequested.get()) { Logger.w(TAG, "Already released") return } + if (job != null) { + Logger.w(TAG, "Already running") + return + } job = coroutineScope.launch { - while (isActive) { - val rawFrame = mutex.withLock { - val unwrapSource = source ?: return@withLock null - try { - val buffer = bufferPool.get(unwrapSource.minBufferSize) - val timestampInUs = unwrapSource.fillAudioFrame(buffer) - pool.get(buffer, timestampInUs) - } catch (t: Throwable) { - Logger.e(TAG, "Failed to get frame: ${t.message}") - null - } - } - if (rawFrame == null) { - continue - } - - // Process buffer with effects - val processedFrame = try { - frameProcessor.process(rawFrame) - } catch (t: Throwable) { - Logger.e(TAG, "Failed to pre-process frame: ${t.message}") - continue - } - - // Store for outputs - onFrame(processedFrame) - } - Logger.e(TAG, "Processing loop ended") + runProcessing() } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 999bc16e8..27f8f79fb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -57,6 +57,12 @@ interface IAudioInput { */ val isStreamingFlow: StateFlow + /** + * Whether the audio input is capturing. + * Capturing means the audio source and processor are running but frames might not be pushed to the encoder. + */ + val isCapturingFlow: StateFlow + /** * The audio source. * It allows access to advanced audio settings. @@ -81,6 +87,16 @@ interface IAudioInput { */ suspend fun setSource(audioSourceFactory: IAudioSourceInternal.Factory) + /** + * Starts the audio capture for dry-mode (does not push to encoders). + */ + suspend fun startCapture() + + /** + * Stops the audio capture. + */ + suspend fun stopCapture() + /** * Whether the audio input has a configuration. * It is true if the audio source has been configured. @@ -132,6 +148,16 @@ internal class AudioInput( } } + // STATE + /** + * Whether the audio input is streaming. + */ + private val _isStreamingFlow = MutableStateFlow(false) + override val isStreamingFlow = _isStreamingFlow.asStateFlow() + + private val _isCapturingFlow = MutableStateFlow(false) + override val isCapturingFlow = _isCapturingFlow.asStateFlow() + // PROCESSOR private val bufferPool = ByteBufferPool(true) @@ -141,7 +167,14 @@ internal class AudioInput( private val processorInternal = AudioFrameProcessor(bufferPool, dispatcherProvider.default) override val processor: IAudioFrameProcessor = processorInternal private val port = if (config is PushConfig) { - PushAudioPort(processorInternal, config, bufferPool, dispatcherProvider) + val wrappedConfig = PushConfig { frame -> + if (_isStreamingFlow.value) { + config.onFrame(frame) + } else { + frame.close() + } + } + PushAudioPort(processorInternal, wrappedConfig, bufferPool, dispatcherProvider) } else { CallbackAudioPort(processorInternal) // No threading needed, called from encoder thread } @@ -160,13 +193,6 @@ internal class AudioInput( override val withConfig: Boolean get() = sourceConfig != null - // STATE - /** - * Whether the audio input is streaming. - */ - private val _isStreamingFlow = MutableStateFlow(false) - override val isStreamingFlow = _isStreamingFlow.asStateFlow() - /** * Sets a new audio source. * @@ -197,9 +223,10 @@ internal class AudioInput( isStreamingJob += coroutineScope.launch { newAudioSource.isStreamingFlow.collect { isStreaming -> - if ((!isStreaming) && isStreamingFlow.value) { + if ((!isStreaming) && (isStreamingFlow.value || isCapturingFlow.value)) { Logger.i(TAG, "Audio source has been stopped.") - stopStream() + if (isStreamingFlow.value) stopStream() + if (isCapturingFlow.value) stopCapture() } } } @@ -296,6 +323,41 @@ internal class AudioInput( } } + override suspend fun startCapture() { + require(port is PushAudioPort) { "Audio capture is not supported in this mode: ${port::class.java}" } + + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } + + withContext(dispatcherProvider.default) { + sourceMutex.withLock { + val source = requireNotNull(sourceInternalFlow.value) { + "Audio source is not set yet" + } + if (isCapturingFlow.value) { + Logger.w(TAG, "Capture is already running") + return@withContext + } + if (!withConfig) { + Logger.w(TAG, "Audio source configuration is not set yet") + } + source.startStream() + try { + port.startStream() + } catch (t: Throwable) { + Logger.w( + TAG, + "startCapture: Can't start audio processor: ${t.message}" + ) + source.stopStream() + throw t + } + _isCapturingFlow.emit(true) + } + } + } + internal suspend fun stopStream() { if (isReleaseRequested.get()) { throw IllegalStateException("Input is released") @@ -303,7 +365,9 @@ internal class AudioInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (!isStreamingFlow.value) return@withContext _isStreamingFlow.emit(false) + if (isCapturingFlow.value) return@withContext try { port.stopStream() } catch (t: Throwable) { @@ -324,6 +388,37 @@ internal class AudioInput( } } + override suspend fun stopCapture() { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } + + withContext(dispatcherProvider.default) { + sourceMutex.withLock { + if (!isCapturingFlow.value) return@withContext + _isCapturingFlow.emit(false) + if (isStreamingFlow.value) return@withContext + + try { + port.stopStream() + } catch (t: Throwable) { + Logger.w( + TAG, + "stopCapture: Can't stop audio processor: ${t.message}" + ) + } + try { + sourceInternalFlow.value?.stopStream() + } catch (t: Throwable) { + Logger.w( + TAG, + "stopCapture: Can't stop audio source: ${t.message}" + ) + } + } + } + } + internal suspend fun release() { if (isReleaseRequested.getAndSet(true)) { Logger.w(TAG, "Already released") From e8babf255ae9f6a0731206f6ade2c874f1a0ee42 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:02:21 +0100 Subject: [PATCH 50/72] feat(core): lifecycle: add support for audio capture --- .../lifecycle/StreamerLifeCycleObserver.kt | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt index c779c909b..e07f9b149 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt @@ -20,6 +20,7 @@ import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.lifecycleScope import io.github.thibaultbee.streampack.core.interfaces.ICloseableStreamer import io.github.thibaultbee.streampack.core.interfaces.IStreamer +import io.github.thibaultbee.streampack.core.interfaces.IWithAudioSource import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.launch @@ -35,11 +36,25 @@ import kotlinx.coroutines.withContext * * @param streamer The streamer to control * @param releaseOnDestroy Whether to release the streamer when application is destroyed. If a view model is used, this should be set to false. + * @param autostartAudioCapture Whether to start audio capture when application is resumed. */ open class StreamerLifeCycleObserver( private val streamer: IStreamer, - private val releaseOnDestroy: Boolean = false + private val releaseOnDestroy: Boolean = false, + private val autostartAudioCapture: Boolean = false ) : DefaultLifecycleObserver { + override fun onResume(owner: LifecycleOwner) { + if (autostartAudioCapture) { + owner.lifecycleScope.launch { + withContext(NonCancellable) { + if (streamer is IWithAudioSource) { + streamer.audioInput.startCapture() + } + } + } + } + } + override fun onPause(owner: LifecycleOwner) { owner.lifecycleScope.launch { withContext(NonCancellable) { @@ -47,6 +62,9 @@ open class StreamerLifeCycleObserver( if (streamer is ICloseableStreamer) { streamer.close() } + if (streamer is IWithAudioSource) { + streamer.audioInput.stopCapture() + } } } } From 337d8c0626822b54424f2a7ba44b33a2e46fe1ba Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:22:16 +0100 Subject: [PATCH 51/72] fix(core): camera: add RequiresPermission annotation for onTap --- .../core/elements/sources/video/camera/CameraSettings.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index e9b9fce1e..a1392623a 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -1427,6 +1427,7 @@ class CameraSettings internal constructor( * @param fovRotationDegree the orientation of the field of view * @param timeoutDurationMs duration in milliseconds after which the focus and metering will be cancelled automatically */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun onTap( context: Context, point: PointF, From 1eac12a4292dff3e5c515a9a8b3349b81b5a7633 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:22:42 +0100 Subject: [PATCH 52/72] fix(core): processor: add the `MutableList` to the `IAudioFrameProcessor` --- .../core/elements/processing/audio/AudioFrameProcessor.kt | 4 ++-- .../core/elements/processing/audio/IAudioFrameProcessor.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index f10f6af43..efa6f9494 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -85,7 +85,7 @@ class AudioFrameProcessor( @Deprecated("'fun toArray(generator: IntFunction!>!): Array<(out) T!>!' is deprecated. This declaration is redundant in Kotlin and might be removed soon.") @Suppress("DEPRECATION") - override fun toArray(generator: IntFunction?>): Array { - return super.toArray(generator) + override fun toArray(generator: IntFunction?>): Array { + return super.toArray(generator) } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt index 19cc6ba9c..c21155385 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio /** * Public interface for audio frame processor. */ -interface IAudioFrameProcessor { +interface IAudioFrameProcessor : MutableList { /** * Whether the processor is muted. */ From ecc72fd84280d4006197de534d97ec757c5d2228 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:30:18 +0100 Subject: [PATCH 53/72] refactor(core): audioInput: factorize startCapture/startStream and stopCapture/stopStream --- .../elements/processing/RawFramePullPush.kt | 2 +- .../core/pipelines/inputs/AudioInput.kt | 132 ++++++++---------- 2 files changed, 61 insertions(+), 73 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt index 4aeb1a004..dc37e2d66 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt @@ -96,7 +96,7 @@ class RawFramePullPush( // Store for outputs onFrame(processedFrame) } - Logger.e(TAG, "Processing loop ended") + Logger.d(TAG, "Processing loop ended") } fun startStream() { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 27f8f79fb..4b1fab597 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -225,8 +225,8 @@ internal class AudioInput( newAudioSource.isStreamingFlow.collect { isStreaming -> if ((!isStreaming) && (isStreamingFlow.value || isCapturingFlow.value)) { Logger.i(TAG, "Audio source has been stopped.") - if (isStreamingFlow.value) stopStream() - if (isCapturingFlow.value) stopCapture() + stopStream() + stopCapture() } } } @@ -297,27 +297,11 @@ internal class AudioInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { - val source = requireNotNull(sourceInternalFlow.value) { - "Audio source is not set yet" - } if (isStreamingFlow.value) { Logger.w(TAG, "Stream is already running") return@withContext } - if (!withConfig) { - Logger.w(TAG, "Audio source configuration is not set yet") - } - source.startStream() - try { - port.startStream() - } catch (t: Throwable) { - Logger.w( - TAG, - "startStream: Can't start audio processor: ${t.message}" - ) - source.stopStream() - throw t - } + start("startStream") _isStreamingFlow.emit(true) } } @@ -325,39 +309,43 @@ internal class AudioInput( override suspend fun startCapture() { require(port is PushAudioPort) { "Audio capture is not supported in this mode: ${port::class.java}" } - + if (isReleaseRequested.get()) { throw IllegalStateException("Input is released") } withContext(dispatcherProvider.default) { sourceMutex.withLock { - val source = requireNotNull(sourceInternalFlow.value) { - "Audio source is not set yet" - } if (isCapturingFlow.value) { Logger.w(TAG, "Capture is already running") return@withContext } - if (!withConfig) { - Logger.w(TAG, "Audio source configuration is not set yet") - } - source.startStream() - try { - port.startStream() - } catch (t: Throwable) { - Logger.w( - TAG, - "startCapture: Can't start audio processor: ${t.message}" - ) - source.stopStream() - throw t - } + start("startCapture") _isCapturingFlow.emit(true) } } } + private suspend fun start(caller: String) { + val source = requireNotNull(sourceInternalFlow.value) { + "Audio source is not set yet" + } + if (!withConfig) { + Logger.w(TAG, "Audio source configuration is not set yet") + } + source.startStream() + try { + port.startStream() + } catch (t: Throwable) { + Logger.w( + TAG, + "$caller: Can't start audio processor: ${t.message}" + ) + source.stopStream() + throw t + } + } + internal suspend fun stopStream() { if (isReleaseRequested.get()) { throw IllegalStateException("Input is released") @@ -365,25 +353,16 @@ internal class AudioInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { - if (!isStreamingFlow.value) return@withContext - _isStreamingFlow.emit(false) - if (isCapturingFlow.value) return@withContext - try { - port.stopStream() - } catch (t: Throwable) { - Logger.w( - TAG, - "stopStream: Can't stop audio processor: ${t.message}" - ) + if (!isStreamingFlow.value) { + Logger.w(TAG, "Stream is already stopped") + return@withContext } - try { - sourceInternalFlow.value?.stopStream() - } catch (t: Throwable) { - Logger.w( - TAG, - "stopStream: Can't stop audio source: ${t.message}" - ) + _isStreamingFlow.emit(false) + if (isCapturingFlow.value) { + Logger.w(TAG, "Stopping capture before stopping stream") + return@withContext } + stop("stopStream") } } } @@ -395,30 +374,39 @@ internal class AudioInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { - if (!isCapturingFlow.value) return@withContext - _isCapturingFlow.emit(false) - if (isStreamingFlow.value) return@withContext - - try { - port.stopStream() - } catch (t: Throwable) { - Logger.w( - TAG, - "stopCapture: Can't stop audio processor: ${t.message}" - ) + if (!isCapturingFlow.value) { + Logger.w(TAG, "Capture is already stopped") + return@withContext } - try { - sourceInternalFlow.value?.stopStream() - } catch (t: Throwable) { - Logger.w( - TAG, - "stopCapture: Can't stop audio source: ${t.message}" - ) + _isCapturingFlow.emit(false) + if (isStreamingFlow.value) { + Logger.w(TAG, "Stopping stream before stopping capture") + return@withContext } + stop("stopCapture") } } } + private suspend fun stop(caller: String) { + try { + port.stopStream() + } catch (t: Throwable) { + Logger.w( + TAG, + "$caller: Can't stop audio processor: ${t.message}" + ) + } + try { + sourceInternalFlow.value?.stopStream() + } catch (t: Throwable) { + Logger.w( + TAG, + "$caller: Can't stop audio source: ${t.message}" + ) + } + } + internal suspend fun release() { if (isReleaseRequested.getAndSet(true)) { Logger.w(TAG, "Already released") From 5603d2047d14efc35b1642084abc58e72ec673ff Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 11 Mar 2026 19:15:40 +0100 Subject: [PATCH 54/72] feat(core): streamer: add the audio input mode as a parameter for single streamer --- .../core/pipelines/StreamerPipelineTest.kt | 4 ++-- .../core/pipelines/StreamerPipeline.kt | 14 +++++++------- .../core/streamers/dual/DualStreamerImpl.kt | 4 ++-- .../lifecycle/StreamerLifeCycleObserver.kt | 11 ++++++++++- .../streamers/single/AudioOnlySingleStreamer.kt | 9 +++++++++ .../core/streamers/single/SingleStreamer.kt | 16 ++++++++++++++++ .../core/streamers/single/SingleStreamerImpl.kt | 4 +++- 7 files changed, 49 insertions(+), 13 deletions(-) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt index d8591b2a3..230440dbe 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt @@ -57,13 +57,13 @@ class StreamerPipelineTest { context: Context, audioSource: IAudioSourceInternal.Factory?, videoSource: IVideoSourceInternal.Factory?, - audioOutputMode: StreamerPipeline.AudioOutputMode = StreamerPipeline.AudioOutputMode.PUSH + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.PUSH ): StreamerPipeline { val pipeline = StreamerPipeline( context, withAudio = audioSource != null, withVideo = videoSource != null, - audioOutputMode = audioOutputMode, + audioInputMode = audioInputMode, ) audioSource?.let { pipeline.setAudioSource(it) } videoSource?.let { pipeline.setVideoSource(it) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt index 28e53f39e..00ffea90c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt @@ -79,7 +79,7 @@ import java.util.concurrent.atomic.AtomicBoolean * @param context the application context * @param withAudio whether the streamer has audio. It will create necessary audio components. * @param withVideo whether the streamer has video. It will create necessary video components. - * @param audioOutputMode the audio output mode. It can be [AudioOutputMode.PUSH] or [AudioOutputMode.CALLBACK]. Only use [AudioOutputMode.CALLBACK] when you have a single output and its implements [IAudioCallbackPipelineOutputInternal]. By default, it is [AudioOutputMode.PUSH]. + * @param audioInputMode the audio output mode. It can be [AudioInputMode.PUSH] or [AudioInputMode.CALLBACK]. Only use [AudioInputMode.CALLBACK] when you have a single output and its implements [IAudioCallbackPipelineOutputInternal]. By default, it is [AudioInputMode.PUSH]. * @param surfaceProcessorFactory the factory to create the surface processor * @param dispatcherProvider the coroutine dispatcher */ @@ -87,7 +87,7 @@ open class StreamerPipeline( protected val context: Context, val withAudio: Boolean = true, val withVideo: Boolean = true, - private val audioOutputMode: AudioOutputMode = AudioOutputMode.PUSH, + private val audioInputMode: AudioInputMode = AudioInputMode.PUSH, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), protected val dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : IWithVideoSource, IWithVideoRotation, IWithAudioSource, IStreamer { @@ -100,14 +100,14 @@ open class StreamerPipeline( // INPUTS private val inputMutex = Mutex() private val _audioInput = if (withAudio) { - when (audioOutputMode) { - AudioOutputMode.PUSH -> AudioInput( + when (audioInputMode) { + AudioInputMode.PUSH -> AudioInput( context, AudioInput.PushConfig(::queueAudioFrame), dispatcherProvider ) - AudioOutputMode.CALLBACK -> AudioInput( + AudioInputMode.CALLBACK -> AudioInput( context, AudioInput.CallbackConfig(), dispatcherProvider @@ -486,7 +486,7 @@ open class StreamerPipeline( if (output is IAudioPipelineOutputInternal) { if (withAudio) { val audioInput = requireNotNull(_audioInput) { "Audio input is not set" } - if (audioOutputMode == AudioOutputMode.CALLBACK) { + if (audioInputMode == AudioInputMode.CALLBACK) { require(output is IAudioCallbackPipelineOutputInternal) { "Output $output must be an audio callback output" } @@ -1043,7 +1043,7 @@ open class StreamerPipeline( /** * Audio output mode. */ - enum class AudioOutputMode { + enum class AudioInputMode { /** * The audio is pushed to the output. */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt index 9455abd3f..d0c76ba13 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt @@ -31,7 +31,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.isCompati import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioOutputMode +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioInputMode import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioVideoEncodingPipelineOutput import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal import io.github.thibaultbee.streampack.core.pipelines.utils.MultiThrowable @@ -77,7 +77,7 @@ internal class DualStreamerImpl( context, withAudio, withVideo, - AudioOutputMode.PUSH, + AudioInputMode.PUSH, surfaceProcessorFactory, dispatcherProvider ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt index e07f9b149..83919f54c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt @@ -22,6 +22,7 @@ import io.github.thibaultbee.streampack.core.interfaces.ICloseableStreamer import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.interfaces.IWithAudioSource import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking +import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -48,7 +49,11 @@ open class StreamerLifeCycleObserver( owner.lifecycleScope.launch { withContext(NonCancellable) { if (streamer is IWithAudioSource) { - streamer.audioInput.startCapture() + try { + streamer.audioInput.startCapture() + } catch (t: Throwable) { + Logger.e(TAG, "Error while starting audio capture: $t") + } } } } @@ -74,4 +79,8 @@ open class StreamerLifeCycleObserver( streamer.releaseBlocking() } } + + companion object { + private const val TAG = "StreamerLifeCycleObserver" + } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt index 2408dda37..e10b6d8f0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt @@ -27,6 +27,9 @@ import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSourceFactory import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioInputMode +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioInputMode.CALLBACK import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo @@ -35,17 +38,20 @@ import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo * * @param context the application context * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If parameter is null, no audio source are set. It can be set later with [AudioOnlySingleStreamer.setAudioSource]. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun AudioOnlySingleStreamer( context: Context, audioSourceFactory: IAudioSourceInternal.Factory = MicrophoneSourceFactory(), + audioInputMode: AudioInputMode = CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): AudioOnlySingleStreamer { val streamer = AudioOnlySingleStreamer( context = context, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, dispatcherProvider = dispatcherProvider ) @@ -57,11 +63,13 @@ suspend fun AudioOnlySingleStreamer( * A [ISingleStreamer] implementation for audio only (without video). * * @param context the application context + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class AudioOnlySingleStreamer( context: Context, + audioInputMode: AudioInputMode = CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : ISingleStreamer, IAudioSingleStreamer { @@ -69,6 +77,7 @@ class AudioOnlySingleStreamer( context = context, withAudio = true, withVideo = false, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, dispatcherProvider = dispatcherProvider ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt index 6b3caa9eb..cfd1b71f1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt @@ -41,6 +41,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRo import io.github.thibaultbee.streampack.core.interfaces.setCameraId import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController @@ -53,6 +54,7 @@ import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo * @param context the application context * @param cameraId the camera id to use. By default, it is the default camera. * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If set to null, you will have to set it later explicitly. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. @@ -63,6 +65,7 @@ suspend fun cameraSingleStreamer( context: Context, cameraId: String = context.defaultCameraId, audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), @@ -70,6 +73,7 @@ suspend fun cameraSingleStreamer( ): SingleStreamer { val streamer = SingleStreamer( context = context, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, defaultRotation = defaultRotation, surfaceProcessorFactory = surfaceProcessorFactory, @@ -87,6 +91,7 @@ suspend fun cameraSingleStreamer( * * @param context the application context * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. @@ -96,6 +101,7 @@ suspend fun cameraSingleStreamer( suspend fun audioVideoMediaProjectionSingleStreamer( context: Context, mediaProjection: MediaProjection, + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), @@ -103,6 +109,7 @@ suspend fun audioVideoMediaProjectionSingleStreamer( ): SingleStreamer { val streamer = SingleStreamer( context = context, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, defaultRotation = defaultRotation, surfaceProcessorFactory = surfaceProcessorFactory, @@ -120,6 +127,7 @@ suspend fun audioVideoMediaProjectionSingleStreamer( * @param context the application context * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If set to null, you will have to set it later explicitly. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. @@ -129,6 +137,7 @@ suspend fun videoMediaProjectionSingleStreamer( context: Context, mediaProjection: MediaProjection, audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), @@ -136,6 +145,7 @@ suspend fun videoMediaProjectionSingleStreamer( ): SingleStreamer { val streamer = SingleStreamer( context = context, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, defaultRotation = defaultRotation, surfaceProcessorFactory = surfaceProcessorFactory, @@ -155,6 +165,7 @@ suspend fun videoMediaProjectionSingleStreamer( * @param context the application context * @param audioSourceFactory the audio source factory. * @param videoSourceFactory the video source factory. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. @@ -164,6 +175,7 @@ suspend fun SingleStreamer( context: Context, audioSourceFactory: IAudioSourceInternal.Factory, videoSourceFactory: IVideoSourceInternal.Factory, + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), @@ -171,6 +183,7 @@ suspend fun SingleStreamer( ): SingleStreamer { val streamer = SingleStreamer( context = context, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, defaultRotation = defaultRotation, surfaceProcessorFactory = surfaceProcessorFactory, @@ -185,6 +198,7 @@ suspend fun SingleStreamer( * A [ISingleStreamer] implementation for both audio and video. * * @param context the application context + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. @@ -192,6 +206,7 @@ suspend fun SingleStreamer( */ class SingleStreamer( context: Context, + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), @@ -201,6 +216,7 @@ class SingleStreamer( context = context, withAudio = true, withVideo = true, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, defaultRotation = defaultRotation, surfaceProcessorFactory = surfaceProcessorFactory, diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt index e334ca746..62af496b8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt @@ -55,6 +55,7 @@ import kotlinx.coroutines.runBlocking * @param context the application context * @param withAudio `true` to capture audio. It can't be changed after instantiation. * @param withVideo `true` to capture video. It can't be changed after instantiation. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. Use [StreamerPipeline.AudioInputMode.PUSH] only to get processor (incl. vumeter) running outside a stream * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. @@ -63,6 +64,7 @@ internal class SingleStreamerImpl( private val context: Context, withAudio: Boolean, withVideo: Boolean, + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), @@ -74,7 +76,7 @@ internal class SingleStreamerImpl( context, withAudio, withVideo, - audioOutputMode = StreamerPipeline.AudioOutputMode.CALLBACK, + audioInputMode = audioInputMode, surfaceProcessorFactory, dispatcherProvider ) From 3c2cab7926a3253307e968c422dfa3d5a006b95c Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Fri, 13 Mar 2026 15:09:12 +0100 Subject: [PATCH 55/72] fix(core): output: avoid a deadlock when stopping the stream and then queueAudioFrame is called --- .../encoding/EncodingPipelineOutput.kt | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt index d3b8a2a05..77bd17e46 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt @@ -397,16 +397,28 @@ internal class EncodingPipelineOutput( } } - override suspend fun queueAudioFrame(frame: RawFrame) = mutex.withLock { - val input = try { - requireNotNull(audioInput) { "Audio input is null" } - } catch (t: Throwable) { + override suspend fun queueAudioFrame(frame: RawFrame) { + /** + * Avoid deadlock when trying to start/stop the stream. + */ + if (!mutex.tryLock()) { + Logger.e(TAG, "queueAudioFrame: not locked") frame.close() - throw t + return + } + try { + val input = try { + requireNotNull(audioInput) { "Audio input is null" } + } catch (t: Throwable) { + frame.close() + throw t + } + input.queueInputFrame( + frame + ) + } finally { + mutex.unlock() } - input.queueInputFrame( - frame - ) } From aa878e72749ecbb9d4c8bf76efc58a6d84ab290f Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 18 Mar 2026 15:28:52 +0100 Subject: [PATCH 56/72] refactor(core): use AutoCloseable for Frame and RawFrame as it is non-IO resources. --- .../streampack/core/elements/data/Frame.kt | 5 ++--- .../elements/encoders/mediacodec/MediaCodecEncoder.kt | 3 +-- .../streampack/app/ui/main/PreviewViewModel.kt | 11 ++++++++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 9c408c632..da62dceec 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -20,13 +20,12 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.deepCopy import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool -import java.io.Closeable import java.nio.ByteBuffer /** * Encoded frame representation */ -interface RawFrame : Closeable { +interface RawFrame : AutoCloseable { /** * Contains an audio or video frame data. */ @@ -78,7 +77,7 @@ fun RawFrame.copy( /** * Encoded frame representation */ -interface Frame : Closeable { +interface Frame : AutoCloseable { /** * Contains an audio or video frame data. */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index 9ebbde6a6..56de657c1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -43,7 +43,6 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.io.Closeable import java.nio.ByteBuffer import kotlin.math.min @@ -605,7 +604,7 @@ internal constructor( private val outputFormat: MediaFormat, private val extra: Extra?, private val extraBuffers: List? - ) : Closeable { + ) : AutoCloseable { private var previousPresentationTimeUs = 0L private val pool = FramePool() diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index 13c97025b..792c09072 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -53,9 +53,11 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.Camer import io.github.thibaultbee.streampack.core.elements.sources.video.camera.ICameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.cameraManager import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId +import io.github.thibaultbee.streampack.core.interfaces.IWithAudioSource import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking import io.github.thibaultbee.streampack.core.interfaces.startStream +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline import io.github.thibaultbee.streampack.core.streamers.single.IAudioSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.IVideoSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer @@ -67,6 +69,7 @@ import io.github.thibaultbee.streampack.ext.srt.regulator.controllers.DefaultSrt import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.drop @@ -198,6 +201,12 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod Log.i(TAG, "Streamer is opened: $it") } } + viewModelScope.launch { + val audioInput = (streamer as IWithAudioSource).audioInput + audioInput.sourceFlow.filterNotNull().first() + delay(3000L) + audioInput.startCapture() + } viewModelScope.launch { streamer.isStreamingFlow .collect { isStreaming -> @@ -257,7 +266,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod private fun buildFirstStreamer(isAudioEnable: Boolean): IVideoSingleStreamer { return if (isAudioEnable) { - SingleStreamer(application) + SingleStreamer(application, audioInputMode = StreamerPipeline.AudioInputMode.PUSH) } else { VideoOnlySingleStreamer(application) } From b0148f96d17d56cca4883f7f29d426e99a828b38 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 30 Mar 2026 13:02:29 +0200 Subject: [PATCH 57/72] fix(ui): move onZoomRatioOnPinchChanged to the main thread --- .../thibaultbee/streampack/ui/views/PreviewView.kt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt index 40bd147a9..19e06c5d8 100644 --- a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt @@ -196,10 +196,18 @@ class PreviewView @JvmOverloads constructor( fun setZoomListener(listener: CameraSettings.Zoom.OnZoomChangedListener?) { if (listener == null) { unregisterZoomListener() + zoomListener = null } else { - registerZoomListener(listener) + val delegatedListener = object : CameraSettings.Zoom.OnZoomChangedListener { + override fun onZoomChanged(zoomRatio: Float) { + post { + listener.onZoomChanged(zoomRatio) + } + } + } + registerZoomListener(delegatedListener) + zoomListener = delegatedListener } - zoomListener = listener } private fun registerZoomListener(listener: CameraSettings.Zoom.OnZoomChangedListener) { From ecbbfd0af1a522a706d33e159ffd59917523d18d Mon Sep 17 00:00:00 2001 From: Thibault Beyou <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 31 Mar 2026 11:28:36 +0200 Subject: [PATCH 58/72] docs(Streamers.md): fix doc title --- docs/Streamers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Streamers.md b/docs/Streamers.md index 6371c1aad..2963ab523 100644 --- a/docs/Streamers.md +++ b/docs/Streamers.md @@ -1,4 +1,4 @@ -# Streamer elements +# Streamers ## Introduction From 75dedd9190e27c19ba41a0269c731a445ce09b35 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 7 Apr 2026 11:16:42 +0200 Subject: [PATCH 59/72] fix(core): try/catch stopStream and stopCapture when audio/video input is released --- .../streampack/core/pipelines/inputs/AudioInput.kt | 12 ++++++++++-- .../streampack/core/pipelines/inputs/VideoInput.kt | 6 +++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 4b1fab597..91ba88073 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -225,8 +225,16 @@ internal class AudioInput( newAudioSource.isStreamingFlow.collect { isStreaming -> if ((!isStreaming) && (isStreamingFlow.value || isCapturingFlow.value)) { Logger.i(TAG, "Audio source has been stopped.") - stopStream() - stopCapture() + try { + stopStream() + } catch (_: Throwable) { + // Nothing to do, input might be already released + } + try { + stopCapture() + } catch (_: Throwable) { + // Nothing to do, input might be already released + } } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt index ff85574f4..59bd28f9b 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt @@ -238,7 +238,11 @@ internal class VideoInput( newVideoSource.isStreamingFlow.collect { isStreaming -> if ((!isStreaming) && isStreamingFlow.value) { Logger.i(TAG, "Video source has been stopped.") - stopStream() + try { + stopStream() + } catch (_: Throwable) { + // Nothing to do, input might be already released + } } } } From 42e6573405c1f5e95d416570e23afcd8546bc0a6 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 7 Apr 2026 14:31:10 +0200 Subject: [PATCH 60/72] chore(core): rename StreamerLifeCycleObserver `autotartAudioCapture` to `startAudioCaptureOnResume` --- .../core/streamers/lifecycle/StreamerLifeCycleObserver.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt index 83919f54c..8382c984b 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt @@ -37,15 +37,15 @@ import kotlinx.coroutines.withContext * * @param streamer The streamer to control * @param releaseOnDestroy Whether to release the streamer when application is destroyed. If a view model is used, this should be set to false. - * @param autostartAudioCapture Whether to start audio capture when application is resumed. + * @param startAudioCaptureOnResume Whether to start audio capture when application is resumed. */ open class StreamerLifeCycleObserver( private val streamer: IStreamer, private val releaseOnDestroy: Boolean = false, - private val autostartAudioCapture: Boolean = false + private val startAudioCaptureOnResume: Boolean = false ) : DefaultLifecycleObserver { override fun onResume(owner: LifecycleOwner) { - if (autostartAudioCapture) { + if (startAudioCaptureOnResume) { owner.lifecycleScope.launch { withContext(NonCancellable) { if (streamer is IWithAudioSource) { From a4490bd7c478433bacdd6dc326b78e05b0551919 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 7 Apr 2026 14:33:50 +0200 Subject: [PATCH 61/72] fix(core): in StreamerLifeCycleObserver try/catch all calls to streamer methods --- .../lifecycle/StreamerLifeCycleObserver.kt | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt index 8382c984b..e95cf501e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt @@ -63,12 +63,24 @@ open class StreamerLifeCycleObserver( override fun onPause(owner: LifecycleOwner) { owner.lifecycleScope.launch { withContext(NonCancellable) { - streamer.stopStream() + try { + streamer.stopStream() + } catch (t: Throwable) { + Logger.e(TAG, "Error while stopping streamer: $t") + } if (streamer is ICloseableStreamer) { - streamer.close() + try { + streamer.close() + } catch (t: Throwable) { + Logger.e(TAG, "Error while closing streamer: $t") + } } if (streamer is IWithAudioSource) { - streamer.audioInput.stopCapture() + try { + streamer.audioInput.stopCapture() + } catch (t: Throwable) { + Logger.e(TAG, "Error while stopping audio capture: $t") + } } } } @@ -76,7 +88,11 @@ open class StreamerLifeCycleObserver( override fun onDestroy(owner: LifecycleOwner) { if (releaseOnDestroy) { - streamer.releaseBlocking() + try { + streamer.releaseBlocking() + } catch (t: Throwable) { + Logger.e(TAG, "Error while releasing streamer: $t") + } } } From a37349ab40223c969879f8f9e442b13964a235bc Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 7 Apr 2026 15:02:58 +0200 Subject: [PATCH 62/72] fix(core): in EncodingPipelineOutput try/catch onStopStream to avoid crash during release --- .../pipelines/outputs/encoding/EncodingPipelineOutput.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt index 77bd17e46..b2276177d 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt @@ -689,7 +689,11 @@ internal class EncodingPipelineOutput( return } - streamEventListener?.onStopStream() + try { + streamEventListener?.onStopStream() + } catch (t: Throwable) { + Logger.w(TAG, "Can't stop stream: ${t.message}") + } stopStreamElements() From 928cf145fc1cb89a5754a482e46a90701c9ed52f Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 7 Apr 2026 15:50:46 +0200 Subject: [PATCH 63/72] fix(core): in StreamerLifeCycleObserver, fix the startAudioCapture when source is emitted --- .../streampack/core/interfaces/IStreamer.kt | 10 +++++++--- .../lifecycle/StreamerLifeCycleObserver.kt | 15 +++++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/IStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/IStreamer.kt index a296a6c3d..b875931ac 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/IStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/IStreamer.kt @@ -19,6 +19,8 @@ import android.net.Uri import androidx.core.net.toUri import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.runBlocking @@ -59,11 +61,13 @@ interface IStreamer { /** * Clean and reset the streamer synchronously. * + * @param dispatcher The coroutine dispatcher to use * @see [IStreamer.release] */ -fun IStreamer.releaseBlocking() = runBlocking { - release() -} +fun IStreamer.releaseBlocking(dispatcher: CoroutineDispatcher = Dispatchers.Default) = + runBlocking(dispatcher) { + release() + } /** * An interface for a component that can be closed. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt index e95cf501e..4d5f08b98 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt @@ -23,7 +23,11 @@ import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.interfaces.IWithAudioSource import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking import io.github.thibaultbee.streampack.core.logger.Logger +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -42,13 +46,16 @@ import kotlinx.coroutines.withContext open class StreamerLifeCycleObserver( private val streamer: IStreamer, private val releaseOnDestroy: Boolean = false, - private val startAudioCaptureOnResume: Boolean = false + private val startAudioCaptureOnResume: Boolean = false, + private val coroutineDispatcher: CoroutineDispatcher = Dispatchers.Default ) : DefaultLifecycleObserver { override fun onResume(owner: LifecycleOwner) { if (startAudioCaptureOnResume) { - owner.lifecycleScope.launch { + owner.lifecycleScope.launch(coroutineDispatcher) { withContext(NonCancellable) { if (streamer is IWithAudioSource) { + // Wait for audio input to be ready + streamer.audioInput.sourceFlow.filterNotNull().first() try { streamer.audioInput.startCapture() } catch (t: Throwable) { @@ -61,7 +68,7 @@ open class StreamerLifeCycleObserver( } override fun onPause(owner: LifecycleOwner) { - owner.lifecycleScope.launch { + owner.lifecycleScope.launch(coroutineDispatcher) { withContext(NonCancellable) { try { streamer.stopStream() @@ -89,7 +96,7 @@ open class StreamerLifeCycleObserver( override fun onDestroy(owner: LifecycleOwner) { if (releaseOnDestroy) { try { - streamer.releaseBlocking() + streamer.releaseBlocking(coroutineDispatcher) } catch (t: Throwable) { Logger.e(TAG, "Error while releasing streamer: $t") } From 9792f25d2ef1646b812f07930880346e044ad67f Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 7 Apr 2026 16:28:50 +0200 Subject: [PATCH 64/72] fix(core): in audio and video inputs, verify isReleased state in before and in the mutex --- .../core/pipelines/inputs/AudioInput.kt | 36 +++++++++++++++---- .../core/pipelines/inputs/VideoInput.kt | 34 +++++++++++------- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 91ba88073..0ec5ade18 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -205,6 +205,10 @@ internal class AudioInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } + val previousAudioSource = sourceInternalFlow.value val isStreaming = previousAudioSource?.isStreamingFlow?.value ?: false @@ -258,11 +262,15 @@ internal class AudioInput( internal suspend fun setSourceConfig(newAudioSourceConfig: AudioSourceConfig) { if (isReleaseRequested.get()) { - throw IllegalStateException("Pipeline is released") + throw IllegalStateException("Input is released") } withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released in lock") + } + if (sourceConfig == newAudioSourceConfig) { Logger.i( TAG, @@ -305,6 +313,9 @@ internal class AudioInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released in lock") + } if (isStreamingFlow.value) { Logger.w(TAG, "Stream is already running") return@withContext @@ -324,6 +335,9 @@ internal class AudioInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released in lock") + } if (isCapturingFlow.value) { Logger.w(TAG, "Capture is already running") return@withContext @@ -356,11 +370,16 @@ internal class AudioInput( internal suspend fun stopStream() { if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") + Logger.w(TAG, "Input is released") + return } withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + Logger.w(TAG, "Input is released in lock") + return@withContext + } if (!isStreamingFlow.value) { Logger.w(TAG, "Stream is already stopped") return@withContext @@ -370,18 +389,23 @@ internal class AudioInput( Logger.w(TAG, "Stopping capture before stopping stream") return@withContext } - stop("stopStream") + stopUnsafe("stopStream") } } } override suspend fun stopCapture() { if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") + Logger.w(TAG, "Input is released") + return } withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + Logger.w(TAG, "Input is released in lock") + return@withContext + } if (!isCapturingFlow.value) { Logger.w(TAG, "Capture is already stopped") return@withContext @@ -391,12 +415,12 @@ internal class AudioInput( Logger.w(TAG, "Stopping stream before stopping capture") return@withContext } - stop("stopCapture") + stopUnsafe("stopCapture") } } } - private suspend fun stop(caller: String) { + private suspend fun stopUnsafe(caller: String) { try { port.stopStream() } catch (t: Throwable) { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt index 59bd28f9b..d157f6897 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt @@ -151,6 +151,9 @@ internal class VideoInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } val previousVideoSource = sourceInternalFlow.value val isStreaming = previousVideoSource?.isStreamingFlow?.value ?: false @@ -293,6 +296,9 @@ internal class VideoInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } if (sourceConfig == newVideoSourceConfig) { Logger.i( TAG, @@ -396,6 +402,9 @@ internal class VideoInput( } return withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } val processor = processor as? ISnapshotable? ?: throw IllegalStateException("Processor is not a snapshotable") @@ -447,6 +456,9 @@ internal class VideoInput( } sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } val sourceConfig = sourceConfig val infoProvider = source?.infoProviderFlow?.value if ((sourceConfig == null) || (infoProvider == null)) { @@ -469,10 +481,6 @@ internal class VideoInput( } internal fun addOutputSurfaceUnsafe(output: ISurfaceOutput) { - if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") - } - processor.addOutputSurface(output) } @@ -480,10 +488,6 @@ internal class VideoInput( infoProvider: ISourceInfoProvider? = source?.infoProviderFlow?.value, videoSourceConfig: VideoSourceConfig? = sourceConfig ) { - if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") - } - if ((videoSourceConfig == null) || (infoProvider == null)) { Logger.w( TAG, @@ -510,10 +514,6 @@ internal class VideoInput( infoProvider: ISourceInfoProvider? = source?.infoProviderFlow?.value, videoSourceConfig: VideoSourceConfig? = sourceConfig ) { - if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") - } - processor.removeAllOutputSurfaces() addOutputSurfacesUnsafe(infoProvider, videoSourceConfig) } @@ -527,6 +527,9 @@ internal class VideoInput( } sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } updateOutputSurfacesUnsafe(infoProvider, videoSourceConfig) } } @@ -603,10 +606,15 @@ internal class VideoInput( suspend fun stopStream() { if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") + Logger.w(TAG, "Input is released") + return } withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + Logger.w(TAG, "Input is released") + return@withContext + } _isStreamingFlow.emit(false) stopStreamUnsafe() } From 97ea532c35fdb7ed118697051fed540d89195cf1 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 8 Apr 2026 14:54:49 +0200 Subject: [PATCH 65/72] fix(core): output: the coroutine scope should run on the supervisor scope to avoid cancelling all children jobs when an exception happens --- .../core/pipelines/outputs/encoding/EncodingPipelineOutput.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt index b2276177d..dcd06bc12 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt @@ -50,6 +50,7 @@ import io.github.thibaultbee.streampack.core.pipelines.outputs.SurfaceDescriptor import io.github.thibaultbee.streampack.core.pipelines.outputs.isStreaming import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.consumeEach @@ -89,7 +90,7 @@ internal class EncodingPipelineOutput( ) : IConfigurableAudioVideoEncodingPipelineOutput, IEncodingPipelineOutputInternal, IVideoSurfacePipelineOutputInternal, IAudioSyncPipelineOutputInternal, IAudioCallbackPipelineOutputInternal { - private val coroutineScope = CoroutineScope(dispatcherProvider.default) + private val coroutineScope = CoroutineScope(dispatcherProvider.default + SupervisorJob()) private val coroutineDispatcher = dispatcherProvider.default /** From ae0ee19dfb4a6cc7c12800b2f851467c3abc0cfb Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 8 Apr 2026 14:56:59 +0200 Subject: [PATCH 66/72] test(core): moving test on VP9 to 640x360 --- .../streamer/dual/file/CameraDualStreamerFileTest.kt | 4 ++-- .../single/file/CameraSingleStreamerFileTest.kt | 4 ++-- .../single/utils/SingleStreamerConfigUtils.kt | 11 ++++++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt index a148b8c03..49505c663 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt @@ -133,8 +133,8 @@ class CameraDualStreamerFileTest( private const val STREAM_DURATION_MS = 30_000L private const val STREAM_POLLING_MS = 1_000L - private const val VIDEO_WIDTH = 1280 - private const val VIDEO_HEIGHT = 720 + private const val VIDEO_WIDTH = 640 + private const val VIDEO_HEIGHT = 360 @JvmStatic @Parameterized.Parameters( diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt index 607d4849e..8d9a15ac7 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt @@ -109,8 +109,8 @@ class CameraSingleStreamerFileTest( private const val STREAM_DURATION_MS = 30_000L private const val STREAM_POLLING_MS = 1_000L - private const val VIDEO_WIDTH = 1280 - private const val VIDEO_HEIGHT = 720 + private const val VIDEO_WIDTH = 640 + private const val VIDEO_HEIGHT = 360 @JvmStatic @Parameterized.Parameters( diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/utils/SingleStreamerConfigUtils.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/utils/SingleStreamerConfigUtils.kt index 72cfbd776..72fae4c40 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/utils/SingleStreamerConfigUtils.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/utils/SingleStreamerConfigUtils.kt @@ -50,18 +50,23 @@ object SingleStreamerConfigUtils { * * @return a [VideoConfig] for test */ - fun defaultVideoConfig(resolution: Size = Size(640, 360)) = VideoConfig( + fun defaultVideoConfig(resolution: Size = defaultSize) = VideoConfig( resolution = resolution ) /** * Creates an video configuration from a [MediaDescriptor] for test */ - fun videoConfig(descriptor: MediaDescriptor, resolution: Size = Size(640, 360)): VideoConfig { + fun videoConfig(descriptor: MediaDescriptor, resolution: Size = defaultSize): VideoConfig { return if (descriptor.type.containerType == MediaContainerType.WEBM) { - VideoCodecConfig(mimeType = MediaFormat.MIMETYPE_VIDEO_VP9, resolution = resolution) + VideoCodecConfig( + mimeType = MediaFormat.MIMETYPE_VIDEO_VP9, + resolution = resolution + ) } else { defaultVideoConfig() } } + + private val defaultSize = Size(640, 360) } \ No newline at end of file From 319c8eb9cd00bad226cbbecbe3e0b4b9f9c1ff4a Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 8 Apr 2026 15:32:31 +0200 Subject: [PATCH 67/72] test(core): workaround: disable test with webM because the VP9 encoder fails to start in the CI emulator --- .../streamer/dual/file/CameraDualStreamerFileTest.kt | 10 +++++++--- .../single/file/CameraSingleStreamerFileTest.kt | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt index 49505c663..4013ea95e 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt @@ -73,8 +73,7 @@ class CameraDualStreamerFileTest( private val videoConfig by lazy { DualStreamerConfigUtils.videoConfig( firstDescriptor, secondDescriptor, Size( - VIDEO_WIDTH, - VIDEO_HEIGHT + VIDEO_WIDTH, VIDEO_HEIGHT ) ) } @@ -158,6 +157,10 @@ class CameraDualStreamerFileTest( false, null ), + /* + * Current test on webM failed because of the VP9 encoder. Disable for now. + */ + /* arrayOf( UriMediaDescriptor(FileUtils.createCacheFile("video.webm").toUri()), true, @@ -165,7 +168,8 @@ class CameraDualStreamerFileTest( UriMediaDescriptor(FileUtils.createCacheFile("video.mp4").toUri()), true, null - ), + ) + */ ) } } diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt index 8d9a15ac7..cda7a9718 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt @@ -133,11 +133,16 @@ class CameraSingleStreamerFileTest( false, null ), + /* + * Current test on webM failed because of the VP9 encoder. Disable for now. + */ + /* arrayOf( UriMediaDescriptor(FileUtils.createCacheFile("video.webm").toUri()), true, null ) + */ ) } } From 5b0d0869726eea3a7eabcf6904bd6fb49db7508d Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 8 Apr 2026 18:03:10 +0200 Subject: [PATCH 68/72] chore(ci): upgrade actions --- .github/workflows/build.yml | 16 ++++++++-------- .github/workflows/codeql.yml | 8 ++++---- .github/workflows/docs.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- .github/workflows/snapshot.yml | 4 ++-- .github/workflows/test.yml | 6 +++--- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 90c3ad221..02fe74e7d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,13 +8,13 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' - name: Cache Gradle packages - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} @@ -24,7 +24,7 @@ jobs: - name: Assemble run: ./gradlew assemble - name: Upload APKs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: apks path: | @@ -34,13 +34,13 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' - name: Cache Gradle packages - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} @@ -52,7 +52,7 @@ jobs: working-directory: . run: ./gradlew test - name: Upload test reports - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: test-reports path: '**/build/reports/tests' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index bd12a2d5d..727f70eb7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,16 +56,16 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v5 with: distribution: 'zulu' java-version: '21' # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -87,6 +87,6 @@ jobs: run: ./gradlew --scan --full-stacktrace -Dorg.gradle.dependency.verification=off compileDebugAndroidTestSources - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c0cd69b11..8cf963265 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -11,8 +11,8 @@ jobs: update-doc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d80b958e0..848b7e9db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,9 +11,9 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index 9d5dc2a5d..c3a63f37e 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -9,9 +9,9 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38f99355a..2b1c47f32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,8 +19,8 @@ jobs: sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm ls /dev/kvm - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' @@ -43,7 +43,7 @@ jobs: env: INTEGRATION_TESTS_API_KEY: ${{ secrets.INTEGRATION_TESTS_API_KEY }} - name: Upload test reports - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: test-reports-${{ matrix.api-level }} path: | From 711b6c8b6cb09e0174647ea0487b1f8834f1af4c Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 8 Apr 2026 19:56:17 +0200 Subject: [PATCH 69/72] fix(core): processor: verify isReleased state in the mutex --- .../video/DefaultSurfaceProcessor.kt | 28 ++++++++++++++++++- .../core/pipelines/inputs/VideoInput.kt | 6 +++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt index 392b3b764..70c736443 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt @@ -85,6 +85,10 @@ private class DefaultSurfaceProcessor( } val future = submitSafely { + if (isReleaseRequested.get()) { + throw IllegalStateException("SurfaceProcessor is released") + } + val surfaceTexture = SurfaceTexture(renderer.textureName) surfaceTexture.setDefaultBufferSize(surfaceSize.width, surfaceSize.height) surfaceTexture.setOnFrameAvailableListener(this, glHandler) @@ -105,6 +109,10 @@ private class DefaultSurfaceProcessor( } override fun removeInputSurface(surface: Surface) { + if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") + return + } executeSafely { val surfaceInput = surfaceInputs.find { it.surface == surface } if (surfaceInput != null) { @@ -140,10 +148,13 @@ private class DefaultSurfaceProcessor( override fun addOutputSurface(surfaceOutput: ISurfaceOutput) { if (isReleaseRequested.get()) { - return + throw IllegalStateException("SurfaceProcessor is released") } executeSafely { + if (isReleaseRequested.get()) { + throw IllegalStateException("SurfaceProcessor is released") + } if (!surfaceOutputs.map { it.targetSurface }.contains(surfaceOutput.targetSurface)) { renderer.registerOutputSurface(surfaceOutput.targetSurface) surfaceOutputs.add(surfaceOutput) @@ -164,20 +175,30 @@ private class DefaultSurfaceProcessor( override fun removeOutputSurface(surfaceOutput: ISurfaceOutput) { if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") return } executeSafely { + if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") + return@executeSafely + } removeOutputSurfaceInternal(surfaceOutput) } } override fun removeOutputSurface(surface: Surface) { if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") return } executeSafely { + if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") + return@executeSafely + } val surfaceOutput = surfaceOutputs.firstOrNull { it.targetSurface == surface } if (surfaceOutput != null) { @@ -197,10 +218,15 @@ private class DefaultSurfaceProcessor( override fun removeAllOutputSurfaces() { if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") return } executeSafely { + if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") + return@executeSafely + } removeAllOutputSurfacesInternal() } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt index d157f6897..091bf0167 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt @@ -481,7 +481,11 @@ internal class VideoInput( } internal fun addOutputSurfaceUnsafe(output: ISurfaceOutput) { - processor.addOutputSurface(output) + try { + processor.addOutputSurface(output) + } catch (t: Throwable) { + Logger.w(TAG, "addOutputSurface: Can't add output surface: ${t.message}") + } } private suspend fun addOutputSurfacesUnsafe( From 945c58179aa3b0b3fb388e4017a621bbff268228 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Fri, 10 Apr 2026 15:22:01 +0200 Subject: [PATCH 70/72] feat(*): add a UriMediaDescriptor for the content resolver --- .../mediadescriptor/UriMediaDescriptor.kt | 47 ++++++++++++++++++- .../app/data/storage/DataStoreRepository.kt | 7 +-- .../streampack/app/utils/Extensions.kt | 43 ++--------------- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/configuration/mediadescriptor/UriMediaDescriptor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/configuration/mediadescriptor/UriMediaDescriptor.kt index 353da3ac0..f7ccfdfd1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/configuration/mediadescriptor/UriMediaDescriptor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/configuration/mediadescriptor/UriMediaDescriptor.kt @@ -1,10 +1,55 @@ package io.github.thibaultbee.streampack.core.configuration.mediadescriptor +import android.content.ContentValues import android.content.Context import android.net.Uri +import android.os.Build +import android.provider.MediaStore +import androidx.core.net.toUri import io.github.thibaultbee.streampack.core.elements.endpoints.MediaContainerType import io.github.thibaultbee.streampack.core.elements.endpoints.MediaSinkType -import androidx.core.net.toUri + +/** + * Creates a media descriptor in the video [android.content.ContentResolver]. + * + * @param context the context to infer container type from content uri + * @param contentValues the content values to create the media descriptor from + * @param customData custom data to attach to the media descriptor + */ +fun videoUriMediaDescriptor( + context: Context, + contentValues: ContentValues, + customData: List = emptyList() +): UriMediaDescriptor { + val collection = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + MediaStore.Video.Media.getContentUri( + MediaStore.VOLUME_EXTERNAL_PRIMARY + ) + } else { + MediaStore.Video.Media.EXTERNAL_CONTENT_URI + } + return UriMediaDescriptor(context, contentValues, collection, customData) +} + +/** + * Creates a media descriptor in the collection [android.content.ContentResolver]. + * + * @param context the context to infer container type from content uri + * @param contentValues the content values to create the media descriptor from + * @param collection the collection to create the media descriptor from + * @param customData custom data to attach to the media descriptor + */ +private fun UriMediaDescriptor( + context: Context, + contentValues: ContentValues, + collection: Uri, + customData: List = emptyList() +): UriMediaDescriptor { + val contentResolver = context.contentResolver + val uri = contentResolver.insert(collection, contentValues) + ?: throw RuntimeException("Unable to create file: $contentValues in $collection") + return UriMediaDescriptor(context, uri, customData) +} /** diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/data/storage/DataStoreRepository.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/data/storage/DataStoreRepository.kt index 9feac88bd..ddab33b3d 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/data/storage/DataStoreRepository.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/data/storage/DataStoreRepository.kt @@ -135,11 +135,8 @@ class DataStoreRepository( val filename = preferences[stringPreferencesKey(context.getString(R.string.file_endpoint_key))] ?: "StreamPack" - UriMediaDescriptor( - context, - context.createVideoContentUri( - filename.appendIfNotEndsWith(FileExtension.fromEndpointType(endpointType).extension) - ) + context.createVideoContentUri( + filename.appendIfNotEndsWith(FileExtension.fromEndpointType(endpointType).extension) ) } diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt index bbe54ba7d..a712b254e 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt @@ -18,13 +18,13 @@ package io.github.thibaultbee.streampack.app.utils import android.Manifest import android.content.ContentValues import android.content.Context -import android.net.Uri -import android.os.Build import android.provider.MediaStore import android.util.Range import androidx.annotation.RequiresPermission import androidx.datastore.preferences.preferencesDataStore import io.github.thibaultbee.streampack.app.ApplicationConstants.userPrefName +import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor +import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.videoUriMediaDescriptor import io.github.thibaultbee.streampack.core.elements.sources.video.camera.ICameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.backCameras import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.cameraManager @@ -77,7 +77,7 @@ val Context.dataStore by preferencesDataStore( name = userPrefName ) -fun Context.createVideoContentUri(name: String): Uri { +fun Context.createVideoContentUri(name: String): UriMediaDescriptor { val videoDetails = ContentValues().apply { put(MediaStore.Video.Media.TITLE, name) put( @@ -85,42 +85,7 @@ fun Context.createVideoContentUri(name: String): Uri { name ) } - - val resolver = this.contentResolver - val collection = - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - MediaStore.Video.Media.getContentUri( - MediaStore.VOLUME_EXTERNAL_PRIMARY - ) - } else { - MediaStore.Video.Media.EXTERNAL_CONTENT_URI - } - - return resolver.insert(collection, videoDetails) - ?: throw Exception("Unable to create video file: $name") -} - -fun Context.createAudioContentUri(name: String): Uri { - val audioDetails = ContentValues().apply { - put(MediaStore.Audio.Media.TITLE, name) - put( - MediaStore.Audio.Media.DISPLAY_NAME, - name - ) - } - - val resolver = this.contentResolver - val collection = - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - MediaStore.Audio.Media.getContentUri( - MediaStore.VOLUME_EXTERNAL_PRIMARY - ) - } else { - MediaStore.Audio.Media.EXTERNAL_CONTENT_URI - } - - return resolver.insert(collection, audioDetails) - ?: throw Exception("Unable to create audio file: $name") + return videoUriMediaDescriptor(this, videoDetails) } fun String.appendIfNotEndsWith(suffix: String): String { From 6e0ddafb15db2feb663756546e5a570fb2b058db Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sun, 12 Apr 2026 14:03:50 +0200 Subject: [PATCH 71/72] refactor(core): replace SRT regulator controller by a direct factory --- ...kt => SimpleBitrateRegulatorController.kt} | 20 +++--- .../app/ui/main/PreviewViewModel.kt | 4 +- ...gulator.kt => DummySrtBitrateRegulator.kt} | 12 ++-- .../DefaultSrtBitrateRegulatorController.kt | 63 ++++++------------- 4 files changed, 40 insertions(+), 59 deletions(-) rename core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/{DummyBitrateRegulatorController.kt => SimpleBitrateRegulatorController.kt} (87%) rename extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/{DefaultSrtBitrateRegulator.kt => DummySrtBitrateRegulator.kt} (93%) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/DummyBitrateRegulatorController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/SimpleBitrateRegulatorController.kt similarity index 87% rename from core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/DummyBitrateRegulatorController.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/SimpleBitrateRegulatorController.kt index a05a69ad7..cb39e2d3d 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/DummyBitrateRegulatorController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/SimpleBitrateRegulatorController.kt @@ -26,23 +26,23 @@ import io.github.thibaultbee.streampack.core.regulator.IBitrateRegulator import kotlinx.coroutines.CoroutineDispatcher /** - * A [BitrateRegulatorController] implementation that triggers [IBitrateRegulator.update] every [delayTimeInMs]. + * A [BitrateRegulatorController] implementation that triggers [IBitrateRegulator.update] every [pollingTimeInMs]. * * @param audioEncoder the audio [IEncoder] * @param videoEncoder the video [IEncoder] * @param endpoint the [IEndpoint] implementation * @param bitrateRegulatorFactory the [IBitrateRegulator.Factory] implementation. Use it to make your own bitrate regulator. * @param bitrateRegulatorConfig bitrate regulator configuration - * @param delayTimeInMs delay between each call to [IBitrateRegulator.update] + * @param pollingTimeInMs delay between each call to [IBitrateRegulator.update] */ -open class DummyBitrateRegulatorController( +open class SimpleBitrateRegulatorController( audioEncoder: IEncoder?, videoEncoder: IEncoder, endpoint: IEndpoint, bitrateRegulatorFactory: IBitrateRegulator.Factory, coroutineDispatcher: CoroutineDispatcher, bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig(), - delayTimeInMs: Long = 500 + pollingTimeInMs: Long = DEFAULT_POLLING_TIME_IN_MS ) : BitrateRegulatorController( audioEncoder, videoEncoder, @@ -64,7 +64,7 @@ open class DummyBitrateRegulatorController( /** * Scheduler for bitrate regulation */ - private val scheduler = CoroutineScheduler(delayTimeInMs, coroutineDispatcher) { + private val scheduler = CoroutineScheduler(pollingTimeInMs, coroutineDispatcher) { bitrateRegulator.update( endpoint.metrics, videoEncoder.bitrate, @@ -80,10 +80,14 @@ open class DummyBitrateRegulatorController( scheduler.stop() } + companion object { + const val DEFAULT_POLLING_TIME_IN_MS = 500L + } + class Factory( private val bitrateRegulatorFactory: IBitrateRegulator.Factory, private val bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig(), - private val delayTimeInMs: Long = 500 + private val pollingTimeInMs: Long = DEFAULT_POLLING_TIME_IN_MS ) : BitrateRegulatorController.Factory() { override fun newBitrateRegulatorController( pipelineOutput: IEncodingPipelineOutput, @@ -102,14 +106,14 @@ open class DummyBitrateRegulatorController( } else { null } - return DummyBitrateRegulatorController( + return SimpleBitrateRegulatorController( audioEncoder, videoEncoder, pipelineOutput.endpoint, bitrateRegulatorFactory, coroutineDispatcher, bitrateRegulatorConfig, - delayTimeInMs + pollingTimeInMs ) } } diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index 792c09072..0b3dbd592 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -65,7 +65,7 @@ import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStr import io.github.thibaultbee.streampack.core.streamers.single.withAudio import io.github.thibaultbee.streampack.core.streamers.single.withVideo import io.github.thibaultbee.streampack.core.utils.extensions.isClosedException -import io.github.thibaultbee.streampack.ext.srt.regulator.controllers.DefaultSrtBitrateRegulatorController +import io.github.thibaultbee.streampack.ext.srt.regulator.controllers.simpleSrtBitrateRegulatorControllerFactory import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -318,7 +318,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod if (bitrateRegulatorConfig != null) { Log.i(TAG, "Add bitrate regulator controller") streamer.addBitrateRegulatorController( - DefaultSrtBitrateRegulatorController.Factory( + simpleSrtBitrateRegulatorControllerFactory( bitrateRegulatorConfig = bitrateRegulatorConfig ) ) diff --git a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DefaultSrtBitrateRegulator.kt b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DummySrtBitrateRegulator.kt similarity index 93% rename from extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DefaultSrtBitrateRegulator.kt rename to extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DummySrtBitrateRegulator.kt index ad7b288bb..65dfd6916 100644 --- a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DefaultSrtBitrateRegulator.kt +++ b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DummySrtBitrateRegulator.kt @@ -27,7 +27,7 @@ import kotlin.math.min * @param onVideoTargetBitrateChange call when you have to change video bitrate * @param onAudioTargetBitrateChange not used in this implementation. */ -class DefaultSrtBitrateRegulator( +class DummySrtBitrateRegulator( bitrateRegulatorConfig: BitrateRegulatorConfig, onVideoTargetBitrateChange: ((Int) -> Unit), onAudioTargetBitrateChange: ((Int) -> Unit) @@ -108,24 +108,24 @@ class DefaultSrtBitrateRegulator( } /** - * Factory that creates a [DefaultSrtBitrateRegulator]. + * Factory that creates a [DummySrtBitrateRegulator]. */ class Factory : SrtBitrateRegulator.Factory { /** - * Creates a [DefaultSrtBitrateRegulator] object from given parameters + * Creates a [DummySrtBitrateRegulator] object from given parameters * * @param bitrateRegulatorConfig bitrate regulation configuration * @param onVideoTargetBitrateChange call when you have to change video bitrate * @param onAudioTargetBitrateChange call when you have to change audio bitrate - * @return a [DefaultSrtBitrateRegulator] object + * @return a [DummySrtBitrateRegulator] object */ override fun newBitrateRegulator( bitrateRegulatorConfig: BitrateRegulatorConfig, onVideoTargetBitrateChange: ((Int) -> Unit), onAudioTargetBitrateChange: ((Int) -> Unit) - ): DefaultSrtBitrateRegulator { - return DefaultSrtBitrateRegulator( + ): DummySrtBitrateRegulator { + return DummySrtBitrateRegulator( bitrateRegulatorConfig, onVideoTargetBitrateChange, onAudioTargetBitrateChange diff --git a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/controllers/DefaultSrtBitrateRegulatorController.kt b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/controllers/DefaultSrtBitrateRegulatorController.kt index ae290cdf4..5d56a545a 100644 --- a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/controllers/DefaultSrtBitrateRegulatorController.kt +++ b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/controllers/DefaultSrtBitrateRegulatorController.kt @@ -16,50 +16,27 @@ package io.github.thibaultbee.streampack.ext.srt.regulator.controllers import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioEncodingPipelineOutput -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutput -import io.github.thibaultbee.streampack.core.regulator.controllers.BitrateRegulatorController -import io.github.thibaultbee.streampack.core.regulator.controllers.DummyBitrateRegulatorController -import io.github.thibaultbee.streampack.ext.srt.regulator.DefaultSrtBitrateRegulator +import io.github.thibaultbee.streampack.core.regulator.controllers.SimpleBitrateRegulatorController +import io.github.thibaultbee.streampack.core.regulator.controllers.SimpleBitrateRegulatorController.Companion.DEFAULT_POLLING_TIME_IN_MS +import io.github.thibaultbee.streampack.ext.srt.regulator.DummySrtBitrateRegulator import io.github.thibaultbee.streampack.ext.srt.regulator.SrtBitrateRegulator -import kotlinx.coroutines.CoroutineDispatcher /** - * A [DummyBitrateRegulatorController] implementation for a [SrtSink]. + * A [SimpleBitrateRegulatorController.Factory] for [SrtBitrateRegulator]. + * + * @param bitrateRegulatorFactory the [SrtBitrateRegulator.Factory] implementation. Use it to make your own bitrate regulator. + * @param bitrateRegulatorConfig bitrate regulator configuration + * @param pollingTimeInMs delay between each call to [IBitrateRegulator.update] + * + * @see SimpleBitrateRegulatorController.Factory + * @see DummySrtBitrateRegulator.Factory */ -class DefaultSrtBitrateRegulatorController { - class Factory( - private val bitrateRegulatorFactory: SrtBitrateRegulator.Factory = DefaultSrtBitrateRegulator.Factory(), - private val bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig(), - private val delayTimeInMs: Long = 500 - ) : BitrateRegulatorController.Factory() { - override fun newBitrateRegulatorController( - pipelineOutput: IEncodingPipelineOutput, - coroutineDispatcher: CoroutineDispatcher - ): DummyBitrateRegulatorController { - require(pipelineOutput is IConfigurableVideoEncodingPipelineOutput) { - "Pipeline output must be an video encoding output" - } - - val videoEncoder = requireNotNull(pipelineOutput.videoEncoder) { - "Video encoder must be set" - } - - val audioEncoder = if (pipelineOutput is IConfigurableAudioEncodingPipelineOutput) { - pipelineOutput.audioEncoder - } else { - null - } - return DummyBitrateRegulatorController( - audioEncoder, - videoEncoder, - pipelineOutput.endpoint, - bitrateRegulatorFactory, - coroutineDispatcher, - bitrateRegulatorConfig, - delayTimeInMs - ) - } - } -} +fun simpleSrtBitrateRegulatorControllerFactory( + bitrateRegulatorFactory: SrtBitrateRegulator.Factory = DummySrtBitrateRegulator.Factory(), + bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig(), + pollingTimeInMs: Long = DEFAULT_POLLING_TIME_IN_MS +) = SimpleBitrateRegulatorController.Factory( + bitrateRegulatorFactory, + bitrateRegulatorConfig, + pollingTimeInMs +) From 61eca9f65a23477f5c321344925cd0b033a340dc Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 13 Apr 2026 12:02:53 +0200 Subject: [PATCH 72/72] fix(demo): camera: do not release streamer in the view model. The system can clear the view model for performance reason and the application won't recover from this. --- .../thibaultbee/streampack/app/ui/main/PreviewFragment.kt | 2 +- .../streampack/app/ui/main/PreviewViewModel.kt | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt index 3f4296514..b658f8ae9 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt @@ -111,7 +111,7 @@ class PreviewFragment : Fragment(R.layout.main_fragment) { if (previousStreamerLifecycleObserver != null) { lifecycle.removeObserver(previousStreamerLifecycleObserver!!) } - val newObserver = StreamerLifeCycleObserver(streamer).apply { + val newObserver = StreamerLifeCycleObserver(streamer, releaseOnDestroy = true).apply { previousStreamerLifecycleObserver = this } lifecycle.addObserver(newObserver) diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index 0b3dbd592..85ddd718f 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -55,7 +55,6 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.interfaces.IWithAudioSource import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource -import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking import io.github.thibaultbee.streampack.core.interfaces.startStream import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline import io.github.thibaultbee.streampack.core.streamers.single.IAudioSingleStreamer @@ -611,11 +610,8 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod override fun onCleared() { super.onCleared() - try { - streamer.releaseBlocking() - } catch (t: Throwable) { - Log.e(TAG, "Streamer release failed", t) - } + startStreamJob?.cancel() + startStreamJob = null } companion object {