Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@

PASS takePhoto() on an 'ended' track should throw "InvalidStateError"
PASS "OperationError" should be thrown if the track ends before the 'takePhoto' promise resolves
PASS Image returned by 'takePhoto' should be at least as big as { photoSettings.imageHeight, photoSettings.imageWidth }
PASS If 'takePhoto' has to reconfigure capture track, 'mute' and 'unmute' should fire and track size should be restored
PASS applyConstraints should not run until 'takePhoto' has completed

111 changes: 111 additions & 0 deletions LayoutTests/fast/mediastream/image-capture-take-photo.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
<!DOCTYPE html>
<html>
<head>
<meta charset='utf-8'>
<title>ImageCapture takePhoto</title>
<script src='../../resources/testharness.js'></script>
<script src='../../resources/testharnessreport.js'></script>
</head>
<body>
<script>

promise_test(async (test) => {
const stream = await navigator.mediaDevices.getUserMedia({ video: { width : 640 } });
const [track] = stream.getVideoTracks();

assert_equals(track.readyState, 'live');
track.stop();
assert_equals(track.readyState, 'ended');

const imageCapture = new ImageCapture(track);
return promise_rejects_dom(test, 'InvalidStateError', imageCapture.takePhoto())

}, `takePhoto() on an 'ended' track should throw "InvalidStateError"`);

promise_test(async (test) => {
const stream = await navigator.mediaDevices.getUserMedia({ video: { width : 640 } });
const [track] = stream.getVideoTracks();

assert_equals(track.readyState, 'live');

const imageCapture = new ImageCapture(track);
const promise = imageCapture.takePhoto();

track.stop();
assert_equals(track.readyState, 'ended');

return promise_rejects_dom(test, 'OperationError', promise);

}, `"OperationError" should be thrown if the track ends before the 'takePhoto' promise resolves`);

promise_test(async (test) => {
const stream = await navigator.mediaDevices.getUserMedia({ video: { width : 640 } });
const [track] = stream.getVideoTracks();

const imageCapture = new ImageCapture(track);
let photoSettings = await imageCapture.getPhotoSettings();

const blob = await imageCapture.takePhoto();
const image = new Image();
image.src = URL.createObjectURL(blob);
await new Promise(resolve => { image.onload = resolve; });

assert_greater_than_equal(image.width, photoSettings.imageWidth, "image width is at least as big as photoSettings().imageWidth");
assert_greater_than_equal(image.height, photoSettings.imageHeight, "image height is at least as big as photoSettings().imageHeight");

}, `Image returned by 'takePhoto' should be at least as big as { photoSettings.imageHeight, photoSettings.imageWidth }`);

promise_test(async (test) => {
const stream = await navigator.mediaDevices.getUserMedia({ video: { width : 320 } });
const [track] = stream.getVideoTracks();
const { width: originalWidth, height: originalHeight } = track.getSettings();

const mutePromise = new Promise(resolve => { track.onmute = resolve; });
const unmutePromise = new Promise(resolve => { track.onunmute = resolve; });

const imageCapture = new ImageCapture(track);
const blob = await imageCapture.takePhoto({ imageWidth: 1280, imageHeight: 720});
const image = new Image();
image.src = URL.createObjectURL(blob);
await new Promise(resolve => { image.onload = resolve; });

assert_greater_than_equal(image.width, 1280, "image width is at least as big as requested width");
assert_greater_than_equal(image.height, 720, "image height is at least as big as requested height");

await Promise.all([mutePromise, unmutePromise]);

const trackSettings = track.getSettings();
assert_equals(track.readyState, 'live');
assert_equals(trackSettings.width, originalWidth, "track width restored after 'takePhoto' resolves");
assert_equals(trackSettings.height, originalHeight, "track height restored after 'takePhoto' resolves");

}, `If 'takePhoto' has to reconfigure capture track, 'mute' and 'unmute' should fire and track size should be restored`);

promise_test(async (test) => {
const stream = await navigator.mediaDevices.getUserMedia({ video: { width : 320 } });
const [track] = stream.getVideoTracks();
const imageCapture = new ImageCapture(track);

let blob;
await Promise.all([
new Promise(async (resolve) => { blob = await imageCapture.takePhoto({ imageWidth: 1280, imageHeight: 720}); resolve(); }),
track.applyConstraints({ width: 320, height: 240 }),
]);

const image = new Image();
image.src = URL.createObjectURL(blob);
await new Promise(resolve => { image.onload = resolve; });

assert_greater_than_equal(image.width, 1280, "image width is at least as big as requested width");
assert_greater_than_equal(image.height, 720, "image height is at least as big as requested height");

const trackSettings = track.getSettings();
assert_equals(track.readyState, 'live');
assert_equals(trackSettings.width, 320, "track width changed by applyConstraints");
assert_equals(trackSettings.height, 240, "track height changed by applyConstraints");

}, `applyConstraints should not run until 'takePhoto' has completed`);

</script>
</body>
</html>
14 changes: 14 additions & 0 deletions Source/WebCore/Modules/mediastream/ImageCapture.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@

#if ENABLE(MEDIA_STREAM)

#include "JSBlob.h"
#include "JSPhotoCapabilities.h"
#include "TaskSource.h"
#include <wtf/IsoMallocInlines.h>

namespace WebCore {
Expand All @@ -53,6 +55,18 @@ ImageCapture::ImageCapture(Document& document, Ref<MediaStreamTrack> track)

ImageCapture::~ImageCapture() = default;

void ImageCapture::takePhoto(PhotoSettings&& settings, DOMPromiseDeferred<IDLInterface<Blob>>&& promise)
{
m_track->takePhoto(WTFMove(settings))->whenSettled(RunLoop::main(), [protectedThis = Ref { *this }, promise = WTFMove(promise)] (auto&& result) mutable {
queueTaskKeepingObjectAlive(protectedThis.get(), TaskSource::ImageCapture, [promise = WTFMove(promise), result = WTFMove(result), protectedThis] () mutable {
if (!result)
promise.reject(WTFMove(result.error()));
else
promise.resolve(Blob::create(protectedThis->scriptExecutionContext(), WTFMove(get<0>(result.value())), WTFMove(get<1>(result.value()))));
});
});
}

void ImageCapture::getPhotoCapabilities(PhotoCapabilitiesPromise&& promise)
{
if (m_track->readyState() == MediaStreamTrack::State::Ended) {
Expand Down
3 changes: 3 additions & 0 deletions Source/WebCore/Modules/mediastream/ImageCapture.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#if ENABLE(MEDIA_STREAM)

#include "ActiveDOMObject.h"
#include "Blob.h"
#include "Document.h"
#include "JSDOMPromiseDeferred.h"
#include "MediaStreamTrack.h"
Expand All @@ -43,6 +44,8 @@ class ImageCapture : public RefCounted<ImageCapture>, public ActiveDOMObject {

~ImageCapture();

void takePhoto(PhotoSettings&&, DOMPromiseDeferred<IDLInterface<Blob>>&&);

using PhotoCapabilitiesPromise = DOMPromiseDeferred<IDLDictionary<PhotoCapabilities>>;
void getPhotoCapabilities(PhotoCapabilitiesPromise&&);

Expand Down
3 changes: 1 addition & 2 deletions Source/WebCore/Modules/mediastream/ImageCapture.idl
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,7 @@

Promise<PhotoCapabilities> getPhotoCapabilities();

// FIXME: https://bugs.webkit.org/show_bug.cgi?id=262467
// Promise<Blob> takePhoto(optional PhotoSettings photoSettings = {});
[NewObject] Promise<Blob> takePhoto(optional PhotoSettings photoSettings = {});

Promise<PhotoSettings> getPhotoSettings();

Expand Down
71 changes: 63 additions & 8 deletions Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#include "Event.h"
#include "EventNames.h"
#include "FrameLoader.h"
#include "JSBlob.h"
#include "JSDOMPromiseDeferred.h"
#include "JSMeteringMode.h"
#include "JSOverconstrainedError.h"
Expand All @@ -61,6 +62,7 @@
#include <wtf/IsoMallocInlines.h>
#include <wtf/NativePromise.h>
#include <wtf/NeverDestroyed.h>
#include <wtf/Scope.h>

namespace WebCore {

Expand Down Expand Up @@ -314,6 +316,47 @@ MediaStreamTrack::TrackCapabilities MediaStreamTrack::getCapabilities() const
return result;
}

void MediaStreamTrack::queueAndProcessSerialAction(Function<Ref<GenericPromise>()>&& action)
{
ASSERT(isMainThread());
m_pendingActions = m_pendingActions->isResolved() ? action() : m_pendingActions->whenSettled(RunLoop::main(), WTFMove(action));
}

auto MediaStreamTrack::takePhoto(PhotoSettings&& settings) -> Ref<TakePhotoPromise>
{
TakePhotoPromise::Producer producer;
Ref<TakePhotoPromise> promise = producer;

queueAndProcessSerialAction([settings = WTFMove(settings), protectedThis = Ref { *this }, producer = WTFMove(producer)]() mutable -> Ref<GenericPromise> {
// https://w3c.github.io/mediacapture-image/#dom-imagecapture-takephoto
// If the readyState of track provided in the constructor is not live, return
// a promise rejected with a new DOMException whose name is InvalidStateError,
// and abort these steps.
if (protectedThis->m_ended || protectedThis->m_readyState != State::Live) {
producer.reject(Exception { InvalidStateError, "Track has ended"_s });
return GenericPromise::createAndResolve();
}
return protectedThis->m_private->takePhoto(WTFMove(settings))->whenSettled(RunLoop::main(),
[protectedThis = WTFMove(protectedThis), producer = WTFMove(producer)] (auto&& result) mutable {

// https://w3c.github.io/mediacapture-image/#dom-imagecapture-takephoto
// If the operation cannot be completed for any reason (for example, upon
// invocation of multiple takePhoto() method calls in rapid succession),
// then reject p with a new DOMException whose name is UnknownError, and
// abort these steps.
if (!result)
producer.reject(Exception { UnknownError, WTFMove(result.error()) });
else if (RefPtr context = protectedThis->scriptExecutionContext(); !context || context->activeDOMObjectsAreStopped() || protectedThis->m_ended)
producer.reject(Exception { OperationError, "Track has ended"_s });
else
producer.resolve(WTFMove(result.value()));
return GenericPromise::createAndResolve();
});
});

return promise;
}

void MediaStreamTrack::getPhotoCapabilities(DOMPromiseDeferred<IDLDictionary<PhotoCapabilities>>&& promise) const
{
m_private->getPhotoCapabilities([protectedThis = Ref { *this }, promise = WTFMove(promise)](auto&& result) mutable {
Expand Down Expand Up @@ -367,15 +410,27 @@ static MediaConstraints createMediaConstraints(const std::optional<MediaTrackCon

void MediaStreamTrack::applyConstraints(const std::optional<MediaTrackConstraints>& constraints, DOMPromiseDeferred<void>&& promise)
{
auto completionHandler = [this, protectedThis = Ref { *this }, constraints, promise = WTFMove(promise)](auto&& error) mutable {
if (error) {
promise.rejectType<IDLInterface<OverconstrainedError>>(OverconstrainedError::create(WTFMove(error->badConstraint), WTFMove(error->message)));
return;
queueAndProcessSerialAction([protectedThis = Ref { *this }, constraints, domPromise = WTFMove(promise)]() mutable {
if (protectedThis->m_ended) {
domPromise.reject(Exception { InvalidAccessError, "Track has ended"_s });
return GenericPromise::createAndResolve();
}
promise.resolve();
m_constraints = valueOrDefault(constraints);
};
m_private->applyConstraints(createMediaConstraints(constraints), WTFMove(completionHandler));
GenericPromise::Producer producer;
Ref<GenericPromise> nativePromise = producer;

protectedThis->m_private->applyConstraints(createMediaConstraints(constraints), [protectedThis = WTFMove(protectedThis), constraints, domPromise = WTFMove(domPromise), producer = WTFMove(producer)](auto&& error) mutable {
if (error) {
domPromise.rejectType<IDLInterface<OverconstrainedError>>(OverconstrainedError::create(WTFMove(error->badConstraint), WTFMove(error->message)));
producer.resolve();
return;
}

protectedThis->m_constraints = valueOrDefault(constraints);
domPromise.resolve();
producer.resolve();
});
return nativePromise;
});
}

void MediaStreamTrack::addObserver(Observer& observer)
Expand Down
14 changes: 10 additions & 4 deletions Source/WebCore/Modules/mediastream/MediaStreamTrack.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,18 @@
#if ENABLE(MEDIA_STREAM)

#include "ActiveDOMObject.h"
#include "DoubleRange.h"
#include "Blob.h"
#include "EventTarget.h"
#include "IDLTypes.h"
#include "LongRange.h"
#include "JSDOMPromiseDeferred.h"
#include "MediaProducer.h"
#include "MediaStreamTrackPrivate.h"
#include "MediaTrackCapabilities.h"
#include "MediaTrackConstraints.h"
#include "PhotoCapabilities.h"
#include "PhotoSettings.h"
#include "PlatformMediaSession.h"
#include <wtf/Deque.h>
#include <wtf/LoggerHelper.h>

namespace WebCore {
Expand All @@ -50,8 +51,6 @@ class Document;

struct MediaTrackConstraints;

template<typename IDLType> class DOMPromiseDeferred;

class MediaStreamTrack
: public RefCounted<MediaStreamTrack>
, public ActiveDOMObject
Expand Down Expand Up @@ -130,11 +129,14 @@ class MediaStreamTrack
using TrackCapabilities = MediaTrackCapabilities;
TrackCapabilities getCapabilities() const;

using TakePhotoPromise = NativePromise<std::pair<Vector<uint8_t>, String>, Exception>;
Ref<TakePhotoPromise> takePhoto(PhotoSettings&&);
void getPhotoCapabilities(DOMPromiseDeferred<IDLDictionary<PhotoCapabilities>>&&) const;
void getPhotoSettings(DOMPromiseDeferred<IDLDictionary<PhotoSettings>>&&) const;

const MediaTrackConstraints& getConstraints() const { return m_constraints; }
void setConstraints(MediaTrackConstraints&& constraints) { m_constraints = WTFMove(constraints); }

void applyConstraints(const std::optional<MediaTrackConstraints>&, DOMPromiseDeferred<void>&&);

RealtimeMediaSource& source() const { return m_private->source(); }
Expand Down Expand Up @@ -203,6 +205,10 @@ class MediaStreamTrack
WTFLogChannel& logChannel() const final;
#endif

using SerialAction = Function<Ref<GenericPromise>()>;
void queueAndProcessSerialAction(SerialAction&&);
Ref<GenericPromise> m_pendingActions { GenericPromise::createAndResolve() };

Vector<Observer*> m_observers;

MediaTrackConstraints m_constraints;
Expand Down
6 changes: 6 additions & 0 deletions Source/WebCore/PAL/pal/cocoa/AVFoundationSoftLink.h
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,12 @@ SOFT_LINK_CLASS_FOR_HEADER(PAL, AVCaptureDevice)
SOFT_LINK_CLASS_FOR_HEADER(PAL, AVCaptureDeviceFormat)
SOFT_LINK_CLASS_FOR_HEADER(PAL, AVCaptureDeviceInput)
SOFT_LINK_CLASS_FOR_HEADER(PAL, AVCaptureOutput)
SOFT_LINK_CLASS_FOR_HEADER(PAL, AVCapturePhotoSettings)
SOFT_LINK_CLASS_FOR_HEADER(PAL, AVCaptureSession)
SOFT_LINK_CLASS_FOR_HEADER(PAL, AVCaptureVideoDataOutput)
SOFT_LINK_CLASS_FOR_HEADER(PAL, AVFrameRateRange)
SOFT_LINK_CLASS_FOR_HEADER(PAL, AVCaptureDeviceDiscoverySession)
SOFT_LINK_CLASS_FOR_HEADER(PAL, AVCapturePhotoOutput)
#endif

SOFT_LINK_CONSTANT_FOR_HEADER(PAL, AVFoundation, AVAudioTimePitchAlgorithmSpectral, NSString *)
Expand Down Expand Up @@ -204,6 +206,8 @@ SOFT_LINK_CONSTANT_FOR_HEADER(PAL, AVFoundation, AVFileTypeQuickTimeMovie, NSStr
#define AVFileTypeQuickTimeMovie PAL::get_AVFoundation_AVFileTypeQuickTimeMovie()
SOFT_LINK_CONSTANT_FOR_HEADER(PAL, AVFoundation, AVVideoCodecKey, NSString *)
#define AVVideoCodecKey PAL::get_AVFoundation_AVVideoCodecKey()
SOFT_LINK_CONSTANT_FOR_HEADER(PAL, AVFoundation, AVVideoCodecTypeJPEG, NSString *)
#define AVVideoCodecTypeJPEG PAL::get_AVFoundation_AVVideoCodecTypeJPEG()
SOFT_LINK_CONSTANT_FOR_HEADER(PAL, AVFoundation, AVVideoCodecH264, NSString *)
#define AVVideoCodecH264 PAL::get_AVFoundation_AVVideoCodecH264()
SOFT_LINK_CONSTANT_FOR_HEADER(PAL, AVFoundation, AVVideoWidthKey, NSString *)
Expand All @@ -222,6 +226,8 @@ SOFT_LINK_CONSTANT_FOR_HEADER(PAL, AVFoundation, AVVideoProfileLevelH264MainAuto
#define AVVideoProfileLevelH264MainAutoLevel PAL::get_AVFoundation_AVVideoProfileLevelH264MainAutoLevel()
SOFT_LINK_CONSTANT_FOR_HEADER(PAL, AVFoundation, AVVideoCompressionPropertiesKey, NSString *)
#define AVVideoCompressionPropertiesKey PAL::get_AVFoundation_AVVideoCompressionPropertiesKey()
SOFT_LINK_CONSTANT_FOR_HEADER(PAL, AVFoundation, AVVideoQualityKey, NSString *)
#define AVVideoQualityKey PAL::get_AVFoundation_AVVideoQualityKey()
SOFT_LINK_CONSTANT_MAY_FAIL_FOR_HEADER(PAL, AVFoundation, AVEncoderBitRateKey, NSString *)
#define AVEncoderBitRateKey PAL::get_AVFoundation_AVEncoderBitRateKey()
SOFT_LINK_CONSTANT_MAY_FAIL_FOR_HEADER(PAL, AVFoundation, AVFormatIDKey, NSString *)
Expand Down
4 changes: 4 additions & 0 deletions Source/WebCore/PAL/pal/cocoa/AVFoundationSoftLink.mm
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,12 @@ static BOOL justReturnsNO()
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVCaptureDeviceFormat, PAL_EXPORT)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVCaptureDeviceInput, PAL_EXPORT)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVCaptureOutput, PAL_EXPORT)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVCapturePhotoSettings, PAL_EXPORT)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVCaptureSession, PAL_EXPORT)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVCaptureVideoDataOutput, PAL_EXPORT)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVFrameRateRange, PAL_EXPORT)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVCaptureDeviceDiscoverySession, PAL_EXPORT)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVCapturePhotoOutput, PAL_EXPORT)
#endif

SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVAssetExportPresetHighestQuality, NSString *, PAL_EXPORT)
Expand All @@ -140,6 +142,7 @@ static BOOL justReturnsNO()
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVCaptureDeviceWasDisconnectedNotification, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVFileTypeMPEG4, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVFileTypeQuickTimeMovie, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVVideoCodecTypeJPEG, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVLayerVideoGravityResize, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVLayerVideoGravityResizeAspect, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVLayerVideoGravityResizeAspectFill, NSString *, PAL_EXPORT)
Expand Down Expand Up @@ -199,6 +202,7 @@ static BOOL justReturnsNO()
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVVideoAverageBitRateKey, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVVideoCodecH264, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVVideoCodecKey, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVVideoQualityKey, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVVideoCompressionPropertiesKey, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVVideoExpectedSourceFrameRateKey, NSString *, PAL_EXPORT)
SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, AVFoundation, AVVideoHeightKey, NSString *, PAL_EXPORT)
Expand Down
1 change: 1 addition & 0 deletions Source/WebCore/dom/TaskSource.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ enum class TaskSource : uint8_t {
Gamepad,
Geolocation,
IdleTask,
ImageCapture,
IndexedDB,
MediaElement,
Microtask,
Expand Down
Loading