/* * Copyright (C) 2021 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #import "config.h" #import "ModelElementController.h" #import #import #if ENABLE(ARKIT_INLINE_PREVIEW) #import "Logging.h" #import "WebPageProxy.h" #import #import #import #import #import #import #import #import #if ENABLE(ARKIT_INLINE_PREVIEW_IOS) #import "APIUIClient.h" #import "RemoteLayerTreeDrawingAreaProxy.h" #import "RemoteLayerTreeHost.h" #import "RemoteLayerTreeViews.h" #import "WKModelView.h" #import #endif #if ENABLE(ARKIT_INLINE_PREVIEW_MAC) #import #endif SOFT_LINK_PRIVATE_FRAMEWORK(AssetViewer); SOFT_LINK_CLASS(AssetViewer, ASVInlinePreview); namespace WebKit { #if ENABLE(ARKIT_INLINE_PREVIEW_IOS) WKModelView * ModelElementController::modelViewForModelIdentifier(ModelIdentifier modelIdentifier) { if (!m_webPageProxy.preferences().modelElementEnabled()) return nil; if (!is(m_webPageProxy.drawingArea())) return nil; auto* node = downcast(*m_webPageProxy.drawingArea()).remoteLayerTreeHost().nodeForID(modelIdentifier.layerIdentifier); if (!node) return nil; return dynamic_objc_cast(node->uiView()); } ASVInlinePreview * ModelElementController::previewForModelIdentifier(ModelIdentifier modelIdentifier) { return [modelViewForModelIdentifier(modelIdentifier) preview]; } void ModelElementController::takeModelElementFullscreen(ModelIdentifier modelIdentifier, const URL& originatingPageURL) { auto *presentingViewController = m_webPageProxy.uiClient().presentingViewController(); if (!presentingViewController) return; auto modelView = modelViewForModelIdentifier(modelIdentifier); if (!modelView) return; CGRect initialFrame = [modelView convertRect:modelView.frame toView:nil]; ASVInlinePreview *preview = [modelView preview]; [preview setCanonicalWebPageURL:originatingPageURL]; [preview setUrlFragment:originatingPageURL.fragmentIdentifier().createNSString().get()]; NSDictionary *previewOptions = @{@"WebKit": @"Model element fullscreen"}; [preview createFullscreenInstanceWithInitialFrame:initialFrame previewOptions:previewOptions completionHandler:^(UIViewController *remoteViewController, CAFenceHandle *fenceHandle, NSError *creationError) { if (creationError) { LOG(ModelElement, "Unable to create fullscreen instance: %@", [creationError localizedDescription]); [fenceHandle invalidate]; return; } dispatch_async(dispatch_get_main_queue(), ^{ remoteViewController.modalPresentationStyle = UIModalPresentationOverFullScreen; remoteViewController.view.backgroundColor = UIColor.clearColor; [presentingViewController presentViewController:remoteViewController animated:NO completion:^(void) { [CATransaction begin]; [modelView.layer.superlayer.context addFence:fenceHandle]; [CATransaction commit]; [fenceHandle invalidate]; }]; [preview observeDismissFullscreenWithCompletionHandler:^(CAFenceHandle *dismissFenceHandle, NSDictionary *payload, NSError *dismissError) { dispatch_async(dispatch_get_main_queue(), ^{ if (dismissError || !dismissFenceHandle) { LOG(ModelElement, "Unable to get fence handle when dismissing fullscreen instance: %@", [dismissError localizedDescription]); [dismissFenceHandle invalidate]; return; } [CATransaction begin]; [modelView.layer.superlayer.context addFence:dismissFenceHandle]; [CATransaction setCompletionBlock:^{ [remoteViewController dismissViewControllerAnimated:NO completion:nil]; }]; [CATransaction commit]; [dismissFenceHandle invalidate]; }); }]; }); }]; } void ModelElementController::setInteractionEnabledForModelElement(ModelIdentifier modelIdentifier, bool isInteractionEnabled) { if (auto *modelView = modelViewForModelIdentifier(modelIdentifier)) modelView.userInteractionEnabled = isInteractionEnabled; } #endif // ENABLE(ARKIT_INLINE_PREVIEW_IOS) #if ENABLE(ARKIT_INLINE_PREVIEW_MAC) ASVInlinePreview * ModelElementController::previewForModelIdentifier(ModelIdentifier modelIdentifier) { if (!m_webPageProxy.preferences().modelElementEnabled()) return nullptr; return m_inlinePreviews.get(modelIdentifier.uuid).get(); } void ModelElementController::modelElementCreateRemotePreview(String uuid, WebCore::FloatSize size, CompletionHandler, WebCore::ResourceError>)>&& completionHandler) { if (!m_webPageProxy.preferences().modelElementEnabled()) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::errorDomainWebKitInternal, 0, { }, "Model element disabled"_s })); return; } auto nsUUID = adoptNS([[NSUUID alloc] initWithUUIDString:uuid]); auto preview = adoptNS([allocASVInlinePreviewInstance() initWithFrame:CGRectMake(0, 0, size.width(), size.height()) UUID:nsUUID.get()]); LOG(ModelElement, "Created remote preview with UUID %s.", uuid.utf8().data()); auto iterator = m_inlinePreviews.find(uuid); if (iterator == m_inlinePreviews.end()) m_inlinePreviews.set(uuid, preview); else iterator->value = preview; auto handler = CompletionHandlerWithFinalizer, WebCore::ResourceError>)>(WTFMove(completionHandler), [] (Function, WebCore::ResourceError>)>& completionHandler) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::ResourceError::Type::General })); }); RELEASE_ASSERT(isMainRunLoop()); [preview setupRemoteConnectionWithCompletionHandler:makeBlockPtr([weakThis = WeakPtr { *this }, preview, uuid = WTFMove(uuid), handler = WTFMove(handler)] (NSError *contextError) mutable { if (contextError) { LOG(ModelElement, "Unable to create remote connection for uuid %s: %@.", uuid.utf8().data(), contextError.localizedDescription); callOnMainRunLoop([weakThis = WTFMove(weakThis), handler = WTFMove(handler), error = WebCore::ResourceError { contextError }] () mutable { if (!weakThis) return; handler(makeUnexpected(error)); }); return; } LOG(ModelElement, "Established remote connection with UUID %s.", uuid.utf8().data()); auto contextId = [preview contextId]; callOnMainRunLoop([weakThis = WTFMove(weakThis), uuid = WTFMove(uuid), handler = WTFMove(handler), contextId] () mutable { if (!weakThis) return; handler(std::make_pair(uuid, contextId)); }); }).get()]; } void ModelElementController::modelElementLoadRemotePreview(String uuid, URL fileURL, CompletionHandler&&)>&& completionHandler) { if (!m_webPageProxy.preferences().modelElementEnabled()) { completionHandler(WebCore::ResourceError { WebCore::errorDomainWebKitInternal, 0, { }, "Model element disabled"_s }); return; } auto preview = previewForUUID(uuid); if (!preview) completionHandler(WebCore::ResourceError { WebCore::errorDomainWebKitInternal, 0, { }, "Could not find a preview for the provided UUID"_s }); auto handler = CompletionHandlerWithFinalizer&&)>(WTFMove(completionHandler), [](Function&&)>& completionHandler) { completionHandler(WebCore::ResourceError { WebCore::ResourceError::Type::General }); }); RELEASE_ASSERT(isMainRunLoop()); [preview preparePreviewOfFileAtURL:[[NSURL alloc] initFileURLWithPath:fileURL.fileSystemPath()] completionHandler:makeBlockPtr([weakThis = WeakPtr { *this }, uuid = WTFMove(uuid), handler = WTFMove(handler)] (NSError *loadError) mutable { if (loadError) { LOG(ModelElement, "Unable to load file for uuid %s: %@.", uuid.utf8().data(), loadError.localizedDescription); callOnMainRunLoop([weakThis = WTFMove(weakThis), handler = WTFMove(handler), error = WebCore::ResourceError { loadError }] () mutable { if (!weakThis) return; handler(error); }); return; } LOG(ModelElement, "Loaded file with UUID %s.", uuid.utf8().data()); callOnMainRunLoop([weakThis = WTFMove(weakThis), handler = WTFMove(handler)] () mutable { if (!weakThis) return; handler({ }); }); }).get()]; } void ModelElementController::modelElementDestroyRemotePreview(String uuid) { m_inlinePreviews.remove(uuid); } RetainPtr ModelElementController::previewForUUID(const String& uuid) { return m_inlinePreviews.get(uuid); } void ModelElementController::handleMouseDownForModelElement(const String& uuid, const WebCore::LayoutPoint& flippedLocationInElement, MonotonicTime timestamp) { if (auto preview = previewForUUID(uuid)) [preview mouseDownAtLocation:CGPointMake(flippedLocationInElement.x().toFloat(), flippedLocationInElement.y().toFloat()) timestamp:timestamp.secondsSinceEpoch().value()]; } void ModelElementController::handleMouseMoveForModelElement(const String& uuid, const WebCore::LayoutPoint& flippedLocationInElement, MonotonicTime timestamp) { if (auto preview = previewForUUID(uuid)) [preview mouseDraggedAtLocation:CGPointMake(flippedLocationInElement.x().toFloat(), flippedLocationInElement.y().toFloat()) timestamp:timestamp.secondsSinceEpoch().value()]; } void ModelElementController::handleMouseUpForModelElement(const String& uuid, const WebCore::LayoutPoint& flippedLocationInElement, MonotonicTime timestamp) { if (auto preview = previewForUUID(uuid)) [preview mouseUpAtLocation:CGPointMake(flippedLocationInElement.x().toFloat(), flippedLocationInElement.y().toFloat()) timestamp:timestamp.secondsSinceEpoch().value()]; } void ModelElementController::modelElementSizeDidChange(const String& uuid, WebCore::FloatSize size, CompletionHandler)>&& completionHandler) { auto preview = previewForUUID(uuid); if (!preview) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::errorDomainWebKitInternal, 0, { }, "Could not find model"_s })); return; } auto handler = CompletionHandlerWithFinalizer)>(WTFMove(completionHandler), [] (Function)>& completionHandler) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::ResourceError::Type::General })); }); [preview updateFrame:CGRectMake(0, 0, size.width(), size.height()) completionHandler:makeBlockPtr([weakThis = WeakPtr { *this }, handler = WTFMove(handler), uuid] (CAFenceHandle *fenceHandle, NSError *error) mutable { if (error) { LOG(ModelElement, "Unable to update frame: %@.", error.localizedDescription); callOnMainRunLoop([weakThis = WTFMove(weakThis), handler = WTFMove(handler), error = WebCore::ResourceError { error }] () mutable { if (!weakThis) return; handler(makeUnexpected(error)); }); [fenceHandle invalidate]; return; } RetainPtr strongFenceHandle = fenceHandle; callOnMainRunLoop([weakThis = WTFMove(weakThis), handler = WTFMove(handler), uuid, strongFenceHandle = WTFMove(strongFenceHandle)] () mutable { if (!weakThis) return; auto fenceSendRight = MachSendRight::adopt([strongFenceHandle copyPort]); [strongFenceHandle invalidate]; handler(fenceSendRight); }); }).get()]; } void ModelElementController::inlinePreviewUUIDs(CompletionHandler&&)>&& completionHandler) { completionHandler(WTF::map(m_inlinePreviews, [](auto& entry) { return entry.key; })); } #endif // ENABLE(ARKIT_INLINE_PREVIEW_MAC) #if ENABLE(ARKIT_INLINE_PREVIEW) static bool previewHasCameraSupport(ASVInlinePreview *preview) { #if ENABLE(ARKIT_INLINE_PREVIEW_CAMERA_TRANSFORM) return [preview respondsToSelector:@selector(getCameraTransform:)]; #else return false; #endif } void ModelElementController::getCameraForModelElement(ModelIdentifier modelIdentifier, CompletionHandler)>&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasCameraSupport(preview)) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::ResourceError::Type::General })); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_CAMERA_TRANSFORM) [preview getCameraTransform:makeBlockPtr([weakThis = WeakPtr { *this }, completionHandler = WTFMove(completionHandler)] (simd_float3 cameraTransform, NSError *error) mutable { if (error) { callOnMainRunLoop([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)] () mutable { if (weakThis) completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::ResourceError::Type::General })); }); return; } callOnMainRunLoop([cameraTransform, weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)] () mutable { if (weakThis) completionHandler(WebCore::HTMLModelElementCamera { cameraTransform.x, cameraTransform.y, cameraTransform.z }); }); }).get()]; #else ASSERT_NOT_REACHED(); #endif } void ModelElementController::setCameraForModelElement(ModelIdentifier modelIdentifier, WebCore::HTMLModelElementCamera camera, CompletionHandler&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasCameraSupport(preview)) { completionHandler(false); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_CAMERA_TRANSFORM) [preview setCameraTransform:simd_make_float3(camera.pitch, camera.yaw, camera.scale)]; completionHandler(true); #else ASSERT_NOT_REACHED(); #endif } static bool previewHasAnimationSupport(ASVInlinePreview *preview) { #if ENABLE(ARKIT_INLINE_PREVIEW_ANIMATIONS_CONTROL) return [preview respondsToSelector:@selector(isPlaying)]; #else return false; #endif } void ModelElementController::isPlayingAnimationForModelElement(ModelIdentifier modelIdentifier, CompletionHandler)>&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasAnimationSupport(preview)) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::ResourceError::Type::General })); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_ANIMATIONS_CONTROL) completionHandler([preview isPlaying]); #else ASSERT_NOT_REACHED(); #endif } void ModelElementController::setAnimationIsPlayingForModelElement(ModelIdentifier modelIdentifier, bool isPlaying, CompletionHandler&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasAnimationSupport(preview)) { completionHandler(false); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_ANIMATIONS_CONTROL) [preview setIsPlaying:isPlaying reply:makeBlockPtr([weakThis = WeakPtr { *this }, completionHandler = WTFMove(completionHandler)] (BOOL, NSError *error) mutable { callOnMainRunLoop([error, weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)] () mutable { if (weakThis) completionHandler(!error); }); }).get()]; #else ASSERT_NOT_REACHED(); #endif } void ModelElementController::isLoopingAnimationForModelElement(ModelIdentifier modelIdentifier, CompletionHandler)>&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasAnimationSupport(preview)) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::ResourceError::Type::General })); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_ANIMATIONS_CONTROL) completionHandler([preview isLooping]); #else ASSERT_NOT_REACHED(); #endif } void ModelElementController::setIsLoopingAnimationForModelElement(ModelIdentifier modelIdentifier, bool isLooping, CompletionHandler&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasAnimationSupport(preview)) { completionHandler(false); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_ANIMATIONS_CONTROL) preview.isLooping = isLooping; completionHandler(true); #else ASSERT_NOT_REACHED(); #endif } void ModelElementController::animationDurationForModelElement(ModelIdentifier modelIdentifier, CompletionHandler)>&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasAnimationSupport(preview)) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::ResourceError::Type::General })); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_ANIMATIONS_CONTROL) completionHandler(Seconds([preview duration])); #else ASSERT_NOT_REACHED(); #endif } void ModelElementController::animationCurrentTimeForModelElement(ModelIdentifier modelIdentifier, CompletionHandler)>&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasAnimationSupport(preview)) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::ResourceError::Type::General })); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_ANIMATIONS_CONTROL) completionHandler(Seconds([preview currentTime])); #else ASSERT_NOT_REACHED(); #endif } void ModelElementController::setAnimationCurrentTimeForModelElement(ModelIdentifier modelIdentifier, Seconds currentTime, CompletionHandler&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasAnimationSupport(preview)) { completionHandler(false); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_ANIMATIONS_CONTROL) preview.currentTime = currentTime.seconds(); completionHandler(true); #else ASSERT_NOT_REACHED(); #endif } static bool previewHasAudioSupport(ASVInlinePreview *preview) { #if ENABLE(ARKIT_INLINE_PREVIEW_AUDIO_CONTROL) return [preview respondsToSelector:@selector(hasAudio)]; #else return false; #endif } void ModelElementController::hasAudioForModelElement(ModelIdentifier modelIdentifier, CompletionHandler)>&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasAudioSupport(preview)) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::ResourceError::Type::General })); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_AUDIO_CONTROL) completionHandler([preview hasAudio]); #else ASSERT_NOT_REACHED(); #endif } void ModelElementController::isMutedForModelElement(ModelIdentifier modelIdentifier, CompletionHandler)>&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasAudioSupport(preview)) { completionHandler(makeUnexpected(WebCore::ResourceError { WebCore::ResourceError::Type::General })); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_AUDIO_CONTROL) completionHandler([preview isMuted]); #else ASSERT_NOT_REACHED(); #endif } void ModelElementController::setIsMutedForModelElement(ModelIdentifier modelIdentifier, bool isMuted, CompletionHandler&& completionHandler) { auto* preview = previewForModelIdentifier(modelIdentifier); if (!previewHasAudioSupport(preview)) { completionHandler(false); return; } #if ENABLE(ARKIT_INLINE_PREVIEW_AUDIO_CONTROL) preview.isMuted = isMuted; completionHandler(true); #else ASSERT_NOT_REACHED(); #endif } #endif // ENABLE(ARKIT_INLINE_PREVIEW) } // namespace WebKit #endif // ENABLE(ARKIT_INLINE_PREVIEW)