/* * Copyright (C) 2017 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #import "config.h" #import "WebCoreDecompressionSession.h" #import "Logging.h" #import "PixelBufferConformerCV.h" #import #import #import #import #import #import #import #import #import #import #import #import "CoreVideoSoftLink.h" #import "VideoToolboxSoftLink.h" #import // Equivalent to WTF_DECLARE_CF_TYPE_TRAIT(CMSampleBuffer); // Needed due to requirement of specifying the PAL namespace. template <> struct WTF::CFTypeTrait { static inline CFTypeID typeID(void) { return PAL::CMSampleBufferGetTypeID(); } }; namespace WebCore { WebCoreDecompressionSession::WebCoreDecompressionSession(Mode mode) : m_mode(mode) , m_decompressionQueue(WorkQueue::create("WebCoreDecompressionSession Decompression Queue")) , m_enqueingQueue(WorkQueue::create("WebCoreDecompressionSession Enqueueing Queue")) { } void WebCoreDecompressionSession::invalidate() { m_invalidated = true; m_notificationCallback = nullptr; m_hasAvailableFrameCallback = nullptr; setTimebase(nullptr); if (m_timerSource) dispatch_source_cancel(m_timerSource.get()); } void WebCoreDecompressionSession::setTimebase(CMTimebaseRef timebase) { if (m_timebase == timebase) return; if (m_timebase) PAL::CMTimebaseRemoveTimerDispatchSource(m_timebase.get(), m_timerSource.get()); m_timebase = timebase; if (m_timebase) { if (!m_timerSource) { m_timerSource = adoptOSObject(dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, dispatch_get_main_queue())); dispatch_source_set_event_handler(m_timerSource.get(), [this] { automaticDequeue(); }); dispatch_activate(m_timerSource.get()); } PAL::CMTimebaseAddTimerDispatchSource(m_timebase.get(), m_timerSource.get()); } } void WebCoreDecompressionSession::maybeBecomeReadyForMoreMediaData() { if (!isReadyForMoreMediaData() || !m_notificationCallback) return; LOG(Media, "WebCoreDecompressionSession::maybeBecomeReadyForMoreMediaData(%p) - isReadyForMoreMediaData(%d), hasCallback(%d)", this, isReadyForMoreMediaData(), !!m_notificationCallback); ensureOnMainThread([protectedThis = Ref { *this }] { if (protectedThis->m_notificationCallback) protectedThis->m_notificationCallback(); }); } void WebCoreDecompressionSession::enqueueSample(CMSampleBufferRef sampleBuffer, bool displaying) { CMItemCount itemCount = 0; if (noErr != PAL::CMSampleBufferGetSampleTimingInfoArray(sampleBuffer, 0, nullptr, &itemCount)) return; Vector timingInfoArray; timingInfoArray.grow(itemCount); if (noErr != PAL::CMSampleBufferGetSampleTimingInfoArray(sampleBuffer, itemCount, timingInfoArray.data(), nullptr)) return; // CMBufferCallbacks contains 64-bit pointers that aren't 8-byte aligned. To suppress the linker // warning about this, we prepend 4 bytes of padding when building. const size_t padSize = 4; if (!m_producerQueue) { CMBufferQueueRef outQueue { nullptr }; #pragma pack(push, 4) struct BufferCallbacks { uint8_t pad[padSize]; CMBufferCallbacks callbacks; } callbacks { { }, { 0, nullptr, &getDecodeTime, &getPresentationTime, &getDuration, nullptr, &compareBuffers, nullptr, nullptr, } }; #pragma pack(pop) static_assert(sizeof(callbacks.callbacks.version) == sizeof(uint32_t), "Version field must be 4 bytes"); static_assert(alignof(BufferCallbacks) == 4, "CMBufferCallbacks struct must have 4 byte alignment"); PAL::CMBufferQueueCreate(kCFAllocatorDefault, kMaximumCapacity, &callbacks.callbacks, &outQueue); m_producerQueue = adoptCF(outQueue); } if (!m_consumerQueue) { CMBufferQueueRef outQueue { nullptr }; #pragma pack(push, 4) struct BufferCallbacks { uint8_t pad[padSize]; CMBufferCallbacks callbacks; } callbacks { { }, { 0, nullptr, &getDecodeTime, &getPresentationTime, &getDuration, nullptr, &compareBuffers, nullptr, nullptr, } }; #pragma pack(pop) static_assert(sizeof(callbacks.callbacks.version) == sizeof(uint32_t), "Version field must be 4 bytes"); static_assert(alignof(BufferCallbacks) == 4, "CMBufferCallbacks struct alignment must be 4"); PAL::CMBufferQueueCreate(kCFAllocatorDefault, kMaximumCapacity, &callbacks.callbacks, &outQueue); m_consumerQueue = adoptCF(outQueue); } ++m_framesBeingDecoded; LOG(Media, "WebCoreDecompressionSession::enqueueSample(%p) - framesBeingDecoded(%d)", this, m_framesBeingDecoded); m_decompressionQueue->dispatch([protectedThis = Ref { *this }, strongBuffer = retainPtr(sampleBuffer), displaying] { protectedThis->decodeSample(strongBuffer.get(), displaying); }); } bool WebCoreDecompressionSession::shouldDecodeSample(CMSampleBufferRef sample, bool displaying) { if (!displaying) return true; if (!m_timebase) return true; auto currentTime = PAL::CMTimebaseGetTime(m_timebase.get()); auto presentationStartTime = PAL::CMSampleBufferGetPresentationTimeStamp(sample); auto duration = PAL::CMSampleBufferGetDuration(sample); auto presentationEndTime = PAL::CMTimeAdd(presentationStartTime, duration); if (PAL::CMTimeCompare(presentationEndTime, currentTime) >= 0) return true; CFArrayRef attachments = PAL::CMSampleBufferGetSampleAttachmentsArray(sample, false); if (!attachments) return true; for (CFIndex index = 0, count = CFArrayGetCount(attachments); index < count; ++index) { CFDictionaryRef attachmentDict = (CFDictionaryRef)CFArrayGetValueAtIndex(attachments, index); if (CFDictionaryGetValue(attachmentDict, PAL::kCMSampleAttachmentKey_IsDependedOnByOthers) == kCFBooleanFalse) return false; } return true; } void WebCoreDecompressionSession::ensureDecompressionSessionForSample(CMSampleBufferRef sample) { if (isInvalidated()) return; CMVideoFormatDescriptionRef videoFormatDescription = PAL::CMSampleBufferGetFormatDescription(sample); if (m_decompressionSession && !VTDecompressionSessionCanAcceptFormatDescription(m_decompressionSession.get(), videoFormatDescription)) { VTDecompressionSessionWaitForAsynchronousFrames(m_decompressionSession.get()); m_decompressionSession = nullptr; } if (!m_decompressionSession) { CMVideoFormatDescriptionRef videoFormatDescription = PAL::CMSampleBufferGetFormatDescription(sample); auto videoDecoderSpecification = @{ (__bridge NSString *)kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder: @( m_hardwareDecoderEnabled ) }; NSDictionary *attributes; if (m_mode == OpenGL) attributes = @{ (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{ /* empty dictionary */ } }; else { attributes = @{ (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{ /* empty dictionary */ } }; } VTDecompressionSessionRef decompressionSessionOut = nullptr; if (noErr == VTDecompressionSessionCreate(kCFAllocatorDefault, videoFormatDescription, (__bridge CFDictionaryRef)videoDecoderSpecification, (__bridge CFDictionaryRef)attributes, nullptr, &decompressionSessionOut)) { m_decompressionSession = adoptCF(decompressionSessionOut); CFArrayRef rawSuggestedQualityOfServiceTiers = nullptr; VTSessionCopyProperty(decompressionSessionOut, kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers, kCFAllocatorDefault, &rawSuggestedQualityOfServiceTiers); m_qosTiers = adoptCF(rawSuggestedQualityOfServiceTiers); m_currentQosTier = 0; resetQosTier(); } } } void WebCoreDecompressionSession::decodeSample(CMSampleBufferRef sample, bool displaying) { if (isInvalidated()) return; ensureDecompressionSessionForSample(sample); VTDecodeInfoFlags flags { kVTDecodeFrame_EnableTemporalProcessing }; if (!displaying) flags |= kVTDecodeFrame_DoNotOutputFrame; if (!shouldDecodeSample(sample, displaying)) { ++m_totalVideoFrames; ++m_droppedVideoFrames; --m_framesBeingDecoded; maybeBecomeReadyForMoreMediaData(); return; } MonotonicTime startTime = MonotonicTime::now(); VTDecompressionSessionDecodeFrameWithOutputHandler(m_decompressionSession.get(), sample, flags, nullptr, [this, displaying, startTime](OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) { double deltaRatio = (MonotonicTime::now() - startTime).seconds() / PAL::CMTimeGetSeconds(presentationDuration); updateQosWithDecodeTimeStatistics(deltaRatio); handleDecompressionOutput(displaying, status, infoFlags, imageBuffer, presentationTimeStamp, presentationDuration); }); } RetainPtr WebCoreDecompressionSession::decodeSampleSync(CMSampleBufferRef sample) { if (isInvalidated()) return nullptr; ensureDecompressionSessionForSample(sample); RetainPtr pixelBuffer; VTDecodeInfoFlags flags { 0 }; WTF::Semaphore syncDecompressionOutputSemaphore { 0 }; VTDecompressionSessionDecodeFrameWithOutputHandler(m_decompressionSession.get(), sample, flags, nullptr, [&] (OSStatus, VTDecodeInfoFlags, CVImageBufferRef imageBuffer, CMTime, CMTime) mutable { if (imageBuffer && CFGetTypeID(imageBuffer) == CVPixelBufferGetTypeID()) pixelBuffer = (CVPixelBufferRef)imageBuffer; syncDecompressionOutputSemaphore.signal(); }); syncDecompressionOutputSemaphore.wait(); return pixelBuffer; } void WebCoreDecompressionSession::handleDecompressionOutput(bool displaying, OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef rawImageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) { ++m_totalVideoFrames; if (infoFlags & kVTDecodeInfo_FrameDropped) ++m_droppedVideoFrames; CMVideoFormatDescriptionRef rawImageBufferDescription = nullptr; if (status != noErr || noErr != PAL::CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, rawImageBuffer, &rawImageBufferDescription)) { ++m_corruptedVideoFrames; --m_framesBeingDecoded; maybeBecomeReadyForMoreMediaData(); return; } RetainPtr imageBufferDescription = adoptCF(rawImageBufferDescription); CMSampleTimingInfo imageBufferTiming { presentationDuration, presentationTimeStamp, presentationTimeStamp, }; CMSampleBufferRef rawImageSampleBuffer = nullptr; if (noErr != PAL::CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, rawImageBuffer, imageBufferDescription.get(), &imageBufferTiming, &rawImageSampleBuffer)) { ++m_corruptedVideoFrames; --m_framesBeingDecoded; maybeBecomeReadyForMoreMediaData(); return; } m_enqueingQueue->dispatch([protectedThis = Ref { *this }, imageSampleBuffer = adoptCF(rawImageSampleBuffer), displaying] { protectedThis->enqueueDecodedSample(imageSampleBuffer.get(), displaying); }); } RetainPtr WebCoreDecompressionSession::getFirstVideoFrame() { if (!m_producerQueue || PAL::CMBufferQueueIsEmpty(m_producerQueue.get())) return nullptr; RetainPtr currentSample = adoptCF(checked_cf_cast(PAL::CMBufferQueueDequeueAndRetain(m_producerQueue.get()))); RetainPtr imageBuffer = (CVPixelBufferRef)PAL::CMSampleBufferGetImageBuffer(currentSample.get()); ASSERT(CFGetTypeID(imageBuffer.get()) == CVPixelBufferGetTypeID()); maybeBecomeReadyForMoreMediaData(); return imageBuffer; } void WebCoreDecompressionSession::automaticDequeue() { if (!m_timebase) return; auto time = PAL::toMediaTime(PAL::CMTimebaseGetTime(m_timebase.get())); LOG(Media, "WebCoreDecompressionSession::automaticDequeue(%p) - purging all samples before time(%s)", this, toString(time).utf8().data()); MediaTime nextFireTime = MediaTime::positiveInfiniteTime(); bool releasedImageBuffers = false; while (CMSampleBufferRef firstSample = checked_cf_cast(PAL::CMBufferQueueGetHead(m_producerQueue.get()))) { MediaTime presentationTimestamp = PAL::toMediaTime(PAL::CMSampleBufferGetPresentationTimeStamp(firstSample)); MediaTime duration = PAL::toMediaTime(PAL::CMSampleBufferGetDuration(firstSample)); MediaTime presentationEndTimestamp = presentationTimestamp + duration; if (time > presentationEndTimestamp) { CFRelease(PAL::CMBufferQueueDequeueAndRetain(m_producerQueue.get())); releasedImageBuffers = true; continue; } #if !LOG_DISABLED auto begin = PAL::toMediaTime(PAL::CMBufferQueueGetFirstPresentationTimeStamp(m_producerQueue.get())); auto end = PAL::toMediaTime(PAL::CMBufferQueueGetEndPresentationTimeStamp(m_producerQueue.get())); LOG(Media, "WebCoreDecompressionSession::automaticDequeue(%p) - queue(%s -> %s)", this, toString(begin).utf8().data(), toString(end).utf8().data()); #endif nextFireTime = presentationEndTimestamp; break; } if (releasedImageBuffers) maybeBecomeReadyForMoreMediaData(); LOG(Media, "WebCoreDecompressionSession::automaticDequeue(%p) - queue empty", this); PAL::CMTimebaseSetTimerDispatchSourceNextFireTime(m_timebase.get(), m_timerSource.get(), PAL::toCMTime(nextFireTime), 0); } void WebCoreDecompressionSession::enqueueDecodedSample(CMSampleBufferRef sample, bool displaying) { if (isInvalidated()) return; --m_framesBeingDecoded; if (!displaying) { maybeBecomeReadyForMoreMediaData(); return; } bool shouldNotify = true; if (displaying && m_timebase) { auto currentRate = PAL::CMTimebaseGetRate(m_timebase.get()); auto currentTime = PAL::toMediaTime(PAL::CMTimebaseGetTime(m_timebase.get())); auto presentationStartTime = PAL::toMediaTime(PAL::CMSampleBufferGetPresentationTimeStamp(sample)); auto presentationEndTime = presentationStartTime + PAL::toMediaTime(PAL::CMSampleBufferGetDuration(sample)); if (currentTime < presentationStartTime || currentTime >= presentationEndTime) shouldNotify = false; if (currentRate > 0 && presentationEndTime < currentTime) { #if !LOG_DISABLED auto begin = PAL::toMediaTime(PAL::CMBufferQueueGetFirstPresentationTimeStamp(m_producerQueue.get())); auto end = PAL::toMediaTime(PAL::CMBufferQueueGetEndPresentationTimeStamp(m_producerQueue.get())); LOG(Media, "WebCoreDecompressionSession::enqueueDecodedSample(%p) - dropping frame late by %s, framesBeingDecoded(%d), producerQueue(%s -> %s)", this, toString(presentationEndTime - currentTime).utf8().data(), m_framesBeingDecoded, toString(begin).utf8().data(), toString(end).utf8().data()); #endif ++m_droppedVideoFrames; return; } } PAL::CMBufferQueueEnqueue(m_producerQueue.get(), sample); #if !LOG_DISABLED auto begin = PAL::toMediaTime(PAL::CMBufferQueueGetFirstPresentationTimeStamp(m_producerQueue.get())); auto end = PAL::toMediaTime(PAL::CMBufferQueueGetEndPresentationTimeStamp(m_producerQueue.get())); auto presentationTime = PAL::toMediaTime(PAL::CMSampleBufferGetPresentationTimeStamp(sample)); LOG(Media, "WebCoreDecompressionSession::enqueueDecodedSample(%p) - presentationTime(%s), framesBeingDecoded(%d), producerQueue(%s -> %s)", this, toString(presentationTime).utf8().data(), m_framesBeingDecoded, toString(begin).utf8().data(), toString(end).utf8().data()); #endif if (m_timebase) PAL::CMTimebaseSetTimerDispatchSourceToFireImmediately(m_timebase.get(), m_timerSource.get()); if (!m_hasAvailableFrameCallback) return; if (!shouldNotify) return; RunLoop::main().dispatch([protectedThis = Ref { *this }, callback = WTFMove(m_hasAvailableFrameCallback)] { callback(); }); } bool WebCoreDecompressionSession::isReadyForMoreMediaData() const { CMItemCount producerCount = m_producerQueue ? PAL::CMBufferQueueGetBufferCount(m_producerQueue.get()) : 0; return m_framesBeingDecoded + producerCount <= kHighWaterMark; } void WebCoreDecompressionSession::requestMediaDataWhenReady(std::function notificationCallback) { LOG(Media, "WebCoreDecompressionSession::requestMediaDataWhenReady(%p), hasNotificationCallback(%d)", this, !!notificationCallback); m_notificationCallback = notificationCallback; if (notificationCallback && isReadyForMoreMediaData()) { RefPtr protectedThis { this }; RunLoop::main().dispatch([protectedThis] { if (protectedThis->m_notificationCallback) protectedThis->m_notificationCallback(); }); } } void WebCoreDecompressionSession::stopRequestingMediaData() { LOG(Media, "WebCoreDecompressionSession::stopRequestingMediaData(%p)", this); m_notificationCallback = nullptr; } void WebCoreDecompressionSession::notifyWhenHasAvailableVideoFrame(std::function callback) { if (callback && m_producerQueue && !PAL::CMBufferQueueIsEmpty(m_producerQueue.get())) { RunLoop::main().dispatch([callback] { callback(); }); return; } m_hasAvailableFrameCallback = callback; } RetainPtr WebCoreDecompressionSession::imageForTime(const MediaTime& time, ImageForTimeFlags flags) { if (PAL::CMBufferQueueIsEmpty(m_producerQueue.get())) { LOG(Media, "WebCoreDecompressionSession::imageForTime(%p) - time(%s), queue empty", this, toString(time).utf8().data()); return nullptr; } bool allowEarlier = flags == WebCoreDecompressionSession::AllowEarlier; bool allowLater = flags == WebCoreDecompressionSession::AllowLater; MediaTime startTime = PAL::toMediaTime(PAL::CMBufferQueueGetFirstPresentationTimeStamp(m_producerQueue.get())); #if !LOG_DISABLED MediaTime endTime = PAL::toMediaTime(PAL::CMBufferQueueGetEndPresentationTimeStamp(m_producerQueue.get())); #endif if (!allowLater && time < startTime) { LOG(Media, "WebCoreDecompressionSession::imageForTime(%p) - time(%s) too early for queue(%s -> %s)", this, toString(time).utf8().data(), toString(startTime).utf8().data(), toString(endTime).utf8().data()); return nullptr; } bool releasedImageBuffers = false; while (CMSampleBufferRef firstSample = checked_cf_cast(PAL::CMBufferQueueGetHead(m_producerQueue.get()))) { MediaTime presentationTimestamp = PAL::toMediaTime(PAL::CMSampleBufferGetPresentationTimeStamp(firstSample)); MediaTime duration = PAL::toMediaTime(PAL::CMSampleBufferGetDuration(firstSample)); MediaTime presentationEndTimestamp = presentationTimestamp + duration; if (!allowLater && presentationTimestamp > time) return nullptr; if (!allowEarlier && presentationEndTimestamp < time) { CFRelease(PAL::CMBufferQueueDequeueAndRetain(m_producerQueue.get())); releasedImageBuffers = true; continue; } RetainPtr currentSample = adoptCF(checked_cf_cast(PAL::CMBufferQueueDequeueAndRetain(m_producerQueue.get()))); RetainPtr imageBuffer = (CVPixelBufferRef)PAL::CMSampleBufferGetImageBuffer(currentSample.get()); ASSERT(CFGetTypeID(imageBuffer.get()) == CVPixelBufferGetTypeID()); if (m_timebase) PAL::CMTimebaseSetTimerDispatchSourceToFireImmediately(m_timebase.get(), m_timerSource.get()); maybeBecomeReadyForMoreMediaData(); LOG(Media, "WebCoreDecompressionSession::imageForTime(%p) - found sample for time(%s) in queue(%s -> %s)", this, toString(time).utf8().data(), toString(startTime).utf8().data(), toString(endTime).utf8().data()); return imageBuffer; } if (m_timebase) PAL::CMTimebaseSetTimerDispatchSourceToFireImmediately(m_timebase.get(), m_timerSource.get()); if (releasedImageBuffers) maybeBecomeReadyForMoreMediaData(); LOG(Media, "WebCoreDecompressionSession::imageForTime(%p) - no matching sample for time(%s) in queue(%s -> %s)", this, toString(time).utf8().data(), toString(startTime).utf8().data(), toString(endTime).utf8().data()); return nullptr; } void WebCoreDecompressionSession::flush() { m_decompressionQueue->dispatchSync([this, protectedThis = Ref { *this }]() mutable { PAL::CMBufferQueueReset(protectedThis->m_producerQueue.get()); m_enqueingQueue->dispatchSync([protectedThis = WTFMove(protectedThis)] { PAL::CMBufferQueueReset(protectedThis->m_consumerQueue.get()); protectedThis->m_framesSinceLastQosCheck = 0; protectedThis->m_currentQosTier = 0; protectedThis->resetQosTier(); }); }); } CMTime WebCoreDecompressionSession::getDecodeTime(CMBufferRef buf, void*) { CMSampleBufferRef sample = checked_cf_cast(buf); return PAL::CMSampleBufferGetDecodeTimeStamp(sample); } CMTime WebCoreDecompressionSession::getPresentationTime(CMBufferRef buf, void*) { CMSampleBufferRef sample = checked_cf_cast(buf); return PAL::CMSampleBufferGetPresentationTimeStamp(sample); } CMTime WebCoreDecompressionSession::getDuration(CMBufferRef buf, void*) { CMSampleBufferRef sample = checked_cf_cast(buf); return PAL::CMSampleBufferGetDuration(sample); } CFComparisonResult WebCoreDecompressionSession::compareBuffers(CMBufferRef buf1, CMBufferRef buf2, void* refcon) { return (CFComparisonResult)PAL::CMTimeCompare(getPresentationTime(buf1, refcon), getPresentationTime(buf2, refcon)); } void WebCoreDecompressionSession::resetQosTier() { if (!m_qosTiers || !m_decompressionSession) return; if (m_currentQosTier < 0 || m_currentQosTier >= CFArrayGetCount(m_qosTiers.get())) return; auto tier = (CFDictionaryRef)CFArrayGetValueAtIndex(m_qosTiers.get(), m_currentQosTier); LOG(Media, "WebCoreDecompressionSession::resetQosTier(%p) - currentQosTier(%ld), tier(%@)", this, m_currentQosTier, [(__bridge NSDictionary *)tier description]); VTSessionSetProperties(m_decompressionSession.get(), tier); m_framesSinceLastQosCheck = 0; } void WebCoreDecompressionSession::increaseQosTier() { if (!m_qosTiers) return; if (m_currentQosTier + 1 >= CFArrayGetCount(m_qosTiers.get())) return; ++m_currentQosTier; resetQosTier(); } void WebCoreDecompressionSession::decreaseQosTier() { if (!m_qosTiers) return; if (m_currentQosTier <= 0) return; --m_currentQosTier; resetQosTier(); } void WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics(double ratio) { static const double kMovingAverageAlphaValue = 0.1; static const unsigned kNumberOfFramesBeforeSwitchingTiers = 60; static const double kHighWaterDecodeRatio = 1.; static const double kLowWaterDecodeRatio = 0.5; if (!m_timebase) return; double rate = PAL::CMTimebaseGetRate(m_timebase.get()); if (!rate) rate = 1; m_decodeRatioMovingAverage += kMovingAverageAlphaValue * (ratio - m_decodeRatioMovingAverage) * rate; if (++m_framesSinceLastQosCheck < kNumberOfFramesBeforeSwitchingTiers) return; LOG(Media, "WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics(%p) - framesSinceLastQosCheck(%ld), decodeRatioMovingAverage(%g)", this, m_framesSinceLastQosCheck, m_decodeRatioMovingAverage); if (m_decodeRatioMovingAverage > kHighWaterDecodeRatio) increaseQosTier(); else if (m_decodeRatioMovingAverage < kLowWaterDecodeRatio) decreaseQosTier(); m_framesSinceLastQosCheck = 0; } }