/* * Copyright (C) 2020 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #import "config.h" #import "VideoSampleBufferCompressor.h" #if ENABLE(MEDIA_RECORDER) && USE(AVFOUNDATION) #import "ContentType.h" #import "Logging.h" #import #import #import #import #import #import namespace WebCore { std::unique_ptr VideoSampleBufferCompressor::create(String mimeType, CMBufferQueueTriggerCallback callback, void* callbackObject) { auto profile = Profile::Baseline; for (auto codec : ContentType(mimeType).codecs()) { if (startsWithLettersIgnoringASCIICase(codec, "avc1."_s) && codec.length() >= 11) { if (codec[5] == '6' && codec[6] == '4') profile = Profile::High; else if (codec[5] == '4' && (codec[6] == 'd' || codec[6] == 'D')) profile = Profile::Main; break; } } auto compressor = std::unique_ptr(new VideoSampleBufferCompressor(kCMVideoCodecType_H264, profile)); if (!compressor->initialize(callback, callbackObject)) return nullptr; return compressor; } VideoSampleBufferCompressor::VideoSampleBufferCompressor(CMVideoCodecType outputCodecType, Profile profile) : m_serialDispatchQueue { WorkQueue::create("com.apple.VideoSampleBufferCompressor") } , m_outputCodecType { outputCodecType } , m_profile { profile } { } VideoSampleBufferCompressor::~VideoSampleBufferCompressor() { if (m_vtSession) { PAL::VTCompressionSessionInvalidate(m_vtSession.get()); m_vtSession = nullptr; } } bool VideoSampleBufferCompressor::initialize(CMBufferQueueTriggerCallback callback, void* callbackObject) { CMBufferQueueRef outputBufferQueue; if (auto error = PAL::CMBufferQueueCreate(kCFAllocatorDefault, 0, PAL::CMBufferQueueGetCallbacksForUnsortedSampleBuffers(), &outputBufferQueue)) { RELEASE_LOG_ERROR(MediaStream, "VideoSampleBufferCompressor unable to create buffer queue %d", error); return false; } m_outputBufferQueue = adoptCF(outputBufferQueue); PAL::CMBufferQueueInstallTrigger(m_outputBufferQueue.get(), callback, callbackObject, kCMBufferQueueTrigger_WhenDataBecomesReady, PAL::kCMTimeZero, NULL); m_isEncoding = true; return true; } void VideoSampleBufferCompressor::setBitsPerSecond(unsigned bitRate) { m_outputBitRate = bitRate; } void VideoSampleBufferCompressor::finish() { m_serialDispatchQueue->dispatchSync([this] { auto error = PAL::VTCompressionSessionCompleteFrames(m_vtSession.get(), PAL::kCMTimeInvalid); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTCompressionSessionCompleteFrames failed with %d", error); error = PAL::CMBufferQueueMarkEndOfData(m_outputBufferQueue.get()); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor CMBufferQueueMarkEndOfData failed with %d", error); m_isEncoding = false; }); } void VideoSampleBufferCompressor::videoCompressionCallback(void *refCon, void*, OSStatus status, VTEncodeInfoFlags, CMSampleBufferRef buffer) { RELEASE_LOG_ERROR_IF(status, MediaStream, "VideoSampleBufferCompressor videoCompressionCallback status is %d", status); if (status != noErr) return; VideoSampleBufferCompressor *compressor = static_cast(refCon); auto error = PAL::CMBufferQueueEnqueue(compressor->m_outputBufferQueue.get(), buffer); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor CMBufferQueueEnqueue failed with %d", error); } static inline OSStatus setCompressionSessionProperty(VTCompressionSessionRef vtSession, CFStringRef key, uint32_t value) { int64_t value64 = value; auto cfValue = adoptCF(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value64)); OSStatus status = VTSessionSetProperty(vtSession, key, cfValue.get()); return status; } CFStringRef VideoSampleBufferCompressor::vtProfileLevel() const { switch (m_profile) { case Profile::Baseline: return PAL::kVTProfileLevel_H264_Baseline_AutoLevel; case Profile::High: return PAL::kVTProfileLevel_H264_High_AutoLevel; case Profile::Main: return PAL::kVTProfileLevel_H264_Main_AutoLevel; } } bool VideoSampleBufferCompressor::initCompressionSession(CMVideoFormatDescriptionRef formatDescription) { CMVideoDimensions dimensions = PAL::CMVideoFormatDescriptionGetDimensions(formatDescription); #if PLATFORM(IOS) NSDictionary *encoderSpecifications = nil; #else NSDictionary *encoderSpecifications = @{(__bridge NSString *)PAL::kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder:@YES}; #endif VTCompressionSessionRef vtSession; auto error = PAL::VTCompressionSessionCreate(kCFAllocatorDefault, dimensions.width, dimensions.height, m_outputCodecType, (__bridge CFDictionaryRef)encoderSpecifications, NULL, NULL, videoCompressionCallback, this, &vtSession); if (error) { RELEASE_LOG_ERROR(MediaStream, "VideoSampleBufferCompressor VTCompressionSessionCreate failed with %d", error); return NO; } m_vtSession = adoptCF(vtSession); error = VTSessionSetProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_RealTime, kCFBooleanTrue); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_RealTime failed with %d", error); error = setCompressionSessionProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, m_maxKeyFrameIntervalDuration); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration failed with %d", error); error = setCompressionSessionProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_MaxKeyFrameInterval, m_maxKeyFrameIntervalDuration * m_expectedFrameRate); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_MaxKeyFrameInterval failed with %d", error); error = setCompressionSessionProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_ExpectedFrameRate, m_expectedFrameRate); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_ExpectedFrameRate failed with %d", error); error = VTSessionSetProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_ProfileLevel, vtProfileLevel()); if (error) { RELEASE_LOG_ERROR(MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_ProfileLevel failed with %d for profile %d", error, m_profile); if (m_profile != Profile::Baseline) { error = VTSessionSetProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_ProfileLevel, PAL::kVTProfileLevel_H264_Baseline_AutoLevel); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_ProfileLevel failed with %d for default profile", error); } } if (m_outputBitRate) { error = setCompressionSessionProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_AverageBitRate, *m_outputBitRate); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_AverageBitRate failed with %d", error); } error = PAL::VTCompressionSessionPrepareToEncodeFrames(m_vtSession.get()); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTCompressionSessionPrepareToEncodeFrames failed with %d", error); return YES; } void VideoSampleBufferCompressor::processSampleBuffer(CMSampleBufferRef buffer) { if (!m_vtSession) { if (!initCompressionSession(PAL::CMSampleBufferGetFormatDescription(buffer))) return; } auto imageBuffer = PAL::CMSampleBufferGetImageBuffer(buffer); auto presentationTimeStamp = PAL::CMSampleBufferGetPresentationTimeStamp(buffer); auto duration = PAL::CMSampleBufferGetDuration(buffer); auto error = PAL::VTCompressionSessionEncodeFrame(m_vtSession.get(), imageBuffer, presentationTimeStamp, duration, NULL, this, NULL); RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTCompressionSessionEncodeFrame failed with %d", error); } void VideoSampleBufferCompressor::addSampleBuffer(CMSampleBufferRef buffer) { m_serialDispatchQueue->dispatchSync([this, buffer] { if (!m_isEncoding) return; processSampleBuffer(buffer); }); } CMSampleBufferRef VideoSampleBufferCompressor::getOutputSampleBuffer() { return (CMSampleBufferRef)(const_cast(PAL::CMBufferQueueGetHead(m_outputBufferQueue.get()))); } RetainPtr VideoSampleBufferCompressor::takeOutputSampleBuffer() { return adoptCF((CMSampleBufferRef)(const_cast(PAL::CMBufferQueueDequeueAndRetain(m_outputBufferQueue.get())))); } unsigned VideoSampleBufferCompressor::bitRate() const { return m_outputBitRate.value_or(0); } } #endif