/* * Copyright (C) 2020 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #import "config.h" #import "SourceBufferParserAVFObjC.h" #if ENABLE(MEDIA_SOURCE) #import "AVAssetMIMETypeCache.h" #import "AVAssetTrackUtilities.h" #import "AVStreamDataParserMIMETypeCache.h" #import "AudioTrackPrivateMediaSourceAVFObjC.h" #import "FourCC.h" #import "InbandTextTrackPrivateAVFObjC.h" #import "Logging.h" #import "MediaDescription.h" #import "MediaSample.h" #import "MediaSampleAVFObjC.h" #import "NotImplemented.h" #import "SharedBuffer.h" #import "SourceBufferPrivate.h" #import "TimeRanges.h" #import "VideoTrackPrivateMediaSourceAVFObjC.h" #import #import #import #import #import #import #import #import #import #pragma mark - #pragma mark WebAVStreamDataParserListener #if HAVE(AVCONTENTKEYSPECIFIER) @interface AVContentKeySpecifier (WebCorePrivate) @property (readonly) NSData *initializationData; @end #endif @interface WebAVStreamDataParserListener : NSObject { WebCore::SourceBufferParserAVFObjC* _parent; AVStreamDataParser* _parser; } @property (assign) WebCore::SourceBufferParserAVFObjC* parent; - (id)initWithParser:(AVStreamDataParser*)parser parent:(WebCore::SourceBufferParserAVFObjC*)parent; @end @implementation WebAVStreamDataParserListener - (id)initWithParser:(AVStreamDataParser*)parser parent:(WebCore::SourceBufferParserAVFObjC*)parent { self = [super init]; if (!self) return nil; ASSERT(parent); _parent = parent; _parser = parser; [_parser setDelegate:self]; return self; } @synthesize parent = _parent; - (void)dealloc { [_parser setDelegate:nil]; [super dealloc]; } - (void)invalidate { [_parser setDelegate:nil]; _parser = nullptr; } - (void)streamDataParser:(AVStreamDataParser *)streamDataParser didParseStreamDataAsAsset:(AVAsset *)asset { ASSERT_UNUSED(streamDataParser, streamDataParser == _parser); _parent->didParseStreamDataAsAsset(asset); } - (void)streamDataParser:(AVStreamDataParser *)streamDataParser didParseStreamDataAsAsset:(AVAsset *)asset withDiscontinuity:(BOOL)discontinuity { UNUSED_PARAM(discontinuity); ASSERT_UNUSED(streamDataParser, streamDataParser == _parser); _parent->didParseStreamDataAsAsset(asset); } - (void)streamDataParser:(AVStreamDataParser *)streamDataParser didFailToParseStreamDataWithError:(NSError *)error { ASSERT_UNUSED(streamDataParser, streamDataParser == _parser); _parent->didFailToParseStreamDataWithError(error); } - (void)streamDataParser:(AVStreamDataParser *)streamDataParser didProvideMediaData:(CMSampleBufferRef)sample forTrackID:(CMPersistentTrackID)trackID mediaType:(NSString *)nsMediaType flags:(AVStreamDataParserOutputMediaDataFlags)flags { ASSERT_UNUSED(streamDataParser, streamDataParser == _parser); _parent->didProvideMediaDataForTrackID(trackID, sample, nsMediaType, flags); } ALLOW_DEPRECATED_IMPLEMENTATIONS_BEGIN - (void)streamDataParserWillProvideContentKeyRequestInitializationData:(AVStreamDataParser *)streamDataParser forTrackID:(CMPersistentTrackID)trackID ALLOW_DEPRECATED_IMPLEMENTATIONS_END { ASSERT_UNUSED(streamDataParser, streamDataParser == _parser); _parent->willProvideContentKeyRequestInitializationDataForTrackID(trackID); } ALLOW_DEPRECATED_IMPLEMENTATIONS_BEGIN - (void)streamDataParser:(AVStreamDataParser *)streamDataParser didProvideContentKeyRequestInitializationData:(NSData *)initData forTrackID:(CMPersistentTrackID)trackID ALLOW_DEPRECATED_IMPLEMENTATIONS_END { ASSERT_UNUSED(streamDataParser, streamDataParser == _parser); _parent->didProvideContentKeyRequestInitializationDataForTrackID(initData, trackID); } @end #if HAVE(AVCONTENTKEYSPECIFIER) @interface WebAVStreamDataParserWithKeySpecifierListener : WebAVStreamDataParserListener @end @implementation WebAVStreamDataParserWithKeySpecifierListener - (void)streamDataParser:(AVStreamDataParser *)streamDataParser didProvideContentKeySpecifier:(AVContentKeySpecifier *)keySpecifier forTrackID:(CMPersistentTrackID)trackID { ASSERT_UNUSED(streamDataParser, streamDataParser == _parser); if ([keySpecifier respondsToSelector:@selector(initializationData)]) _parent->didProvideContentKeyRequestSpecifierForTrackID(keySpecifier.initializationData, trackID); } @end #endif namespace WebCore { #pragma mark - #pragma mark MediaDescriptionAVFObjC class MediaDescriptionAVFObjC final : public MediaDescription { public: static Ref create(AVAssetTrack* track) { return adoptRef(*new MediaDescriptionAVFObjC(track)); } virtual ~MediaDescriptionAVFObjC() { } bool isVideo() const override { return m_isVideo; } bool isAudio() const override { return m_isAudio; } bool isText() const override { return m_isText; } AtomString codec() const override { // AtomStrings can only be destroyed from the same thread they're // created in. Since this structure is created in a parser thread // only create the AtomString the first time this function is accessed // which is presumably on the main thread. ASSERT(isMainThread()); if (!m_codec) m_codec = AtomString::fromLatin1(m_originalCodec.string().data()); return *m_codec; } private: MediaDescriptionAVFObjC(AVAssetTrack* track) : m_isVideo([track hasMediaCharacteristic:AVMediaCharacteristicVisual]) , m_isAudio([track hasMediaCharacteristic:AVMediaCharacteristicAudible]) , m_isText([track hasMediaCharacteristic:AVMediaCharacteristicLegible]) { NSArray* formatDescriptions = [track formatDescriptions]; CMFormatDescriptionRef description = [formatDescriptions count] ? (__bridge CMFormatDescriptionRef)[formatDescriptions objectAtIndex:0] : 0; if (description) { m_originalCodec = PAL::softLink_CoreMedia_CMFormatDescriptionGetMediaSubType(description); CFStringRef originalFormatKey = PAL::canLoad_CoreMedia_kCMFormatDescriptionExtension_ProtectedContentOriginalFormat() ? PAL::get_CoreMedia_kCMFormatDescriptionExtension_ProtectedContentOriginalFormat() : CFSTR("CommonEncryptionOriginalFormat"); if (auto originalFormat = dynamic_cf_cast(PAL::CMFormatDescriptionGetExtension(description, originalFormatKey))) CFNumberGetValue(originalFormat, kCFNumberSInt32Type, &m_originalCodec.value); } } FourCC m_originalCodec; mutable std::optional m_codec; bool m_isVideo; bool m_isAudio; bool m_isText; }; #pragma mark - #pragma mark SourceBufferParserAVFObjC MediaPlayerEnums::SupportsType SourceBufferParserAVFObjC::isContentTypeSupported(const ContentType& type) { String extendedType = type.raw(); String outputCodecs = type.parameter(ContentType::codecsParameter()); if (!outputCodecs.isEmpty() && [PAL::getAVStreamDataParserClass() respondsToSelector:@selector(outputMIMECodecParameterForInputMIMECodecParameter:)]) { outputCodecs = [PAL::getAVStreamDataParserClass() outputMIMECodecParameterForInputMIMECodecParameter:outputCodecs]; extendedType = makeString(type.containerType(), "; codecs=\"", outputCodecs, "\""); } return AVStreamDataParserMIMETypeCache::singleton().canDecodeType(extendedType); } SourceBufferParserAVFObjC::SourceBufferParserAVFObjC() : m_parser(adoptNS([PAL::allocAVStreamDataParserInstance() init])) { #if HAVE(AVCONTENTKEYSPECIFIER) if (MediaSessionManagerCocoa::sampleBufferContentKeySessionSupportEnabled()) m_delegate = adoptNS([[WebAVStreamDataParserWithKeySpecifierListener alloc] initWithParser:m_parser.get() parent:this]); else #endif m_delegate = adoptNS([[WebAVStreamDataParserListener alloc] initWithParser:m_parser.get() parent:this]); } SourceBufferParserAVFObjC::~SourceBufferParserAVFObjC() { m_delegate.get().parent = nullptr; [m_delegate invalidate]; } void SourceBufferParserAVFObjC::appendData(Segment&& segment, CompletionHandler&& completionHandler, AppendFlags flags) { auto sharedBuffer = segment.takeSharedBuffer(); auto nsData = sharedBuffer->makeContiguous()->createNSData(); if (m_parserStateWasReset || flags == AppendFlags::Discontinuity) [m_parser appendStreamData:nsData.get() withFlags:AVStreamDataParserStreamDataDiscontinuity]; else [m_parser appendStreamData:nsData.get()]; m_parserStateWasReset = false; completionHandler(); } void SourceBufferParserAVFObjC::flushPendingMediaData() { [m_parser providePendingMediaData]; } void SourceBufferParserAVFObjC::setShouldProvideMediaDataForTrackID(bool should, uint64_t trackID) { BEGIN_BLOCK_OBJC_EXCEPTIONS [m_parser setShouldProvideMediaData:should forTrackID:trackID]; END_BLOCK_OBJC_EXCEPTIONS } bool SourceBufferParserAVFObjC::shouldProvideMediadataForTrackID(uint64_t trackID) { return [m_parser shouldProvideMediaDataForTrackID:trackID]; } void SourceBufferParserAVFObjC::resetParserState() { m_parserStateWasReset = true; } void SourceBufferParserAVFObjC::invalidate() { [m_delegate invalidate]; m_delegate = nullptr; m_parser = nullptr; } #if !RELEASE_LOG_DISABLED void SourceBufferParserAVFObjC::setLogger(const Logger& logger, const void* logIdentifier) { m_logger = &logger; m_logIdentifier = logIdentifier; } #endif void SourceBufferParserAVFObjC::didParseStreamDataAsAsset(AVAsset* asset) { m_callOnClientThreadCallback([this, strongThis = Ref { *this }, asset = retainPtr(asset)] { if (!m_didParseInitializationDataCallback) return; InitializationSegment segment; if ([asset respondsToSelector:@selector(overallDurationHint)]) segment.duration = PAL::toMediaTime([asset overallDurationHint]); if (segment.duration.isInvalid() || segment.duration == MediaTime::zeroTime()) segment.duration = PAL::toMediaTime([asset duration]); for (AVAssetTrack* track in [asset tracks]) { if ([track hasMediaCharacteristic:AVMediaCharacteristicLegible]) { // FIXME(125161): Handle in-band text tracks. continue; } if ([track hasMediaCharacteristic:AVMediaCharacteristicVisual]) { SourceBufferPrivateClient::InitializationSegment::VideoTrackInformation info; info.track = VideoTrackPrivateMediaSourceAVFObjC::create(track); info.description = MediaDescriptionAVFObjC::create(track); segment.videoTracks.append(WTFMove(info)); } else if ([track hasMediaCharacteristic:AVMediaCharacteristicAudible]) { SourceBufferPrivateClient::InitializationSegment::AudioTrackInformation info; info.track = AudioTrackPrivateMediaSourceAVFObjC::create(track); info.description = MediaDescriptionAVFObjC::create(track); segment.audioTracks.append(WTFMove(info)); } // FIXME(125161) : Add TextTrack support } m_didParseInitializationDataCallback(WTFMove(segment)); }); } void SourceBufferParserAVFObjC::didFailToParseStreamDataWithError(NSError* error) { m_callOnClientThreadCallback([this, strongThis = Ref { *this }, error = retainPtr(error)] { if (m_didEncounterErrorDuringParsingCallback) m_didEncounterErrorDuringParsingCallback(error.get().code); }); } void SourceBufferParserAVFObjC::didProvideMediaDataForTrackID(uint64_t trackID, CMSampleBufferRef sampleBuffer, const String& mediaType, unsigned flags) { UNUSED_PARAM(flags); m_callOnClientThreadCallback([this, strongThis = Ref { *this }, sampleBuffer = retainPtr(sampleBuffer), trackID, mediaType = mediaType] { if (!m_didProvideMediaDataCallback) return; auto mediaSample = MediaSampleAVFObjC::create(sampleBuffer.get(), trackID); if (mediaSample->isHomogeneous()) { m_didProvideMediaDataCallback(WTFMove(mediaSample), trackID, mediaType); return; } for (auto& sample : mediaSample->divideIntoHomogeneousSamples()) m_didProvideMediaDataCallback(WTFMove(sample), trackID, mediaType); }); } void SourceBufferParserAVFObjC::willProvideContentKeyRequestInitializationDataForTrackID(uint64_t trackID) { m_willProvideContentKeyRequestInitializationDataForTrackIDCallback(trackID); } void SourceBufferParserAVFObjC::didProvideContentKeyRequestInitializationDataForTrackID(NSData* nsInitData, uint64_t trackID) { m_didProvideContentKeyRequestInitializationDataForTrackIDCallback(SharedBuffer::create(nsInitData), trackID); } void SourceBufferParserAVFObjC::didProvideContentKeyRequestSpecifierForTrackID(NSData* nsInitData, uint64_t trackID) { m_callOnClientThreadCallback([this, strongThis = Ref { *this }, nsInitData = retainPtr(nsInitData), trackID] { m_didProvideContentKeyRequestIdentifierForTrackIDCallback(SharedBuffer::create(nsInitData.get()), trackID); }); } } #endif // ENABLE(MEDIA_SOURCE)