From 2ac5311ec5f947f7fd5697442ad7de12d444fc34 Mon Sep 17 00:00:00 2001 From: upstream Date: Fri, 11 Sep 2026 22:56:57 +0200 Subject: [PATCH] Upstream video-dev/hls.js state of the touched files before 3e60a934a2 --- src/remux/passthrough-remuxer.ts | 710 ++++++++++++++++++++++++ tests/unit/remux/passthrough-remuxer.ts | 658 ++++++++++++++++++++++ 2 files changed, 1368 insertions(+) create mode 100644 src/remux/passthrough-remuxer.ts create mode 100644 tests/unit/remux/passthrough-remuxer.ts diff --git a/src/remux/passthrough-remuxer.ts b/src/remux/passthrough-remuxer.ts new file mode 100644 index 0000000..511fede --- /dev/null +++ b/src/remux/passthrough-remuxer.ts @@ -0,0 +1,710 @@ +import { + flushTextTrackMetadataCueSamples, + flushTextTrackUserdataCueSamples, +} from './mp4-remuxer'; +import { ElementaryStreamTypes } from '../loader/fragment'; +import { getCodecCompatibleName } from '../utils/codecs'; +import { type ILogger, Logger } from '../utils/logger'; +import { + getSampleData, + parseInitSegment, + patchEncyptionData, + remuxVideoOnlyIFrameMoof, + videoOnlyInitSegment, +} from '../utils/mp4-tools'; +import type { HlsConfig } from '../config'; +import type { HlsEventEmitter } from '../events'; +import type { DecryptData } from '../loader/level-key'; +import type { + DemuxedAudioTrack, + DemuxedMetadataTrack, + DemuxedUserdataTrack, + PassthroughTrack, +} from '../types/demuxer'; +import type { PlaylistLevelType } from '../types/loader'; +import type { + InitSegmentData, + RemuxedTrack, + Remuxer, + RemuxerResult, +} from '../types/remuxer'; +import type { TrackSet } from '../types/track'; +import type { ChunkMetadata } from '../types/transmuxer'; +import type { TypeSupported } from '../utils/codecs'; +import type { + IFrameDurationRewrite, + InitData, + InitDataTrack, + ParsedSampleData, + TrackTimes, +} from '../utils/mp4-tools'; +import type { TimestampOffset } from '../utils/timescale-conversion'; + +class PassThroughRemuxer extends Logger implements Remuxer { + private readonly observer: HlsEventEmitter; + private emitInitSegment: boolean = false; + private audioCodec?: string; + private videoCodec?: string; + private initData?: InitData; + private initPTS: TimestampOffset | null = null; + private initTracks?: TrackSet; + private lastEndTime: number | null = null; + private isVideoContiguous: boolean = false; + private videoOnlyRemux: boolean = false; + private decryptdata: DecryptData | null = null; + private pendingInitSegment?: Uint8Array; + + constructor( + observer: HlsEventEmitter, + config: HlsConfig, + typeSupported: TypeSupported, + logger: ILogger, + ) { + super('passthrough-remuxer', logger); + this.observer = observer; + } + + public destroy() { + if (this.observer) { + this.observer.removeAllListeners(); + } + // @ts-ignore + this.observer = null; + } + + public resetTimeStamp(defaultInitPTS: TimestampOffset | null) { + this.lastEndTime = null; + const initPTS = this.initPTS; + if (initPTS && defaultInitPTS) { + if ( + initPTS.baseTime === defaultInitPTS.baseTime && + initPTS.timescale === defaultInitPTS.timescale + ) { + return; + } + } + this.initPTS = defaultInitPTS; + } + + public resetNextTimestamp() { + this.isVideoContiguous = false; + this.lastEndTime = null; + } + + public resetInitSegment( + initSegment: Uint8Array | undefined, + audioCodec: string | undefined, + videoCodec: string | undefined, + decryptdata: DecryptData | null, + ) { + this.audioCodec = audioCodec; + this.videoCodec = videoCodec; + this.decryptdata = decryptdata; + this.pendingInitSegment = initSegment; + this.videoOnlyRemux = false; + this.initTracks = undefined; + this.initData = undefined; + this.emitInitSegment = true; + } + + private generateInitSegment( + initSegment: Uint8Array | undefined, + decryptdata?: DecryptData | null, + parsedInitData?: InitData, + ) { + this.videoOnlyRemux = false; + let { audioCodec, videoCodec } = this; + if (!initSegment?.byteLength) { + this.initTracks = undefined; + this.initData = undefined; + return; + } + const { audio, video } = (this.initData = + parsedInitData || parseInitSegment(initSegment)); + + if (decryptdata) { + patchEncyptionData(initSegment, decryptdata); + } else { + const eitherTrack = audio || video; + if (eitherTrack?.encrypted) { + this.warn( + `Init segment with encrypted track with has no key ("${eitherTrack.codec}")!`, + ); + } + } + + // Get codec from initSegment + if (audio) { + audioCodec = getParsedTrackCodec( + audio, + ElementaryStreamTypes.AUDIO, + this, + ); + } + + if (video) { + videoCodec = getParsedTrackCodec( + video, + ElementaryStreamTypes.VIDEO, + this, + ); + } + + const tracks: TrackSet = {}; + if (audio && video) { + tracks.audiovideo = { + container: 'video/mp4', + codec: audioCodec + ',' + videoCodec, + supplemental: video.supplemental, + encrypted: video.encrypted, + initSegment, + id: 'main', + }; + } else if (audio) { + tracks.audio = { + container: 'audio/mp4', + codec: audioCodec, + encrypted: audio.encrypted, + initSegment, + id: 'audio', + }; + } else if (video) { + tracks.video = { + container: 'video/mp4', + codec: videoCodec, + supplemental: video.supplemental, + encrypted: video.encrypted, + initSegment, + id: 'main', + }; + } else { + this.warn('initSegment does not contain moov or trak boxes.'); + } + this.initTracks = tracks; + } + + public remux( + audioTrack: DemuxedAudioTrack, + videoTrack: PassthroughTrack, + id3Track: DemuxedMetadataTrack, + textTrack: DemuxedUserdataTrack, + timeOffset: number, + accurateTimeOffset: boolean, + flush: boolean, + playlistType: PlaylistLevelType, + chunkMeta: ChunkMetadata, + parsedInitData?: InitData, + parsedSampleData?: ParsedSampleData, + ): RemuxerResult { + let { initPTS, lastEndTime } = this; + const result: RemuxerResult = { + audio: undefined, + video: undefined, + text: undefined, + id3: id3Track, + initSegment: undefined, + }; + + // If we haven't yet set a lastEndDTS, or it was reset, set it to the provided timeOffset. We want to use the + // lastEndDTS over timeOffset whenever possible; during progressive playback, the media source will not update + // the media duration (which is what timeOffset is provided as) before we need to process the next chunk. + if (!Number.isFinite(lastEndTime!)) { + lastEndTime = this.lastEndTime = timeOffset || 0; + } + + // The binary segment data is added to the videoTrack in the mp4demuxer. We don't check to see if the data is only + // audio or video (or both); adding it to video was an arbitrary choice. + const data = videoTrack.samples; + if (!data.length) { + return result; + } + + const pendingInitSegment = this.pendingInitSegment; + if (pendingInitSegment) { + this.pendingInitSegment = undefined; + this.generateInitSegment( + pendingInitSegment, + this.decryptdata, + parsedInitData, + ); + } + + const initSegment: InitSegmentData = { + initPTS: undefined, + timescale: undefined, + trackId: undefined, + }; + let initData = this.initData; + if (!initData?.length) { + this.generateInitSegment(data); + initData = this.initData; + } + if (!initData?.length) { + // We can't remux if the initSegment could not be generated + this.warn('Failed to generate initSegment.'); + return result; + } + if ( + __USE_IFRAMES__ && + chunkMeta.iframe && + initData.audio && + initData.video + ) { + // Video-only I-Frame appends must not starve a muxed audio track buffer + const muxedInit = this.initTracks?.audiovideo?.initSegment; + const videoInit = muxedInit ? videoOnlyInitSegment(muxedInit) : null; + if (videoInit) { + this.log( + 'Converted muxed init segment to video-only for I-Frame playback', + ); + this.generateInitSegment(videoInit, this.decryptdata); + if (this.initData?.length) { + initData = this.initData; + this.emitInitSegment = true; + this.videoOnlyRemux = true; + } + } + } + if (this.emitInitSegment) { + initSegment.tracks = this.initTracks; + result.initSegment = initSegment; + this.emitInitSegment = false; + } + + const includeSampleDetails = __USE_IFRAMES__ && chunkMeta.iframe; + const trackSampleData = + parsedSampleData && + (!includeSampleDetails || parsedSampleData.includeSampleDetails) + ? parsedSampleData.tracks + : getSampleData(data, initData, chunkMeta, this, includeSampleDetails); + const audioSampleTimestamps = initData.audio + ? trackSampleData[initData.audio.id] + : null; + const videoSampleTimestamps = initData.video + ? trackSampleData[initData.video.id] + : null; + + const hasAudio = !!initData.audio; + const hasVideo = !!initData.video; + + let type: any = ''; + if (hasAudio) { + type += 'audio'; + } + + if (hasVideo) { + type += 'video'; + } + + const videoStartTime = toStartEndOrDefault(videoSampleTimestamps, Infinity); + const audioStartTime = toStartEndOrDefault(audioSampleTimestamps, Infinity); + const videoEndTime = toStartEndOrDefault(videoSampleTimestamps, 0, true); + const audioEndTime = toStartEndOrDefault(audioSampleTimestamps, 0, true); + + let decodeTime = timeOffset; + let duration = 0; + + if ( + videoSampleTimestamps && + audioSampleTimestamps && + initData.audio && + (audioStartTime > videoEndTime || videoStartTime > audioEndTime) + ) { + this.warn( + `audio and video track sample timestamps do not overlap. v: ${videoStartTime}-${videoEndTime} a: ${audioStartTime}-${audioEndTime}}`, + videoSampleTimestamps, + audioSampleTimestamps, + ); + } + + const syncOnAudio = + !!audioSampleTimestamps && + (!videoSampleTimestamps || + (!initPTS && audioStartTime < videoStartTime) || + (!!initPTS && initPTS.trackId === initData.audio!.id)); + const baseOffsetSamples = syncOnAudio + ? audioSampleTimestamps + : videoSampleTimestamps; + + if (!baseOffsetSamples) { + this.log( + `No media samples found in ${playlistType} ${chunkMeta.level} ${ + chunkMeta.part === -1 ? '' : `part: ${chunkMeta.part} of` + } sn: ${chunkMeta.sn} at playlist time: ${timeOffset}`, + ); + return result; + } + + let data1 = data; + let data2: Uint8Array | undefined; + // Presentation end of stretched I-Frame samples (video timescale units) + let videoPtsEnd: number | undefined; + if ( + __USE_IFRAMES__ && + videoSampleTimestamps && + videoSampleTimestamps.sampleCount >= 1 && + initData.video && + chunkMeta.iframe + ) { + const { trun, start } = videoSampleTimestamps; + const singleRun = trun.length === 1; + const samples = !trun.length + ? [] + : singleRun + ? trun[0].samples + : trun[0].samples.concat(...trun.slice(1).map((run) => run.samples)); + if (samples.length) { + const fragRun = trun[0]; + let lastRun = fragRun; + for (let i = trun.length; i--; ) { + if (trun[i].samples.length) { + lastRun = trun[i]; + break; + } + } + const lastSampleDurationOffset = lastRun.lastSampleDurationOffset; + const { defaultSampleDurationOffset } = fragRun; + // Stretch samples to playlist time when the moof's + // sample timing is off by more than 1/20s. + const needsDurationAdjustment = + Math.abs(chunkMeta.duration - (videoEndTime - videoStartTime)) > 0.05; + const partialMdat = samples.length < videoSampleTimestamps.sampleCount; + const needsRemux = + needsDurationAdjustment || + partialMdat || + this.videoOnlyRemux || + !singleRun; + if (!needsRemux) { + // MP4 timing already spans EXTINF and every declared sample's + // data is in the slice — pass through unchanged. + duration = videoEndTime - videoStartTime; + } else { + const keepEndOffset = partialMdat + ? lastRun.sampleOffset + + lastRun.samples.reduce((sum, sample) => sum + sample.size, 0) + : undefined; + // A partial mdat or dropped runs leave fewer samples than declared, so + // stretch the kept samples to the playlist time whenever a duration + // field can be rewritten (undefined keeps native durations) + const target = needsDurationAdjustment + ? chunkMeta.duration + : videoEndTime - videoStartTime; + const durationRewrite = computeIFrameDurationRewrite( + samples, + target * initData.video.timescale, + lastSampleDurationOffset, + defaultSampleDurationOffset, + ); + const remuxed = remuxVideoOnlyIFrameMoof( + data, + initData.video.id, + samples.length, + keepEndOffset, + durationRewrite, + ); + if (remuxed) { + data1 = remuxed; + videoPtsEnd = samplesPresentationEnd(start, samples); + } else { + // Could not rewrite (e.g. muxed without moof-relative sample + // offsets) — fall back to the native sample duration + this.warn( + `IFrame remux skipped (sn ${chunkMeta.sn}); using native sample duration`, + ); + } + duration = + remuxed && durationRewrite ? target : videoEndTime - videoStartTime; + } + } else { + this.warn( + `Could not remux IFrame track fragment (no samples in range, trun count ${trun.length})`, + ); + duration = videoEndTime - videoStartTime; + } + } else { + duration = syncOnAudio + ? audioEndTime - audioStartTime + : videoEndTime - videoStartTime; + } + + if (__USE_IFRAMES__ && this.videoOnlyRemux && data1 === data) { + // Muxed data that was not remuxed cannot enter the video-only buffer + this.warn( + `Dropping muxed I-Frame fragment that could not be remuxed to video-only (sn: ${chunkMeta.sn})`, + ); + return result; + } + + const timescale = baseOffsetSamples.timescale; + const baseTime = baseOffsetSamples.start - timeOffset * timescale; + const trackId = syncOnAudio ? initData.audio!.id : initData.video!.id; + + decodeTime = baseOffsetSamples.start / timescale; + + if ( + (accurateTimeOffset || !initPTS) && + (isInvalidInitPts(initPTS, decodeTime, timeOffset, duration) || + timescale !== initPTS.timescale) + ) { + let detectedDrift = false; + const trackType = syncOnAudio ? 'audio' : 'video'; + if (initPTS) { + const driftEstimate = + timeOffset !== 0 ? decodeTime / timeOffset : 1 + decodeTime / 1; + detectedDrift = decodeTime >= 0 && Math.abs(1 - driftEstimate) < 0.001; + this.log( + `${trackType} timestamps in track ${trackId} at playlist time: ${accurateTimeOffset ? '' : '~'}${timeOffset} maps to ${decodeTime} with initPTS: ${initPTS.baseTime / initPTS.timescale} (${ + baseTime / timescale - initPTS.baseTime / initPTS.timescale + }s diff) (${type}) drift estimate: ${driftEstimate} ${detectedDrift ? '(ignoring drift)' : 'remapping timestamps (initPTS)'}`, + ); + } + if (!detectedDrift) { + this.log( + `Found initPTS in ${trackType} track ${trackId} at playlist time: ${timeOffset} offset: ${decodeTime - timeOffset} (${baseTime}/${timescale})`, + ); + initPTS = null; + initSegment.initPTS = baseTime; + initSegment.timescale = timescale; + initSegment.trackId = trackId; + } + } + + if (!initPTS) { + if ( + !initSegment.timescale || + initSegment.trackId === undefined || + initSegment.initPTS === undefined + ) { + this.warn('Could not set initPTS'); + initSegment.initPTS = decodeTime; + initSegment.timescale = 1; + initSegment.trackId = -1; + } + this.initPTS = initPTS = { + baseTime: initSegment.initPTS, + timescale: initSegment.timescale, + trackId: initSegment.trackId, + }; + } else { + initSegment.initPTS = initPTS.baseTime; + initSegment.timescale = initPTS.timescale; + initSegment.trackId = initPTS.trackId; + } + + const startDTS = decodeTime - initPTS.baseTime / initPTS.timescale; + const endDTS = startDTS + duration; + const startPTS = + hasVideo && baseOffsetSamples?.ptsMin !== undefined + ? baseOffsetSamples.ptsMin / baseOffsetSamples.timescale - + initPTS.baseTime / initPTS.timescale + : startDTS; + // Report the presentation end of stretched I-Frame samples so fragment + // start/end updates (updateFragPTSDTS) match the appended media + const endPTS = + videoPtsEnd !== undefined && videoSampleTimestamps + ? videoPtsEnd / videoSampleTimestamps.timescale - + initPTS.baseTime / initPTS.timescale + : hasVideo && baseOffsetSamples?.ptsMax + ? baseOffsetSamples.ptsMax / baseOffsetSamples.timescale - + initPTS.baseTime / initPTS.timescale + : endDTS; + + // For troubleshooting duplicates of https://github.com/video-dev/hls.js/issues/6777 + // if (videoSampleTimestamps) { + // console.log( + // `#6777 segment ${chunkMeta.sn}: dts: ${videoSampleTimestamps.start}-${videoSampleTimestamps.start + videoSampleTimestamps.duration} pts: ${videoSampleTimestamps.ptsMin}-${ + // videoSampleTimestamps.ptsMax + // }`, + // ); + // } + + if (duration > 0) { + this.lastEndTime = endDTS; + } else { + this.warn('Duration parsed from mp4 should be greater than zero'); + this.resetNextTimestamp(); + } + + const encrypted = + (initData.audio ? initData.audio.encrypted : false) || + (initData.video ? initData.video.encrypted : false); + + const track: RemuxedTrack = { + data1, + data2, + startPTS, + startDTS, + endPTS, + endDTS, + type, + hasAudio, + hasVideo, + nb: 1, + dropped: 0, + encrypted, + }; + + result.audio = hasAudio && !hasVideo ? track : undefined; + result.video = hasVideo ? track : undefined; + const isVideoContiguous = this.isVideoContiguous; + const videoSampleCount = videoSampleTimestamps?.sampleCount; + if (videoSampleCount) { + const firstKeyFrame = videoSampleTimestamps.keyFrameIndex; + const independent = firstKeyFrame !== -1; + track.nb = videoSampleCount; + track.dropped = + firstKeyFrame === 0 || isVideoContiguous + ? 0 + : independent + ? firstKeyFrame + : videoSampleCount; + track.independent = independent; + track.firstKeyFrame = firstKeyFrame; + if (independent && videoSampleTimestamps.keyFrameStart) { + track.firstKeyFramePTS = + (videoSampleTimestamps.keyFrameStart - initPTS.baseTime) / + initPTS.timescale; + } + if (!isVideoContiguous) { + result.independent = independent; + } + this.isVideoContiguous ||= independent; + if (track.dropped) { + this.warn( + `fmp4 does not start with IDR: firstIDR ${firstKeyFrame}/${videoSampleCount} dropped: ${track.dropped} start: ${track.firstKeyFramePTS || 'NA'}`, + ); + } + } + + result.initSegment = initSegment; + result.id3 = flushTextTrackMetadataCueSamples( + id3Track, + timeOffset, + initPTS, + initPTS, + ); + + if (textTrack.samples.length) { + result.text = flushTextTrackUserdataCueSamples( + textTrack, + timeOffset, + initPTS, + ); + } + + return result; + } +} + +function toStartEndOrDefault( + trackTimes: TrackTimes | null, + defaultValue: number, + end: boolean = false, +): number { + return trackTimes?.start !== undefined + ? (trackTimes.start + (end ? trackTimes.duration : 0)) / + trackTimes.timescale + : defaultValue; +} + +function samplesPresentationEnd( + baseDecodeTime: number, + samples: Array<{ cts?: number; duration: number }>, +): number { + let dts = baseDecodeTime; + let end = baseDecodeTime; + for (let i = 0; i < samples.length; i++) { + const { cts, duration } = samples[i]; + end = Math.max(end, dts + (cts || 0) + duration); + dts += duration; + } + return end; +} + +// Stretch the kept I-Frame samples to `totalDuration`, mutating them so the +// reported presentation end matches. Returns undefined when no duration field +// can be written (durations come from moov trex defaults). +function computeIFrameDurationRewrite( + samples: Array<{ duration: number }>, + totalDuration: number, + lastSampleDurationOffset: number | undefined, + defaultSampleDurationOffset: number | undefined, +): IFrameDurationRewrite | undefined { + if (lastSampleDurationOffset !== undefined) { + const lastSample = samples[samples.length - 1]; + let lastSampleDuration = totalDuration; + for (let i = 0; i < samples.length - 1; i++) { + lastSampleDuration -= samples[i].duration; + } + if (lastSampleDuration <= 0) { + // Keep the native duration rather than write a wrapped value + lastSampleDuration = lastSample.duration; + } + lastSample.duration = lastSampleDuration; + return { + durationOffset: lastSampleDurationOffset, + value: lastSampleDuration, + }; + } + if (defaultSampleDurationOffset === undefined) { + return undefined; + } + const defaultSampleDuration = Math.round(totalDuration / samples.length); + for (let i = 0; i < samples.length; i++) { + samples[i].duration = defaultSampleDuration; + } + return { + durationOffset: defaultSampleDurationOffset, + value: defaultSampleDuration, + }; +} + +function isInvalidInitPts( + initPTS: TimestampOffset | null, + startDTS: number, + timeOffset: number, + duration: number, +): initPTS is null { + if (initPTS === null) { + return true; + } + // InitPTS is invalid when distance from program would be more than or equal to segment duration or a minimum of one second + const minDuration = Math.max(duration, 1); + const startTime = startDTS - initPTS.baseTime / initPTS.timescale; + return Math.abs(startTime - timeOffset) >= minDuration; +} + +function getParsedTrackCodec( + track: InitDataTrack, + type: ElementaryStreamTypes.AUDIO | ElementaryStreamTypes.VIDEO, + logger: ILogger, +): string { + const parsedCodec = track.codec; + if (parsedCodec && parsedCodec.length > 4) { + return parsedCodec; + } + if (type === ElementaryStreamTypes.AUDIO) { + if ( + parsedCodec === 'ec-3' || + parsedCodec === 'ac-3' || + parsedCodec === 'alac' + ) { + return parsedCodec; + } + if (parsedCodec === 'fLaC' || parsedCodec === 'Opus') { + // Opting not to get `preferManagedMediaSource` from player config for isSupported() check for simplicity + const preferManagedMediaSource = false; + return getCodecCompatibleName(parsedCodec, preferManagedMediaSource); + } + + logger.warn(`Unhandled audio codec "${parsedCodec}" in mp4 MAP`); + return parsedCodec || 'mp4a'; + } else if (parsedCodec === 'mjpg') { + return parsedCodec; + } + // Provide defaults based on codec type + // This allows for some playback of some fmp4 playlists without CODECS defined in manifest + logger.warn(`Unhandled video codec "${parsedCodec}" in mp4 MAP`); + return parsedCodec || 'avc1'; +} +export default PassThroughRemuxer; diff --git a/tests/unit/remux/passthrough-remuxer.ts b/tests/unit/remux/passthrough-remuxer.ts new file mode 100644 index 0000000..7847266 --- /dev/null +++ b/tests/unit/remux/passthrough-remuxer.ts @@ -0,0 +1,658 @@ +import { expect } from 'chai'; +import EventEmitter from 'eventemitter3'; +import { hlsDefaultConfig } from '../../../src/config'; +import MP4 from '../../../src/remux/mp4-generator'; +import PassThroughRemuxer from '../../../src/remux/passthrough-remuxer'; +import { PlaylistLevelType } from '../../../src/types/loader'; +import { ChunkMetadata } from '../../../src/types/transmuxer'; +import { logger } from '../../../src/utils/logger'; +import { + appendUint8Array, + findBox, + parseInitSegment, + types, + writeUint32, +} from '../../../src/utils/mp4-tools'; +import type { HlsEventEmitter } from '../../../src/events'; +import type { TrackFragmentSample } from '../../../src/remux/mp4-generator'; +import type { + DemuxedAudioTrack, + DemuxedMetadataTrack, + DemuxedUserdataTrack, + PassthroughTrack, +} from '../../../src/types/demuxer'; +import type { TypeSupported } from '../../../src/utils/codecs'; + +describe('passthrough-remuxer', function () { + let remuxer: PassThroughRemuxer; + + beforeEach(function () { + const observer: HlsEventEmitter = new EventEmitter() as HlsEventEmitter; + const typeSupported: TypeSupported = { + ac3: true, + mpeg: true, + mp3: true, + }; + remuxer = new PassThroughRemuxer( + observer, + { ...hlsDefaultConfig }, + typeSupported, + logger, + ); + }); + + afterEach(function () { + remuxer.destroy(); + }); + + function remuxIFrameFragment( + fragmentData: Uint8Array, + extinfDuration: number, + encrypted = false, + ) { + const initSegment = MP4.initSegment([videoInitTrack()]); + if (encrypted) { + markVideoInitSegmentEncrypted(initSegment); + } + remuxer.resetInitSegment(initSegment, undefined, 'avc1.42001e', null); + + return remuxer.remux( + audioTrack(), + passthroughTrack(fragmentData), + metadataTrack(), + userdataTrack(), + 0, + true, + true, + PlaylistLevelType.MAIN, + new ChunkMetadata( + 0, + 0, + 0, + fragmentData.byteLength, + -1, + false, + extinfDuration, + true, + ), + ); + } + + function remuxIFrame( + fragmentData: Uint8Array, + extinfDuration: number, + ) { + return remuxer.remux( + audioTrack(), + passthroughTrack(fragmentData), + metadataTrack(), + userdataTrack(), + 0, + true, + true, + PlaylistLevelType.MAIN, + new ChunkMetadata( + 0, + 0, + 0, + fragmentData.byteLength, + -1, + false, + extinfDuration, + true, + ), + ); + } + + it('remuxes moof+mdat to stretch a single-keyframe iframe to the EXTINF duration', function () { + const extinfDuration = 4; + const fragmentData = mp4Fragment([sample(3003, 4, 0)]); + + const result = remuxIFrameFragment(fragmentData, extinfDuration); + + expect(result.video, 'video track').to.exist; + // Rewritten in place: the moof+mdat stays in data1, no separate mdat + expect(result.video!.data1, 'data1').to.equal(fragmentData); + expect(result.video!.data2, 'data2').to.not.exist; + // The single sample is stretched to the EXTINF duration + expect(trunSampleDurations(result.video!.data1)).to.deep.equal([ + extinfDuration * 90000, + ]); + expect(result.video!.endDTS - result.video!.startDTS).to.equal( + extinfDuration, + ); + // The reported PTS range must cover the stretched sample so that + // fragment start/end updates match the appended media. + expect(result.video!.endPTS - result.video!.startPTS).to.equal( + extinfDuration, + ); + expect(result.video!.nb).to.equal(1); + }); + + it('does not remux a byte-range iframe segment whose sample duration already matches EXTINF', function () { + // Byte-range addressed iframe (single sample spanning the whole playback segment) + const extinfDuration = 4; + const fragmentData = mp4Fragment([sample(extinfDuration * 90000, 4, 0)]); + + const result = remuxIFrameFragment(fragmentData, extinfDuration); + + expect(result.video, 'video track').to.exist; + expect(result.video!.data2, 'data2').to.not.exist; + expect(result.video!.data1).to.equal(fragmentData); + }); + + it('remuxes a multi-sample iframe fragment in place, keeping every sample', function () { + const extinfDuration = 4; + const fragmentData = mp4Fragment([ + sample(3003, 4, 0), + sample(3003, 4, 0), + sample(3003, 4, 0), + ]); + + const result = remuxIFrameFragment(fragmentData, extinfDuration); + + expect(result.video, 'video track').to.exist; + expect(result.video!.data1, 'data1').to.equal(fragmentData); + expect(result.video!.data2, 'data2').to.not.exist; + // all three parsed samples are reported, not collapsed to one + expect(result.video!.nb).to.equal(3); + expect(result.video!.endDTS - result.video!.startDTS).to.equal( + extinfDuration, + ); + expect(result.video!.endPTS - result.video!.startPTS).to.equal( + extinfDuration, + ); + // Each sample keeps its own duration; only the last is stretched to + // balance EXTINF. + expect(trunSampleDurations(result.video!.data1)).to.deep.equal([ + 3003, + 3003, + extinfDuration * 90000 - 6006, + ]); + }); + + it('reports the presentation end of stretched samples with composition offsets', function () { + const extinfDuration = 4; + const fragmentData = mp4Fragment([ + sample(3003, 4, 9000), + sample(3003, 4, 0), + ]); + + const result = remuxIFrameFragment(fragmentData, extinfDuration); + + expect(result.video, 'video track').to.exist; + expect(result.video!.startPTS).to.equal(3003 / 90000); + expect(result.video!.endPTS).to.equal(extinfDuration); + }); + + it('keeps native sample durations when EXTINF is shorter than the preceding samples', function () { + // Stretching would make the last sample duration negative (writing an + // unsigned wrap-around value into the trun). + const extinfDuration = 1; + const fragmentData = mp4Fragment([ + sample(45045, 4, 0), + sample(45045, 4, 0), + sample(45045, 4, 0), + ]); + + const result = remuxIFrameFragment(fragmentData, extinfDuration); + + expect(result.video, 'video track').to.exist; + expect(trunSampleDurations(result.video!.data1)).to.deep.equal([ + 45045, 45045, 45045, + ]); + expect(result.video!.endPTS - result.video!.startPTS).to.equal( + (3 * 45045) / 90000, + ); + }); + + it('rewrites the sample duration of encrypted iframe fragments in place', function () { + const extinfDuration = 4; + const fragmentData = mp4Fragment([sample(3003, 4, 9000)]); + + const result = remuxIFrameFragment(fragmentData, extinfDuration, true); + + expect(result.video, 'video track').to.exist; + // The original moof is retained so encryption boxes survive intact + expect(result.video!.data1).to.equal(fragmentData); + expect(result.video!.data2, 'data2').to.not.exist; + expect(result.video!.encrypted).to.equal(true); + expect(trunSampleDurations(fragmentData)).to.deep.equal([ + extinfDuration * 90000, + ]); + expect(result.video!.endPTS - result.video!.startPTS).to.be.closeTo( + extinfDuration, + 1 / 90000, + ); + }); + + it('converts muxed input to video-only for iframe fragments', function () { + const extinfDuration = 4; + const initSegment = MP4.initSegment([ + videoInitTrack(), + audioInitTrack(), + ] as any); + remuxer.resetInitSegment(initSegment, 'mp4a.40.2', 'avc1.42001e', null); + // Video sample duration matches EXTINF: the muxed moof must still be + // regenerated so the audio track fragment is not appended + const fragmentData = muxedMp4Fragment( + [sample(extinfDuration * 90000, 24, 0)], + [50, 60], + ); + + const result = remuxer.remux( + audioTrack(), + passthroughTrack(fragmentData), + metadataTrack(), + userdataTrack(), + 0, + true, + true, + PlaylistLevelType.MAIN, + new ChunkMetadata( + 0, + 0, + 0, + fragmentData.byteLength, + -1, + false, + extinfDuration, + true, + ), + ); + + // The emitted init segment is converted to video-only + const videoInitTrackInfo = result.initSegment?.tracks?.video; + expect(videoInitTrackInfo, 'video init track').to.exist; + expect(result.initSegment?.tracks?.audiovideo, 'audiovideo init track').to + .not.exist; + const parsedInit = parseInitSegment(videoInitTrackInfo!.initSegment!); + expect(parsedInit.video, 'filtered init video').to.exist; + expect(parsedInit.audio, 'filtered init audio').to.not.exist; + + // Video-only output: the audio traf is spliced out (audio sample bytes are + // left in the mdat as unreferenced data) + expect(result.video, 'video track').to.exist; + expect(result.video!.type).to.equal('video'); + expect(result.video!.hasAudio).to.equal(false); + expect(result.video!.data1, 'data1').to.not.equal(fragmentData); + expect(result.video!.data2, 'data2').to.not.exist; + const trafs = findBox(result.video!.data1, ['moof', 'traf']); + expect(trafs, 'trafs').to.have.lengthOf(1); + const tfhd = findBox(trafs[0], ['tfhd'])[0]; + expect(readUint32(tfhd, 4), 'video track id').to.equal(1); + expect(result.video!.endPTS - result.video!.startPTS).to.equal( + extinfDuration, + ); + }); + + it('remuxes muxed iframe fragments with the audio run first in the mdat', function () { + const extinfDuration = 4; + const initSegment = MP4.initSegment([ + videoInitTrack(), + audioInitTrack(), + ] as any); + remuxer.resetInitSegment(initSegment, 'mp4a.40.2', 'avc1.42001e', null); + const fragmentData = muxedMp4Fragment( + [sample(3003, 24, 0)], + [50, 60], + true, + ); + + const result = remuxIFrame(fragmentData, extinfDuration); + + expect(result.video, 'video track').to.exist; + expect(result.video!.type).to.equal('video'); + expect(result.video!.data2, 'data2').to.not.exist; + const trafs = findBox(result.video!.data1, ['moof', 'traf']); + expect(trafs, 'trafs').to.have.lengthOf(1); + expect(readUint32(findBox(trafs[0], ['tfhd'])[0], 4), 'video id').to.equal( + 1, + ); + expect(result.video!.endPTS - result.video!.startPTS).to.equal( + extinfDuration, + ); + }); + + it('remuxes iframe fragments with multiple video track fragment runs', function () { + const extinfDuration = 4; + const initSegment = MP4.initSegment([ + videoInitTrack(), + audioInitTrack(), + ] as any); + remuxer.resetInitSegment(initSegment, 'mp4a.40.2', 'avc1.42001e', null); + const fragmentData = withSecondVideoTraf( + muxedMp4Fragment([sample(3003, 24, 0)], [50]), + ); + + const result = remuxIFrame(fragmentData, extinfDuration); + + expect(result.video, 'video track').to.exist; + expect(result.video!.data2, 'data2').to.not.exist; + // The audio traf is spliced out; both video trafs are retained + const trafs = findBox(result.video!.data1, ['moof', 'traf']); + expect(trafs, 'video trafs').to.have.lengthOf(2); + trafs.forEach((traf) => { + expect(readUint32(findBox(traf, ['tfhd'])[0], 4), 'video id').to.equal(1); + }); + expect(result.video!.endPTS - result.video!.startPTS).to.equal( + extinfDuration, + ); + }); + + it('drops muxed iframe fragments with no video sample in range', function () { + const extinfDuration = 4; + const initSegment = MP4.initSegment([ + videoInitTrack(), + audioInitTrack(), + ] as any); + remuxer.resetInitSegment(initSegment, 'mp4a.40.2', 'avc1.42001e', null); + // Audio-first layout sliced before the video sample: nothing can be + // appended to the video-only SourceBuffer + const fragmentData = muxedMp4Fragment([sample(3003, 24, 0)], [50], true); + const moofSize = findBox(fragmentData, ['moof'])[0].byteLength + 8; + const partialFragment = fragmentData.subarray(0, moofSize + 8 + 30); + + const result = remuxIFrame(partialFragment, extinfDuration); + + expect(result.initSegment?.tracks?.video, 'video init track').to.exist; + expect(result.video, 'video track').to.equal(undefined); + }); + + it('truncates encrypted partial-mdat iframe fragments on the mdat box boundary', function () { + const extinfDuration = 4; + const samples = [sample(3003, 10, 0), sample(3003, 20, 0)]; + const fragmentData = mp4Fragment(samples); + // Slice mid-way through the second sample's data, as a byte-range + // addressed I-Frame request would. + const mdatPayloadOffset = fragmentData.byteLength - 30; + const partialFragment = fragmentData.subarray(0, mdatPayloadOffset + 13); + + const result = remuxIFrameFragment(partialFragment, extinfDuration, true); + + expect(result.video, 'video track').to.exist; + expect(result.video!.encrypted).to.equal(true); + // Appended data ends at the rewritten mdat boundary, dropping the + // partially loaded sample's bytes. + expect(result.video!.data1.byteLength).to.equal(mdatPayloadOffset + 10); + expect(readUint32(result.video!.data1, mdatPayloadOffset - 8)).to.equal(18); + const trun = findBox(result.video!.data1, ['moof', 'traf', 'trun'])[0]; + expect(readUint32(trun, 4), 'trun sample_count').to.equal(1); + expect(trunSampleDurations(result.video!.data1)).to.deep.equal([ + extinfDuration * 90000, + ]); + expect(result.video!.endPTS - result.video!.startPTS).to.equal( + extinfDuration, + ); + }); + + it('stretches a partial-mdat iframe whose declared duration already matches EXTINF', function () { + // The moof declares two samples that together already span EXTINF, so + // needsDurationAdjustment is false — but only the first sample's data is + // present in the byte-range slice (as with a muxed I-Frame byte range). + const extinfDuration = 4; + const fragmentData = mp4Fragment([ + sample(extinfDuration * 90000 - 3003, 10, 0), + sample(3003, 20, 0), + ]); + const mdatPayloadOffset = fragmentData.byteLength - 30; + const partialFragment = fragmentData.subarray(0, mdatPayloadOffset + 13); + + const result = remuxIFrameFragment(partialFragment, extinfDuration); + + expect(result.video, 'video track').to.exist; + // The single kept keyframe is stretched to the full declared duration even + // though the moof timing did not disagree with EXTINF + expect(trunSampleDurations(result.video!.data1)).to.deep.equal([ + extinfDuration * 90000, + ]); + expect(result.video!.endPTS - result.video!.startPTS).to.equal( + extinfDuration, + ); + }); +}); + +function markVideoInitSegmentEncrypted( + initSegment: Uint8Array, +): void { + const stsd = findBox(initSegment, [ + 'moov', + 'trak', + 'mdia', + 'minf', + 'stbl', + 'stsd', + ])[0]; + const sampleEntry = findBox(stsd.subarray(8), ['avc1'])[0]; + const typeOffset = sampleEntry.byteOffset - initSegment.byteOffset - 4; + initSegment.set([0x65, 0x6e, 0x63, 0x76], typeOffset); // 'encv' +} + +function readUint32(buffer: Uint8Array, offset: number): number { + return ( + ((buffer[offset] << 24) | + (buffer[offset + 1] << 16) | + (buffer[offset + 2] << 8) | + buffer[offset + 3]) >>> + 0 + ); +} + +function trunSampleDurations(data: Uint8Array): number[] { + const trun = findBox(data, ['moof', 'traf', 'trun'])[0]; + const sampleCount = readUint32(trun, 4); + const durations: number[] = []; + // full box header (4) + sample_count (4) + data_offset (4); + // entries are duration, size, flags, cts (16 bytes each) + for (let i = 0; i < sampleCount; i++) { + durations.push(readUint32(trun, 12 + i * 16)); + } + return durations; +} + +function videoInitTrack(): any { + return { + codec: 'avc1.42001e', + dropped: 0, + duration: 4, + id: 1, + inputTimeScale: 90000, + len: 0, + nbNalu: 0, + pid: -1, + pixelRatio: [1, 1], + pps: [new Uint8Array([0x68, 0xce, 0x06, 0xe2])], + samples: [], + segmentCodec: 'avc', + sequenceNumber: 0, + sps: [new Uint8Array([0x67, 0x42, 0x00, 0x1e, 0xab, 0x40])], + timescale: 90000, + type: 'video', + width: 16, + height: 16, + }; +} + +function audioInitTrack(): any { + return { + codec: 'mp4a.40.2', + segmentCodec: 'aac', + channelCount: 2, + samplerate: 48000, + config: [0x11, 0x90], + duration: 4, + id: 2, + timescale: 48000, + type: 'audio', + }; +} + +// moof with a video traf (track 1) and an audio traf (track 2) sharing one +// mdat: video run first, audio run after (ffmpeg muxed fMP4 layout) +function muxedMp4Fragment( + videoSamples: TrackFragmentSample[], + audioSampleSizes: number[], + audioFirst = false, +): Uint8Array { + const videoBytes = videoSamples.reduce((total, s) => total + s.size, 0); + const audioBytes = audioSampleSizes.reduce((total, s) => total + s, 0); + const videoMoof = MP4.moof(0, 0, { + type: 'video', + id: 1, + samples: videoSamples, + }); + + const tfhd = new Uint8Array(8); + tfhd.set([0x00, 0x02, 0x00, 0x00]); // version 0, default-base-is-moof + writeUint32(tfhd, 4, 2); // track_ID + const tfdt = new Uint8Array(8); + const audioTrun = new Uint8Array(12 + audioSampleSizes.length * 8); + audioTrun.set([0x00, 0x00, 0x03, 0x01]); // data-offset + duration + size + writeUint32(audioTrun, 4, audioSampleSizes.length); + audioSampleSizes.forEach((size, i) => { + writeUint32(audioTrun, 12 + i * 8, 1024); + writeUint32(audioTrun, 12 + i * 8 + 4, size); + }); + const audioTraf = MP4.box( + types.traf, + MP4.box(types.tfhd, tfhd), + MP4.box(types.tfdt, tfdt), + MP4.box(types.trun, audioTrun), + ); + + // Splice the audio traf into the moof, then patch the moof size and both + // trun data offsets for the combined mdat layout + const moof = new Uint8Array(videoMoof.byteLength + audioTraf.byteLength); + moof.set(videoMoof, 0); + moof.set(audioTraf, videoMoof.byteLength); + writeUint32(moof, 0, moof.byteLength); + const truns = findBox(moof, ['moof', 'traf', 'trun']); + writeUint32( + moof, + truns[0].byteOffset - moof.byteOffset + 8, + moof.byteLength + 8 + (audioFirst ? audioBytes : 0), + ); + writeUint32( + moof, + truns[1].byteOffset - moof.byteOffset + 8, + moof.byteLength + 8 + (audioFirst ? 0 : videoBytes), + ); + // CMAF muxed content addresses samples from the moof; the video traf must + // set default-base-is-moof so the remuxer can splice out the audio traf + const videoTfhd = findBox(moof, ['moof', 'traf', 'tfhd'])[0]; + videoTfhd[1] = 0x02; + + return appendUint8Array( + moof, + MP4.mdat(new Uint8Array(videoBytes + audioBytes)), + ) as Uint8Array; +} + +// Appends a second (empty) video track fragment to the moof so the video +// track has more than one trun +function withSecondVideoTraf( + fragment: Uint8Array, +): Uint8Array { + const tfhd = new Uint8Array(8); + tfhd.set([0x00, 0x02, 0x00, 0x00]); + writeUint32(tfhd, 4, 1); // video track_ID + const trun = new Uint8Array(12); + trun[2] = 0x01; // data-offset present + const traf = MP4.box( + types.traf, + MP4.box(types.tfhd, tfhd), + MP4.box(types.trun, trun), + ); + const moofSize = findBox(fragment, ['moof'])[0].byteLength + 8; + const result = new Uint8Array(fragment.byteLength + traf.byteLength); + result.set(fragment.subarray(0, moofSize), 0); + result.set(traf, moofSize); + result.set(fragment.subarray(moofSize), moofSize + traf.byteLength); + writeUint32(result, 0, moofSize + traf.byteLength); + return result; +} + +function mp4Fragment(samples: TrackFragmentSample[]): Uint8Array { + const mdatPayload = new Uint8Array( + samples.reduce((total, sample) => total + sample.size, 0), + ); + return appendUint8Array( + MP4.moof(0, 0, { type: 'video', id: 1, samples }), + MP4.mdat(mdatPayload), + ); +} + +function sample( + duration: number, + size: number, + cts: number, +): TrackFragmentSample { + return { + cts, + duration, + flags: { + degradPrio: 0, + dependsOn: 2, + hasRedundancy: 0, + isDependedOn: 0, + isLeading: 0, + isNonSync: 0, + paddingValue: 0, + }, + size, + }; +} + +function passthroughTrack(samples: Uint8Array): PassthroughTrack { + return { + codec: 'avc1.42001e', + dropped: 0, + duration: 4, + id: 1, + inputTimeScale: 90000, + pid: -1, + sampleDuration: 3003, + samples, + sequenceNumber: 0, + supplemental: undefined, + timescale: 90000, + type: 'video', + }; +} + +function audioTrack(): DemuxedAudioTrack { + return { + dropped: 0, + id: 2, + inputTimeScale: 90000, + pid: -1, + samples: [], + segmentCodec: 'aac', + sequenceNumber: 0, + type: 'audio', + }; +} + +function metadataTrack(): DemuxedMetadataTrack { + return { + dropped: 0, + id: 3, + inputTimeScale: 90000, + pid: -1, + samples: [], + sequenceNumber: 0, + type: 'id3', + }; +} + +function userdataTrack(): DemuxedUserdataTrack { + return { + dropped: 0, + id: 4, + inputTimeScale: 90000, + pid: -1, + samples: [], + sequenceNumber: 0, + type: 'text', + }; +}