Upstream video-dev/hls.js state of the touched files before 3e60a934a2
This commit is contained in:
upstream committed
commit
2ac5311ec5
2 files changed
+1368
No files matched your search
@@ -0,0 +1,710 @@
|
|||||||
|
import {
|
||||||
|
flushTextTrackMetadataCueSamples,
|
||||||
|
flushTextTrackUserdataCueSamples,
|
||||||
|
} from './mp4-remuxer';
|
||||||
|
import { ElementaryStreamTypes } from '../loader/fragment';
|
||||||
|
import { getCodecCompatibleName } from '../utils/codecs';
|
||||||
|
import { type ILogger, Logger } from '../utils/logger';
|
||||||
|
import {
|
||||||
|
getSampleData,
|
||||||
|
parseInitSegment,
|
||||||
|
patchEncyptionData,
|
||||||
|
remuxVideoOnlyIFrameMoof,
|
||||||
|
videoOnlyInitSegment,
|
||||||
|
} from '../utils/mp4-tools';
|
||||||
|
import type { HlsConfig } from '../config';
|
||||||
|
import type { HlsEventEmitter } from '../events';
|
||||||
|
import type { DecryptData } from '../loader/level-key';
|
||||||
|
import type {
|
||||||
|
DemuxedAudioTrack,
|
||||||
|
DemuxedMetadataTrack,
|
||||||
|
DemuxedUserdataTrack,
|
||||||
|
PassthroughTrack,
|
||||||
|
} from '../types/demuxer';
|
||||||
|
import type { PlaylistLevelType } from '../types/loader';
|
||||||
|
import type {
|
||||||
|
InitSegmentData,
|
||||||
|
RemuxedTrack,
|
||||||
|
Remuxer,
|
||||||
|
RemuxerResult,
|
||||||
|
} from '../types/remuxer';
|
||||||
|
import type { TrackSet } from '../types/track';
|
||||||
|
import type { ChunkMetadata } from '../types/transmuxer';
|
||||||
|
import type { TypeSupported } from '../utils/codecs';
|
||||||
|
import type {
|
||||||
|
IFrameDurationRewrite,
|
||||||
|
InitData,
|
||||||
|
InitDataTrack,
|
||||||
|
ParsedSampleData,
|
||||||
|
TrackTimes,
|
||||||
|
} from '../utils/mp4-tools';
|
||||||
|
import type { TimestampOffset } from '../utils/timescale-conversion';
|
||||||
|
|
||||||
|
class PassThroughRemuxer extends Logger implements Remuxer {
|
||||||
|
private readonly observer: HlsEventEmitter;
|
||||||
|
private emitInitSegment: boolean = false;
|
||||||
|
private audioCodec?: string;
|
||||||
|
private videoCodec?: string;
|
||||||
|
private initData?: InitData;
|
||||||
|
private initPTS: TimestampOffset | null = null;
|
||||||
|
private initTracks?: TrackSet;
|
||||||
|
private lastEndTime: number | null = null;
|
||||||
|
private isVideoContiguous: boolean = false;
|
||||||
|
private videoOnlyRemux: boolean = false;
|
||||||
|
private decryptdata: DecryptData | null = null;
|
||||||
|
private pendingInitSegment?: Uint8Array<ArrayBuffer>;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
observer: HlsEventEmitter,
|
||||||
|
config: HlsConfig,
|
||||||
|
typeSupported: TypeSupported,
|
||||||
|
logger: ILogger,
|
||||||
|
) {
|
||||||
|
super('passthrough-remuxer', logger);
|
||||||
|
this.observer = observer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public destroy() {
|
||||||
|
if (this.observer) {
|
||||||
|
this.observer.removeAllListeners();
|
||||||
|
}
|
||||||
|
// @ts-ignore
|
||||||
|
this.observer = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public resetTimeStamp(defaultInitPTS: TimestampOffset | null) {
|
||||||
|
this.lastEndTime = null;
|
||||||
|
const initPTS = this.initPTS;
|
||||||
|
if (initPTS && defaultInitPTS) {
|
||||||
|
if (
|
||||||
|
initPTS.baseTime === defaultInitPTS.baseTime &&
|
||||||
|
initPTS.timescale === defaultInitPTS.timescale
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.initPTS = defaultInitPTS;
|
||||||
|
}
|
||||||
|
|
||||||
|
public resetNextTimestamp() {
|
||||||
|
this.isVideoContiguous = false;
|
||||||
|
this.lastEndTime = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public resetInitSegment(
|
||||||
|
initSegment: Uint8Array<ArrayBuffer> | undefined,
|
||||||
|
audioCodec: string | undefined,
|
||||||
|
videoCodec: string | undefined,
|
||||||
|
decryptdata: DecryptData | null,
|
||||||
|
) {
|
||||||
|
this.audioCodec = audioCodec;
|
||||||
|
this.videoCodec = videoCodec;
|
||||||
|
this.decryptdata = decryptdata;
|
||||||
|
this.pendingInitSegment = initSegment;
|
||||||
|
this.videoOnlyRemux = false;
|
||||||
|
this.initTracks = undefined;
|
||||||
|
this.initData = undefined;
|
||||||
|
this.emitInitSegment = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private generateInitSegment(
|
||||||
|
initSegment: Uint8Array<ArrayBuffer> | undefined,
|
||||||
|
decryptdata?: DecryptData | null,
|
||||||
|
parsedInitData?: InitData,
|
||||||
|
) {
|
||||||
|
this.videoOnlyRemux = false;
|
||||||
|
let { audioCodec, videoCodec } = this;
|
||||||
|
if (!initSegment?.byteLength) {
|
||||||
|
this.initTracks = undefined;
|
||||||
|
this.initData = undefined;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const { audio, video } = (this.initData =
|
||||||
|
parsedInitData || parseInitSegment(initSegment));
|
||||||
|
|
||||||
|
if (decryptdata) {
|
||||||
|
patchEncyptionData(initSegment, decryptdata);
|
||||||
|
} else {
|
||||||
|
const eitherTrack = audio || video;
|
||||||
|
if (eitherTrack?.encrypted) {
|
||||||
|
this.warn(
|
||||||
|
`Init segment with encrypted track with has no key ("${eitherTrack.codec}")!`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get codec from initSegment
|
||||||
|
if (audio) {
|
||||||
|
audioCodec = getParsedTrackCodec(
|
||||||
|
audio,
|
||||||
|
ElementaryStreamTypes.AUDIO,
|
||||||
|
this,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (video) {
|
||||||
|
videoCodec = getParsedTrackCodec(
|
||||||
|
video,
|
||||||
|
ElementaryStreamTypes.VIDEO,
|
||||||
|
this,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const tracks: TrackSet = {};
|
||||||
|
if (audio && video) {
|
||||||
|
tracks.audiovideo = {
|
||||||
|
container: 'video/mp4',
|
||||||
|
codec: audioCodec + ',' + videoCodec,
|
||||||
|
supplemental: video.supplemental,
|
||||||
|
encrypted: video.encrypted,
|
||||||
|
initSegment,
|
||||||
|
id: 'main',
|
||||||
|
};
|
||||||
|
} else if (audio) {
|
||||||
|
tracks.audio = {
|
||||||
|
container: 'audio/mp4',
|
||||||
|
codec: audioCodec,
|
||||||
|
encrypted: audio.encrypted,
|
||||||
|
initSegment,
|
||||||
|
id: 'audio',
|
||||||
|
};
|
||||||
|
} else if (video) {
|
||||||
|
tracks.video = {
|
||||||
|
container: 'video/mp4',
|
||||||
|
codec: videoCodec,
|
||||||
|
supplemental: video.supplemental,
|
||||||
|
encrypted: video.encrypted,
|
||||||
|
initSegment,
|
||||||
|
id: 'main',
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
this.warn('initSegment does not contain moov or trak boxes.');
|
||||||
|
}
|
||||||
|
this.initTracks = tracks;
|
||||||
|
}
|
||||||
|
|
||||||
|
public remux(
|
||||||
|
audioTrack: DemuxedAudioTrack,
|
||||||
|
videoTrack: PassthroughTrack,
|
||||||
|
id3Track: DemuxedMetadataTrack,
|
||||||
|
textTrack: DemuxedUserdataTrack,
|
||||||
|
timeOffset: number,
|
||||||
|
accurateTimeOffset: boolean,
|
||||||
|
flush: boolean,
|
||||||
|
playlistType: PlaylistLevelType,
|
||||||
|
chunkMeta: ChunkMetadata,
|
||||||
|
parsedInitData?: InitData,
|
||||||
|
parsedSampleData?: ParsedSampleData,
|
||||||
|
): RemuxerResult {
|
||||||
|
let { initPTS, lastEndTime } = this;
|
||||||
|
const result: RemuxerResult = {
|
||||||
|
audio: undefined,
|
||||||
|
video: undefined,
|
||||||
|
text: undefined,
|
||||||
|
id3: id3Track,
|
||||||
|
initSegment: undefined,
|
||||||
|
};
|
||||||
|
|
||||||
|
// If we haven't yet set a lastEndDTS, or it was reset, set it to the provided timeOffset. We want to use the
|
||||||
|
// lastEndDTS over timeOffset whenever possible; during progressive playback, the media source will not update
|
||||||
|
// the media duration (which is what timeOffset is provided as) before we need to process the next chunk.
|
||||||
|
if (!Number.isFinite(lastEndTime!)) {
|
||||||
|
lastEndTime = this.lastEndTime = timeOffset || 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The binary segment data is added to the videoTrack in the mp4demuxer. We don't check to see if the data is only
|
||||||
|
// audio or video (or both); adding it to video was an arbitrary choice.
|
||||||
|
const data = videoTrack.samples;
|
||||||
|
if (!data.length) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
const pendingInitSegment = this.pendingInitSegment;
|
||||||
|
if (pendingInitSegment) {
|
||||||
|
this.pendingInitSegment = undefined;
|
||||||
|
this.generateInitSegment(
|
||||||
|
pendingInitSegment,
|
||||||
|
this.decryptdata,
|
||||||
|
parsedInitData,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const initSegment: InitSegmentData = {
|
||||||
|
initPTS: undefined,
|
||||||
|
timescale: undefined,
|
||||||
|
trackId: undefined,
|
||||||
|
};
|
||||||
|
let initData = this.initData;
|
||||||
|
if (!initData?.length) {
|
||||||
|
this.generateInitSegment(data);
|
||||||
|
initData = this.initData;
|
||||||
|
}
|
||||||
|
if (!initData?.length) {
|
||||||
|
// We can't remux if the initSegment could not be generated
|
||||||
|
this.warn('Failed to generate initSegment.');
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
__USE_IFRAMES__ &&
|
||||||
|
chunkMeta.iframe &&
|
||||||
|
initData.audio &&
|
||||||
|
initData.video
|
||||||
|
) {
|
||||||
|
// Video-only I-Frame appends must not starve a muxed audio track buffer
|
||||||
|
const muxedInit = this.initTracks?.audiovideo?.initSegment;
|
||||||
|
const videoInit = muxedInit ? videoOnlyInitSegment(muxedInit) : null;
|
||||||
|
if (videoInit) {
|
||||||
|
this.log(
|
||||||
|
'Converted muxed init segment to video-only for I-Frame playback',
|
||||||
|
);
|
||||||
|
this.generateInitSegment(videoInit, this.decryptdata);
|
||||||
|
if (this.initData?.length) {
|
||||||
|
initData = this.initData;
|
||||||
|
this.emitInitSegment = true;
|
||||||
|
this.videoOnlyRemux = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (this.emitInitSegment) {
|
||||||
|
initSegment.tracks = this.initTracks;
|
||||||
|
result.initSegment = initSegment;
|
||||||
|
this.emitInitSegment = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const includeSampleDetails = __USE_IFRAMES__ && chunkMeta.iframe;
|
||||||
|
const trackSampleData =
|
||||||
|
parsedSampleData &&
|
||||||
|
(!includeSampleDetails || parsedSampleData.includeSampleDetails)
|
||||||
|
? parsedSampleData.tracks
|
||||||
|
: getSampleData(data, initData, chunkMeta, this, includeSampleDetails);
|
||||||
|
const audioSampleTimestamps = initData.audio
|
||||||
|
? trackSampleData[initData.audio.id]
|
||||||
|
: null;
|
||||||
|
const videoSampleTimestamps = initData.video
|
||||||
|
? trackSampleData[initData.video.id]
|
||||||
|
: null;
|
||||||
|
|
||||||
|
const hasAudio = !!initData.audio;
|
||||||
|
const hasVideo = !!initData.video;
|
||||||
|
|
||||||
|
let type: any = '';
|
||||||
|
if (hasAudio) {
|
||||||
|
type += 'audio';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasVideo) {
|
||||||
|
type += 'video';
|
||||||
|
}
|
||||||
|
|
||||||
|
const videoStartTime = toStartEndOrDefault(videoSampleTimestamps, Infinity);
|
||||||
|
const audioStartTime = toStartEndOrDefault(audioSampleTimestamps, Infinity);
|
||||||
|
const videoEndTime = toStartEndOrDefault(videoSampleTimestamps, 0, true);
|
||||||
|
const audioEndTime = toStartEndOrDefault(audioSampleTimestamps, 0, true);
|
||||||
|
|
||||||
|
let decodeTime = timeOffset;
|
||||||
|
let duration = 0;
|
||||||
|
|
||||||
|
if (
|
||||||
|
videoSampleTimestamps &&
|
||||||
|
audioSampleTimestamps &&
|
||||||
|
initData.audio &&
|
||||||
|
(audioStartTime > videoEndTime || videoStartTime > audioEndTime)
|
||||||
|
) {
|
||||||
|
this.warn(
|
||||||
|
`audio and video track sample timestamps do not overlap. v: ${videoStartTime}-${videoEndTime} a: ${audioStartTime}-${audioEndTime}}`,
|
||||||
|
videoSampleTimestamps,
|
||||||
|
audioSampleTimestamps,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const syncOnAudio =
|
||||||
|
!!audioSampleTimestamps &&
|
||||||
|
(!videoSampleTimestamps ||
|
||||||
|
(!initPTS && audioStartTime < videoStartTime) ||
|
||||||
|
(!!initPTS && initPTS.trackId === initData.audio!.id));
|
||||||
|
const baseOffsetSamples = syncOnAudio
|
||||||
|
? audioSampleTimestamps
|
||||||
|
: videoSampleTimestamps;
|
||||||
|
|
||||||
|
if (!baseOffsetSamples) {
|
||||||
|
this.log(
|
||||||
|
`No media samples found in ${playlistType} ${chunkMeta.level} ${
|
||||||
|
chunkMeta.part === -1 ? '' : `part: ${chunkMeta.part} of`
|
||||||
|
} sn: ${chunkMeta.sn} at playlist time: ${timeOffset}`,
|
||||||
|
);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
let data1 = data;
|
||||||
|
let data2: Uint8Array<ArrayBuffer> | undefined;
|
||||||
|
// Presentation end of stretched I-Frame samples (video timescale units)
|
||||||
|
let videoPtsEnd: number | undefined;
|
||||||
|
if (
|
||||||
|
__USE_IFRAMES__ &&
|
||||||
|
videoSampleTimestamps &&
|
||||||
|
videoSampleTimestamps.sampleCount >= 1 &&
|
||||||
|
initData.video &&
|
||||||
|
chunkMeta.iframe
|
||||||
|
) {
|
||||||
|
const { trun, start } = videoSampleTimestamps;
|
||||||
|
const singleRun = trun.length === 1;
|
||||||
|
const samples = !trun.length
|
||||||
|
? []
|
||||||
|
: singleRun
|
||||||
|
? trun[0].samples
|
||||||
|
: trun[0].samples.concat(...trun.slice(1).map((run) => run.samples));
|
||||||
|
if (samples.length) {
|
||||||
|
const fragRun = trun[0];
|
||||||
|
let lastRun = fragRun;
|
||||||
|
for (let i = trun.length; i--; ) {
|
||||||
|
if (trun[i].samples.length) {
|
||||||
|
lastRun = trun[i];
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const lastSampleDurationOffset = lastRun.lastSampleDurationOffset;
|
||||||
|
const { defaultSampleDurationOffset } = fragRun;
|
||||||
|
// Stretch samples to playlist time when the moof's
|
||||||
|
// sample timing is off by more than 1/20s.
|
||||||
|
const needsDurationAdjustment =
|
||||||
|
Math.abs(chunkMeta.duration - (videoEndTime - videoStartTime)) > 0.05;
|
||||||
|
const partialMdat = samples.length < videoSampleTimestamps.sampleCount;
|
||||||
|
const needsRemux =
|
||||||
|
needsDurationAdjustment ||
|
||||||
|
partialMdat ||
|
||||||
|
this.videoOnlyRemux ||
|
||||||
|
!singleRun;
|
||||||
|
if (!needsRemux) {
|
||||||
|
// MP4 timing already spans EXTINF and every declared sample's
|
||||||
|
// data is in the slice — pass through unchanged.
|
||||||
|
duration = videoEndTime - videoStartTime;
|
||||||
|
} else {
|
||||||
|
const keepEndOffset = partialMdat
|
||||||
|
? lastRun.sampleOffset +
|
||||||
|
lastRun.samples.reduce((sum, sample) => sum + sample.size, 0)
|
||||||
|
: undefined;
|
||||||
|
// A partial mdat or dropped runs leave fewer samples than declared, so
|
||||||
|
// stretch the kept samples to the playlist time whenever a duration
|
||||||
|
// field can be rewritten (undefined keeps native durations)
|
||||||
|
const target = needsDurationAdjustment
|
||||||
|
? chunkMeta.duration
|
||||||
|
: videoEndTime - videoStartTime;
|
||||||
|
const durationRewrite = computeIFrameDurationRewrite(
|
||||||
|
samples,
|
||||||
|
target * initData.video.timescale,
|
||||||
|
lastSampleDurationOffset,
|
||||||
|
defaultSampleDurationOffset,
|
||||||
|
);
|
||||||
|
const remuxed = remuxVideoOnlyIFrameMoof(
|
||||||
|
data,
|
||||||
|
initData.video.id,
|
||||||
|
samples.length,
|
||||||
|
keepEndOffset,
|
||||||
|
durationRewrite,
|
||||||
|
);
|
||||||
|
if (remuxed) {
|
||||||
|
data1 = remuxed;
|
||||||
|
videoPtsEnd = samplesPresentationEnd(start, samples);
|
||||||
|
} else {
|
||||||
|
// Could not rewrite (e.g. muxed without moof-relative sample
|
||||||
|
// offsets) — fall back to the native sample duration
|
||||||
|
this.warn(
|
||||||
|
`IFrame remux skipped (sn ${chunkMeta.sn}); using native sample duration`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
duration =
|
||||||
|
remuxed && durationRewrite ? target : videoEndTime - videoStartTime;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.warn(
|
||||||
|
`Could not remux IFrame track fragment (no samples in range, trun count ${trun.length})`,
|
||||||
|
);
|
||||||
|
duration = videoEndTime - videoStartTime;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
duration = syncOnAudio
|
||||||
|
? audioEndTime - audioStartTime
|
||||||
|
: videoEndTime - videoStartTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (__USE_IFRAMES__ && this.videoOnlyRemux && data1 === data) {
|
||||||
|
// Muxed data that was not remuxed cannot enter the video-only buffer
|
||||||
|
this.warn(
|
||||||
|
`Dropping muxed I-Frame fragment that could not be remuxed to video-only (sn: ${chunkMeta.sn})`,
|
||||||
|
);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
const timescale = baseOffsetSamples.timescale;
|
||||||
|
const baseTime = baseOffsetSamples.start - timeOffset * timescale;
|
||||||
|
const trackId = syncOnAudio ? initData.audio!.id : initData.video!.id;
|
||||||
|
|
||||||
|
decodeTime = baseOffsetSamples.start / timescale;
|
||||||
|
|
||||||
|
if (
|
||||||
|
(accurateTimeOffset || !initPTS) &&
|
||||||
|
(isInvalidInitPts(initPTS, decodeTime, timeOffset, duration) ||
|
||||||
|
timescale !== initPTS.timescale)
|
||||||
|
) {
|
||||||
|
let detectedDrift = false;
|
||||||
|
const trackType = syncOnAudio ? 'audio' : 'video';
|
||||||
|
if (initPTS) {
|
||||||
|
const driftEstimate =
|
||||||
|
timeOffset !== 0 ? decodeTime / timeOffset : 1 + decodeTime / 1;
|
||||||
|
detectedDrift = decodeTime >= 0 && Math.abs(1 - driftEstimate) < 0.001;
|
||||||
|
this.log(
|
||||||
|
`${trackType} timestamps in track ${trackId} at playlist time: ${accurateTimeOffset ? '' : '~'}${timeOffset} maps to ${decodeTime} with initPTS: ${initPTS.baseTime / initPTS.timescale} (${
|
||||||
|
baseTime / timescale - initPTS.baseTime / initPTS.timescale
|
||||||
|
}s diff) (${type}) drift estimate: ${driftEstimate} ${detectedDrift ? '(ignoring drift)' : 'remapping timestamps (initPTS)'}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!detectedDrift) {
|
||||||
|
this.log(
|
||||||
|
`Found initPTS in ${trackType} track ${trackId} at playlist time: ${timeOffset} offset: ${decodeTime - timeOffset} (${baseTime}/${timescale})`,
|
||||||
|
);
|
||||||
|
initPTS = null;
|
||||||
|
initSegment.initPTS = baseTime;
|
||||||
|
initSegment.timescale = timescale;
|
||||||
|
initSegment.trackId = trackId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!initPTS) {
|
||||||
|
if (
|
||||||
|
!initSegment.timescale ||
|
||||||
|
initSegment.trackId === undefined ||
|
||||||
|
initSegment.initPTS === undefined
|
||||||
|
) {
|
||||||
|
this.warn('Could not set initPTS');
|
||||||
|
initSegment.initPTS = decodeTime;
|
||||||
|
initSegment.timescale = 1;
|
||||||
|
initSegment.trackId = -1;
|
||||||
|
}
|
||||||
|
this.initPTS = initPTS = {
|
||||||
|
baseTime: initSegment.initPTS,
|
||||||
|
timescale: initSegment.timescale,
|
||||||
|
trackId: initSegment.trackId,
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
initSegment.initPTS = initPTS.baseTime;
|
||||||
|
initSegment.timescale = initPTS.timescale;
|
||||||
|
initSegment.trackId = initPTS.trackId;
|
||||||
|
}
|
||||||
|
|
||||||
|
const startDTS = decodeTime - initPTS.baseTime / initPTS.timescale;
|
||||||
|
const endDTS = startDTS + duration;
|
||||||
|
const startPTS =
|
||||||
|
hasVideo && baseOffsetSamples?.ptsMin !== undefined
|
||||||
|
? baseOffsetSamples.ptsMin / baseOffsetSamples.timescale -
|
||||||
|
initPTS.baseTime / initPTS.timescale
|
||||||
|
: startDTS;
|
||||||
|
// Report the presentation end of stretched I-Frame samples so fragment
|
||||||
|
// start/end updates (updateFragPTSDTS) match the appended media
|
||||||
|
const endPTS =
|
||||||
|
videoPtsEnd !== undefined && videoSampleTimestamps
|
||||||
|
? videoPtsEnd / videoSampleTimestamps.timescale -
|
||||||
|
initPTS.baseTime / initPTS.timescale
|
||||||
|
: hasVideo && baseOffsetSamples?.ptsMax
|
||||||
|
? baseOffsetSamples.ptsMax / baseOffsetSamples.timescale -
|
||||||
|
initPTS.baseTime / initPTS.timescale
|
||||||
|
: endDTS;
|
||||||
|
|
||||||
|
// For troubleshooting duplicates of https://github.com/video-dev/hls.js/issues/6777
|
||||||
|
// if (videoSampleTimestamps) {
|
||||||
|
// console.log(
|
||||||
|
// `#6777 segment ${chunkMeta.sn}: dts: ${videoSampleTimestamps.start}-${videoSampleTimestamps.start + videoSampleTimestamps.duration} pts: ${videoSampleTimestamps.ptsMin}-${
|
||||||
|
// videoSampleTimestamps.ptsMax
|
||||||
|
// }`,
|
||||||
|
// );
|
||||||
|
// }
|
||||||
|
|
||||||
|
if (duration > 0) {
|
||||||
|
this.lastEndTime = endDTS;
|
||||||
|
} else {
|
||||||
|
this.warn('Duration parsed from mp4 should be greater than zero');
|
||||||
|
this.resetNextTimestamp();
|
||||||
|
}
|
||||||
|
|
||||||
|
const encrypted =
|
||||||
|
(initData.audio ? initData.audio.encrypted : false) ||
|
||||||
|
(initData.video ? initData.video.encrypted : false);
|
||||||
|
|
||||||
|
const track: RemuxedTrack = {
|
||||||
|
data1,
|
||||||
|
data2,
|
||||||
|
startPTS,
|
||||||
|
startDTS,
|
||||||
|
endPTS,
|
||||||
|
endDTS,
|
||||||
|
type,
|
||||||
|
hasAudio,
|
||||||
|
hasVideo,
|
||||||
|
nb: 1,
|
||||||
|
dropped: 0,
|
||||||
|
encrypted,
|
||||||
|
};
|
||||||
|
|
||||||
|
result.audio = hasAudio && !hasVideo ? track : undefined;
|
||||||
|
result.video = hasVideo ? track : undefined;
|
||||||
|
const isVideoContiguous = this.isVideoContiguous;
|
||||||
|
const videoSampleCount = videoSampleTimestamps?.sampleCount;
|
||||||
|
if (videoSampleCount) {
|
||||||
|
const firstKeyFrame = videoSampleTimestamps.keyFrameIndex;
|
||||||
|
const independent = firstKeyFrame !== -1;
|
||||||
|
track.nb = videoSampleCount;
|
||||||
|
track.dropped =
|
||||||
|
firstKeyFrame === 0 || isVideoContiguous
|
||||||
|
? 0
|
||||||
|
: independent
|
||||||
|
? firstKeyFrame
|
||||||
|
: videoSampleCount;
|
||||||
|
track.independent = independent;
|
||||||
|
track.firstKeyFrame = firstKeyFrame;
|
||||||
|
if (independent && videoSampleTimestamps.keyFrameStart) {
|
||||||
|
track.firstKeyFramePTS =
|
||||||
|
(videoSampleTimestamps.keyFrameStart - initPTS.baseTime) /
|
||||||
|
initPTS.timescale;
|
||||||
|
}
|
||||||
|
if (!isVideoContiguous) {
|
||||||
|
result.independent = independent;
|
||||||
|
}
|
||||||
|
this.isVideoContiguous ||= independent;
|
||||||
|
if (track.dropped) {
|
||||||
|
this.warn(
|
||||||
|
`fmp4 does not start with IDR: firstIDR ${firstKeyFrame}/${videoSampleCount} dropped: ${track.dropped} start: ${track.firstKeyFramePTS || 'NA'}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result.initSegment = initSegment;
|
||||||
|
result.id3 = flushTextTrackMetadataCueSamples(
|
||||||
|
id3Track,
|
||||||
|
timeOffset,
|
||||||
|
initPTS,
|
||||||
|
initPTS,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (textTrack.samples.length) {
|
||||||
|
result.text = flushTextTrackUserdataCueSamples(
|
||||||
|
textTrack,
|
||||||
|
timeOffset,
|
||||||
|
initPTS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function toStartEndOrDefault(
|
||||||
|
trackTimes: TrackTimes | null,
|
||||||
|
defaultValue: number,
|
||||||
|
end: boolean = false,
|
||||||
|
): number {
|
||||||
|
return trackTimes?.start !== undefined
|
||||||
|
? (trackTimes.start + (end ? trackTimes.duration : 0)) /
|
||||||
|
trackTimes.timescale
|
||||||
|
: defaultValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
function samplesPresentationEnd(
|
||||||
|
baseDecodeTime: number,
|
||||||
|
samples: Array<{ cts?: number; duration: number }>,
|
||||||
|
): number {
|
||||||
|
let dts = baseDecodeTime;
|
||||||
|
let end = baseDecodeTime;
|
||||||
|
for (let i = 0; i < samples.length; i++) {
|
||||||
|
const { cts, duration } = samples[i];
|
||||||
|
end = Math.max(end, dts + (cts || 0) + duration);
|
||||||
|
dts += duration;
|
||||||
|
}
|
||||||
|
return end;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stretch the kept I-Frame samples to `totalDuration`, mutating them so the
|
||||||
|
// reported presentation end matches. Returns undefined when no duration field
|
||||||
|
// can be written (durations come from moov trex defaults).
|
||||||
|
function computeIFrameDurationRewrite(
|
||||||
|
samples: Array<{ duration: number }>,
|
||||||
|
totalDuration: number,
|
||||||
|
lastSampleDurationOffset: number | undefined,
|
||||||
|
defaultSampleDurationOffset: number | undefined,
|
||||||
|
): IFrameDurationRewrite | undefined {
|
||||||
|
if (lastSampleDurationOffset !== undefined) {
|
||||||
|
const lastSample = samples[samples.length - 1];
|
||||||
|
let lastSampleDuration = totalDuration;
|
||||||
|
for (let i = 0; i < samples.length - 1; i++) {
|
||||||
|
lastSampleDuration -= samples[i].duration;
|
||||||
|
}
|
||||||
|
if (lastSampleDuration <= 0) {
|
||||||
|
// Keep the native duration rather than write a wrapped value
|
||||||
|
lastSampleDuration = lastSample.duration;
|
||||||
|
}
|
||||||
|
lastSample.duration = lastSampleDuration;
|
||||||
|
return {
|
||||||
|
durationOffset: lastSampleDurationOffset,
|
||||||
|
value: lastSampleDuration,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (defaultSampleDurationOffset === undefined) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
const defaultSampleDuration = Math.round(totalDuration / samples.length);
|
||||||
|
for (let i = 0; i < samples.length; i++) {
|
||||||
|
samples[i].duration = defaultSampleDuration;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
durationOffset: defaultSampleDurationOffset,
|
||||||
|
value: defaultSampleDuration,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isInvalidInitPts(
|
||||||
|
initPTS: TimestampOffset | null,
|
||||||
|
startDTS: number,
|
||||||
|
timeOffset: number,
|
||||||
|
duration: number,
|
||||||
|
): initPTS is null {
|
||||||
|
if (initPTS === null) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// InitPTS is invalid when distance from program would be more than or equal to segment duration or a minimum of one second
|
||||||
|
const minDuration = Math.max(duration, 1);
|
||||||
|
const startTime = startDTS - initPTS.baseTime / initPTS.timescale;
|
||||||
|
return Math.abs(startTime - timeOffset) >= minDuration;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getParsedTrackCodec(
|
||||||
|
track: InitDataTrack,
|
||||||
|
type: ElementaryStreamTypes.AUDIO | ElementaryStreamTypes.VIDEO,
|
||||||
|
logger: ILogger,
|
||||||
|
): string {
|
||||||
|
const parsedCodec = track.codec;
|
||||||
|
if (parsedCodec && parsedCodec.length > 4) {
|
||||||
|
return parsedCodec;
|
||||||
|
}
|
||||||
|
if (type === ElementaryStreamTypes.AUDIO) {
|
||||||
|
if (
|
||||||
|
parsedCodec === 'ec-3' ||
|
||||||
|
parsedCodec === 'ac-3' ||
|
||||||
|
parsedCodec === 'alac'
|
||||||
|
) {
|
||||||
|
return parsedCodec;
|
||||||
|
}
|
||||||
|
if (parsedCodec === 'fLaC' || parsedCodec === 'Opus') {
|
||||||
|
// Opting not to get `preferManagedMediaSource` from player config for isSupported() check for simplicity
|
||||||
|
const preferManagedMediaSource = false;
|
||||||
|
return getCodecCompatibleName(parsedCodec, preferManagedMediaSource);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.warn(`Unhandled audio codec "${parsedCodec}" in mp4 MAP`);
|
||||||
|
return parsedCodec || 'mp4a';
|
||||||
|
} else if (parsedCodec === 'mjpg') {
|
||||||
|
return parsedCodec;
|
||||||
|
}
|
||||||
|
// Provide defaults based on codec type
|
||||||
|
// This allows for some playback of some fmp4 playlists without CODECS defined in manifest
|
||||||
|
logger.warn(`Unhandled video codec "${parsedCodec}" in mp4 MAP`);
|
||||||
|
return parsedCodec || 'avc1';
|
||||||
|
}
|
||||||
|
export default PassThroughRemuxer;
|
||||||
@@ -0,0 +1,658 @@
|
|||||||
|
import { expect } from 'chai';
|
||||||
|
import EventEmitter from 'eventemitter3';
|
||||||
|
import { hlsDefaultConfig } from '../../../src/config';
|
||||||
|
import MP4 from '../../../src/remux/mp4-generator';
|
||||||
|
import PassThroughRemuxer from '../../../src/remux/passthrough-remuxer';
|
||||||
|
import { PlaylistLevelType } from '../../../src/types/loader';
|
||||||
|
import { ChunkMetadata } from '../../../src/types/transmuxer';
|
||||||
|
import { logger } from '../../../src/utils/logger';
|
||||||
|
import {
|
||||||
|
appendUint8Array,
|
||||||
|
findBox,
|
||||||
|
parseInitSegment,
|
||||||
|
types,
|
||||||
|
writeUint32,
|
||||||
|
} from '../../../src/utils/mp4-tools';
|
||||||
|
import type { HlsEventEmitter } from '../../../src/events';
|
||||||
|
import type { TrackFragmentSample } from '../../../src/remux/mp4-generator';
|
||||||
|
import type {
|
||||||
|
DemuxedAudioTrack,
|
||||||
|
DemuxedMetadataTrack,
|
||||||
|
DemuxedUserdataTrack,
|
||||||
|
PassthroughTrack,
|
||||||
|
} from '../../../src/types/demuxer';
|
||||||
|
import type { TypeSupported } from '../../../src/utils/codecs';
|
||||||
|
|
||||||
|
describe('passthrough-remuxer', function () {
|
||||||
|
let remuxer: PassThroughRemuxer;
|
||||||
|
|
||||||
|
beforeEach(function () {
|
||||||
|
const observer: HlsEventEmitter = new EventEmitter() as HlsEventEmitter;
|
||||||
|
const typeSupported: TypeSupported = {
|
||||||
|
ac3: true,
|
||||||
|
mpeg: true,
|
||||||
|
mp3: true,
|
||||||
|
};
|
||||||
|
remuxer = new PassThroughRemuxer(
|
||||||
|
observer,
|
||||||
|
{ ...hlsDefaultConfig },
|
||||||
|
typeSupported,
|
||||||
|
logger,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(function () {
|
||||||
|
remuxer.destroy();
|
||||||
|
});
|
||||||
|
|
||||||
|
function remuxIFrameFragment(
|
||||||
|
fragmentData: Uint8Array<ArrayBuffer>,
|
||||||
|
extinfDuration: number,
|
||||||
|
encrypted = false,
|
||||||
|
) {
|
||||||
|
const initSegment = MP4.initSegment([videoInitTrack()]);
|
||||||
|
if (encrypted) {
|
||||||
|
markVideoInitSegmentEncrypted(initSegment);
|
||||||
|
}
|
||||||
|
remuxer.resetInitSegment(initSegment, undefined, 'avc1.42001e', null);
|
||||||
|
|
||||||
|
return remuxer.remux(
|
||||||
|
audioTrack(),
|
||||||
|
passthroughTrack(fragmentData),
|
||||||
|
metadataTrack(),
|
||||||
|
userdataTrack(),
|
||||||
|
0,
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
PlaylistLevelType.MAIN,
|
||||||
|
new ChunkMetadata(
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
fragmentData.byteLength,
|
||||||
|
-1,
|
||||||
|
false,
|
||||||
|
extinfDuration,
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function remuxIFrame(
|
||||||
|
fragmentData: Uint8Array<ArrayBuffer>,
|
||||||
|
extinfDuration: number,
|
||||||
|
) {
|
||||||
|
return remuxer.remux(
|
||||||
|
audioTrack(),
|
||||||
|
passthroughTrack(fragmentData),
|
||||||
|
metadataTrack(),
|
||||||
|
userdataTrack(),
|
||||||
|
0,
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
PlaylistLevelType.MAIN,
|
||||||
|
new ChunkMetadata(
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
fragmentData.byteLength,
|
||||||
|
-1,
|
||||||
|
false,
|
||||||
|
extinfDuration,
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
it('remuxes moof+mdat to stretch a single-keyframe iframe to the EXTINF duration', function () {
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const fragmentData = mp4Fragment([sample(3003, 4, 0)]);
|
||||||
|
|
||||||
|
const result = remuxIFrameFragment(fragmentData, extinfDuration);
|
||||||
|
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
// Rewritten in place: the moof+mdat stays in data1, no separate mdat
|
||||||
|
expect(result.video!.data1, 'data1').to.equal(fragmentData);
|
||||||
|
expect(result.video!.data2, 'data2').to.not.exist;
|
||||||
|
// The single sample is stretched to the EXTINF duration
|
||||||
|
expect(trunSampleDurations(result.video!.data1)).to.deep.equal([
|
||||||
|
extinfDuration * 90000,
|
||||||
|
]);
|
||||||
|
expect(result.video!.endDTS - result.video!.startDTS).to.equal(
|
||||||
|
extinfDuration,
|
||||||
|
);
|
||||||
|
// The reported PTS range must cover the stretched sample so that
|
||||||
|
// fragment start/end updates match the appended media.
|
||||||
|
expect(result.video!.endPTS - result.video!.startPTS).to.equal(
|
||||||
|
extinfDuration,
|
||||||
|
);
|
||||||
|
expect(result.video!.nb).to.equal(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not remux a byte-range iframe segment whose sample duration already matches EXTINF', function () {
|
||||||
|
// Byte-range addressed iframe (single sample spanning the whole playback segment)
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const fragmentData = mp4Fragment([sample(extinfDuration * 90000, 4, 0)]);
|
||||||
|
|
||||||
|
const result = remuxIFrameFragment(fragmentData, extinfDuration);
|
||||||
|
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
expect(result.video!.data2, 'data2').to.not.exist;
|
||||||
|
expect(result.video!.data1).to.equal(fragmentData);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('remuxes a multi-sample iframe fragment in place, keeping every sample', function () {
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const fragmentData = mp4Fragment([
|
||||||
|
sample(3003, 4, 0),
|
||||||
|
sample(3003, 4, 0),
|
||||||
|
sample(3003, 4, 0),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const result = remuxIFrameFragment(fragmentData, extinfDuration);
|
||||||
|
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
expect(result.video!.data1, 'data1').to.equal(fragmentData);
|
||||||
|
expect(result.video!.data2, 'data2').to.not.exist;
|
||||||
|
// all three parsed samples are reported, not collapsed to one
|
||||||
|
expect(result.video!.nb).to.equal(3);
|
||||||
|
expect(result.video!.endDTS - result.video!.startDTS).to.equal(
|
||||||
|
extinfDuration,
|
||||||
|
);
|
||||||
|
expect(result.video!.endPTS - result.video!.startPTS).to.equal(
|
||||||
|
extinfDuration,
|
||||||
|
);
|
||||||
|
// Each sample keeps its own duration; only the last is stretched to
|
||||||
|
// balance EXTINF.
|
||||||
|
expect(trunSampleDurations(result.video!.data1)).to.deep.equal([
|
||||||
|
3003,
|
||||||
|
3003,
|
||||||
|
extinfDuration * 90000 - 6006,
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reports the presentation end of stretched samples with composition offsets', function () {
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const fragmentData = mp4Fragment([
|
||||||
|
sample(3003, 4, 9000),
|
||||||
|
sample(3003, 4, 0),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const result = remuxIFrameFragment(fragmentData, extinfDuration);
|
||||||
|
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
expect(result.video!.startPTS).to.equal(3003 / 90000);
|
||||||
|
expect(result.video!.endPTS).to.equal(extinfDuration);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps native sample durations when EXTINF is shorter than the preceding samples', function () {
|
||||||
|
// Stretching would make the last sample duration negative (writing an
|
||||||
|
// unsigned wrap-around value into the trun).
|
||||||
|
const extinfDuration = 1;
|
||||||
|
const fragmentData = mp4Fragment([
|
||||||
|
sample(45045, 4, 0),
|
||||||
|
sample(45045, 4, 0),
|
||||||
|
sample(45045, 4, 0),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const result = remuxIFrameFragment(fragmentData, extinfDuration);
|
||||||
|
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
expect(trunSampleDurations(result.video!.data1)).to.deep.equal([
|
||||||
|
45045, 45045, 45045,
|
||||||
|
]);
|
||||||
|
expect(result.video!.endPTS - result.video!.startPTS).to.equal(
|
||||||
|
(3 * 45045) / 90000,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rewrites the sample duration of encrypted iframe fragments in place', function () {
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const fragmentData = mp4Fragment([sample(3003, 4, 9000)]);
|
||||||
|
|
||||||
|
const result = remuxIFrameFragment(fragmentData, extinfDuration, true);
|
||||||
|
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
// The original moof is retained so encryption boxes survive intact
|
||||||
|
expect(result.video!.data1).to.equal(fragmentData);
|
||||||
|
expect(result.video!.data2, 'data2').to.not.exist;
|
||||||
|
expect(result.video!.encrypted).to.equal(true);
|
||||||
|
expect(trunSampleDurations(fragmentData)).to.deep.equal([
|
||||||
|
extinfDuration * 90000,
|
||||||
|
]);
|
||||||
|
expect(result.video!.endPTS - result.video!.startPTS).to.be.closeTo(
|
||||||
|
extinfDuration,
|
||||||
|
1 / 90000,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('converts muxed input to video-only for iframe fragments', function () {
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const initSegment = MP4.initSegment([
|
||||||
|
videoInitTrack(),
|
||||||
|
audioInitTrack(),
|
||||||
|
] as any);
|
||||||
|
remuxer.resetInitSegment(initSegment, 'mp4a.40.2', 'avc1.42001e', null);
|
||||||
|
// Video sample duration matches EXTINF: the muxed moof must still be
|
||||||
|
// regenerated so the audio track fragment is not appended
|
||||||
|
const fragmentData = muxedMp4Fragment(
|
||||||
|
[sample(extinfDuration * 90000, 24, 0)],
|
||||||
|
[50, 60],
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = remuxer.remux(
|
||||||
|
audioTrack(),
|
||||||
|
passthroughTrack(fragmentData),
|
||||||
|
metadataTrack(),
|
||||||
|
userdataTrack(),
|
||||||
|
0,
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
PlaylistLevelType.MAIN,
|
||||||
|
new ChunkMetadata(
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
fragmentData.byteLength,
|
||||||
|
-1,
|
||||||
|
false,
|
||||||
|
extinfDuration,
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
// The emitted init segment is converted to video-only
|
||||||
|
const videoInitTrackInfo = result.initSegment?.tracks?.video;
|
||||||
|
expect(videoInitTrackInfo, 'video init track').to.exist;
|
||||||
|
expect(result.initSegment?.tracks?.audiovideo, 'audiovideo init track').to
|
||||||
|
.not.exist;
|
||||||
|
const parsedInit = parseInitSegment(videoInitTrackInfo!.initSegment!);
|
||||||
|
expect(parsedInit.video, 'filtered init video').to.exist;
|
||||||
|
expect(parsedInit.audio, 'filtered init audio').to.not.exist;
|
||||||
|
|
||||||
|
// Video-only output: the audio traf is spliced out (audio sample bytes are
|
||||||
|
// left in the mdat as unreferenced data)
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
expect(result.video!.type).to.equal('video');
|
||||||
|
expect(result.video!.hasAudio).to.equal(false);
|
||||||
|
expect(result.video!.data1, 'data1').to.not.equal(fragmentData);
|
||||||
|
expect(result.video!.data2, 'data2').to.not.exist;
|
||||||
|
const trafs = findBox(result.video!.data1, ['moof', 'traf']);
|
||||||
|
expect(trafs, 'trafs').to.have.lengthOf(1);
|
||||||
|
const tfhd = findBox(trafs[0], ['tfhd'])[0];
|
||||||
|
expect(readUint32(tfhd, 4), 'video track id').to.equal(1);
|
||||||
|
expect(result.video!.endPTS - result.video!.startPTS).to.equal(
|
||||||
|
extinfDuration,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('remuxes muxed iframe fragments with the audio run first in the mdat', function () {
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const initSegment = MP4.initSegment([
|
||||||
|
videoInitTrack(),
|
||||||
|
audioInitTrack(),
|
||||||
|
] as any);
|
||||||
|
remuxer.resetInitSegment(initSegment, 'mp4a.40.2', 'avc1.42001e', null);
|
||||||
|
const fragmentData = muxedMp4Fragment(
|
||||||
|
[sample(3003, 24, 0)],
|
||||||
|
[50, 60],
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = remuxIFrame(fragmentData, extinfDuration);
|
||||||
|
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
expect(result.video!.type).to.equal('video');
|
||||||
|
expect(result.video!.data2, 'data2').to.not.exist;
|
||||||
|
const trafs = findBox(result.video!.data1, ['moof', 'traf']);
|
||||||
|
expect(trafs, 'trafs').to.have.lengthOf(1);
|
||||||
|
expect(readUint32(findBox(trafs[0], ['tfhd'])[0], 4), 'video id').to.equal(
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
expect(result.video!.endPTS - result.video!.startPTS).to.equal(
|
||||||
|
extinfDuration,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('remuxes iframe fragments with multiple video track fragment runs', function () {
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const initSegment = MP4.initSegment([
|
||||||
|
videoInitTrack(),
|
||||||
|
audioInitTrack(),
|
||||||
|
] as any);
|
||||||
|
remuxer.resetInitSegment(initSegment, 'mp4a.40.2', 'avc1.42001e', null);
|
||||||
|
const fragmentData = withSecondVideoTraf(
|
||||||
|
muxedMp4Fragment([sample(3003, 24, 0)], [50]),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = remuxIFrame(fragmentData, extinfDuration);
|
||||||
|
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
expect(result.video!.data2, 'data2').to.not.exist;
|
||||||
|
// The audio traf is spliced out; both video trafs are retained
|
||||||
|
const trafs = findBox(result.video!.data1, ['moof', 'traf']);
|
||||||
|
expect(trafs, 'video trafs').to.have.lengthOf(2);
|
||||||
|
trafs.forEach((traf) => {
|
||||||
|
expect(readUint32(findBox(traf, ['tfhd'])[0], 4), 'video id').to.equal(1);
|
||||||
|
});
|
||||||
|
expect(result.video!.endPTS - result.video!.startPTS).to.equal(
|
||||||
|
extinfDuration,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('drops muxed iframe fragments with no video sample in range', function () {
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const initSegment = MP4.initSegment([
|
||||||
|
videoInitTrack(),
|
||||||
|
audioInitTrack(),
|
||||||
|
] as any);
|
||||||
|
remuxer.resetInitSegment(initSegment, 'mp4a.40.2', 'avc1.42001e', null);
|
||||||
|
// Audio-first layout sliced before the video sample: nothing can be
|
||||||
|
// appended to the video-only SourceBuffer
|
||||||
|
const fragmentData = muxedMp4Fragment([sample(3003, 24, 0)], [50], true);
|
||||||
|
const moofSize = findBox(fragmentData, ['moof'])[0].byteLength + 8;
|
||||||
|
const partialFragment = fragmentData.subarray(0, moofSize + 8 + 30);
|
||||||
|
|
||||||
|
const result = remuxIFrame(partialFragment, extinfDuration);
|
||||||
|
|
||||||
|
expect(result.initSegment?.tracks?.video, 'video init track').to.exist;
|
||||||
|
expect(result.video, 'video track').to.equal(undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('truncates encrypted partial-mdat iframe fragments on the mdat box boundary', function () {
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const samples = [sample(3003, 10, 0), sample(3003, 20, 0)];
|
||||||
|
const fragmentData = mp4Fragment(samples);
|
||||||
|
// Slice mid-way through the second sample's data, as a byte-range
|
||||||
|
// addressed I-Frame request would.
|
||||||
|
const mdatPayloadOffset = fragmentData.byteLength - 30;
|
||||||
|
const partialFragment = fragmentData.subarray(0, mdatPayloadOffset + 13);
|
||||||
|
|
||||||
|
const result = remuxIFrameFragment(partialFragment, extinfDuration, true);
|
||||||
|
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
expect(result.video!.encrypted).to.equal(true);
|
||||||
|
// Appended data ends at the rewritten mdat boundary, dropping the
|
||||||
|
// partially loaded sample's bytes.
|
||||||
|
expect(result.video!.data1.byteLength).to.equal(mdatPayloadOffset + 10);
|
||||||
|
expect(readUint32(result.video!.data1, mdatPayloadOffset - 8)).to.equal(18);
|
||||||
|
const trun = findBox(result.video!.data1, ['moof', 'traf', 'trun'])[0];
|
||||||
|
expect(readUint32(trun, 4), 'trun sample_count').to.equal(1);
|
||||||
|
expect(trunSampleDurations(result.video!.data1)).to.deep.equal([
|
||||||
|
extinfDuration * 90000,
|
||||||
|
]);
|
||||||
|
expect(result.video!.endPTS - result.video!.startPTS).to.equal(
|
||||||
|
extinfDuration,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stretches a partial-mdat iframe whose declared duration already matches EXTINF', function () {
|
||||||
|
// The moof declares two samples that together already span EXTINF, so
|
||||||
|
// needsDurationAdjustment is false — but only the first sample's data is
|
||||||
|
// present in the byte-range slice (as with a muxed I-Frame byte range).
|
||||||
|
const extinfDuration = 4;
|
||||||
|
const fragmentData = mp4Fragment([
|
||||||
|
sample(extinfDuration * 90000 - 3003, 10, 0),
|
||||||
|
sample(3003, 20, 0),
|
||||||
|
]);
|
||||||
|
const mdatPayloadOffset = fragmentData.byteLength - 30;
|
||||||
|
const partialFragment = fragmentData.subarray(0, mdatPayloadOffset + 13);
|
||||||
|
|
||||||
|
const result = remuxIFrameFragment(partialFragment, extinfDuration);
|
||||||
|
|
||||||
|
expect(result.video, 'video track').to.exist;
|
||||||
|
// The single kept keyframe is stretched to the full declared duration even
|
||||||
|
// though the moof timing did not disagree with EXTINF
|
||||||
|
expect(trunSampleDurations(result.video!.data1)).to.deep.equal([
|
||||||
|
extinfDuration * 90000,
|
||||||
|
]);
|
||||||
|
expect(result.video!.endPTS - result.video!.startPTS).to.equal(
|
||||||
|
extinfDuration,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
function markVideoInitSegmentEncrypted(
|
||||||
|
initSegment: Uint8Array<ArrayBuffer>,
|
||||||
|
): void {
|
||||||
|
const stsd = findBox(initSegment, [
|
||||||
|
'moov',
|
||||||
|
'trak',
|
||||||
|
'mdia',
|
||||||
|
'minf',
|
||||||
|
'stbl',
|
||||||
|
'stsd',
|
||||||
|
])[0];
|
||||||
|
const sampleEntry = findBox(stsd.subarray(8), ['avc1'])[0];
|
||||||
|
const typeOffset = sampleEntry.byteOffset - initSegment.byteOffset - 4;
|
||||||
|
initSegment.set([0x65, 0x6e, 0x63, 0x76], typeOffset); // 'encv'
|
||||||
|
}
|
||||||
|
|
||||||
|
function readUint32(buffer: Uint8Array, offset: number): number {
|
||||||
|
return (
|
||||||
|
((buffer[offset] << 24) |
|
||||||
|
(buffer[offset + 1] << 16) |
|
||||||
|
(buffer[offset + 2] << 8) |
|
||||||
|
buffer[offset + 3]) >>>
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function trunSampleDurations(data: Uint8Array): number[] {
|
||||||
|
const trun = findBox(data, ['moof', 'traf', 'trun'])[0];
|
||||||
|
const sampleCount = readUint32(trun, 4);
|
||||||
|
const durations: number[] = [];
|
||||||
|
// full box header (4) + sample_count (4) + data_offset (4);
|
||||||
|
// entries are duration, size, flags, cts (16 bytes each)
|
||||||
|
for (let i = 0; i < sampleCount; i++) {
|
||||||
|
durations.push(readUint32(trun, 12 + i * 16));
|
||||||
|
}
|
||||||
|
return durations;
|
||||||
|
}
|
||||||
|
|
||||||
|
function videoInitTrack(): any {
|
||||||
|
return {
|
||||||
|
codec: 'avc1.42001e',
|
||||||
|
dropped: 0,
|
||||||
|
duration: 4,
|
||||||
|
id: 1,
|
||||||
|
inputTimeScale: 90000,
|
||||||
|
len: 0,
|
||||||
|
nbNalu: 0,
|
||||||
|
pid: -1,
|
||||||
|
pixelRatio: [1, 1],
|
||||||
|
pps: [new Uint8Array([0x68, 0xce, 0x06, 0xe2])],
|
||||||
|
samples: [],
|
||||||
|
segmentCodec: 'avc',
|
||||||
|
sequenceNumber: 0,
|
||||||
|
sps: [new Uint8Array([0x67, 0x42, 0x00, 0x1e, 0xab, 0x40])],
|
||||||
|
timescale: 90000,
|
||||||
|
type: 'video',
|
||||||
|
width: 16,
|
||||||
|
height: 16,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function audioInitTrack(): any {
|
||||||
|
return {
|
||||||
|
codec: 'mp4a.40.2',
|
||||||
|
segmentCodec: 'aac',
|
||||||
|
channelCount: 2,
|
||||||
|
samplerate: 48000,
|
||||||
|
config: [0x11, 0x90],
|
||||||
|
duration: 4,
|
||||||
|
id: 2,
|
||||||
|
timescale: 48000,
|
||||||
|
type: 'audio',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// moof with a video traf (track 1) and an audio traf (track 2) sharing one
|
||||||
|
// mdat: video run first, audio run after (ffmpeg muxed fMP4 layout)
|
||||||
|
function muxedMp4Fragment(
|
||||||
|
videoSamples: TrackFragmentSample[],
|
||||||
|
audioSampleSizes: number[],
|
||||||
|
audioFirst = false,
|
||||||
|
): Uint8Array<ArrayBuffer> {
|
||||||
|
const videoBytes = videoSamples.reduce((total, s) => total + s.size, 0);
|
||||||
|
const audioBytes = audioSampleSizes.reduce((total, s) => total + s, 0);
|
||||||
|
const videoMoof = MP4.moof(0, 0, {
|
||||||
|
type: 'video',
|
||||||
|
id: 1,
|
||||||
|
samples: videoSamples,
|
||||||
|
});
|
||||||
|
|
||||||
|
const tfhd = new Uint8Array(8);
|
||||||
|
tfhd.set([0x00, 0x02, 0x00, 0x00]); // version 0, default-base-is-moof
|
||||||
|
writeUint32(tfhd, 4, 2); // track_ID
|
||||||
|
const tfdt = new Uint8Array(8);
|
||||||
|
const audioTrun = new Uint8Array(12 + audioSampleSizes.length * 8);
|
||||||
|
audioTrun.set([0x00, 0x00, 0x03, 0x01]); // data-offset + duration + size
|
||||||
|
writeUint32(audioTrun, 4, audioSampleSizes.length);
|
||||||
|
audioSampleSizes.forEach((size, i) => {
|
||||||
|
writeUint32(audioTrun, 12 + i * 8, 1024);
|
||||||
|
writeUint32(audioTrun, 12 + i * 8 + 4, size);
|
||||||
|
});
|
||||||
|
const audioTraf = MP4.box(
|
||||||
|
types.traf,
|
||||||
|
MP4.box(types.tfhd, tfhd),
|
||||||
|
MP4.box(types.tfdt, tfdt),
|
||||||
|
MP4.box(types.trun, audioTrun),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Splice the audio traf into the moof, then patch the moof size and both
|
||||||
|
// trun data offsets for the combined mdat layout
|
||||||
|
const moof = new Uint8Array(videoMoof.byteLength + audioTraf.byteLength);
|
||||||
|
moof.set(videoMoof, 0);
|
||||||
|
moof.set(audioTraf, videoMoof.byteLength);
|
||||||
|
writeUint32(moof, 0, moof.byteLength);
|
||||||
|
const truns = findBox(moof, ['moof', 'traf', 'trun']);
|
||||||
|
writeUint32(
|
||||||
|
moof,
|
||||||
|
truns[0].byteOffset - moof.byteOffset + 8,
|
||||||
|
moof.byteLength + 8 + (audioFirst ? audioBytes : 0),
|
||||||
|
);
|
||||||
|
writeUint32(
|
||||||
|
moof,
|
||||||
|
truns[1].byteOffset - moof.byteOffset + 8,
|
||||||
|
moof.byteLength + 8 + (audioFirst ? 0 : videoBytes),
|
||||||
|
);
|
||||||
|
// CMAF muxed content addresses samples from the moof; the video traf must
|
||||||
|
// set default-base-is-moof so the remuxer can splice out the audio traf
|
||||||
|
const videoTfhd = findBox(moof, ['moof', 'traf', 'tfhd'])[0];
|
||||||
|
videoTfhd[1] = 0x02;
|
||||||
|
|
||||||
|
return appendUint8Array(
|
||||||
|
moof,
|
||||||
|
MP4.mdat(new Uint8Array(videoBytes + audioBytes)),
|
||||||
|
) as Uint8Array<ArrayBuffer>;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Appends a second (empty) video track fragment to the moof so the video
|
||||||
|
// track has more than one trun
|
||||||
|
function withSecondVideoTraf(
|
||||||
|
fragment: Uint8Array<ArrayBuffer>,
|
||||||
|
): Uint8Array<ArrayBuffer> {
|
||||||
|
const tfhd = new Uint8Array(8);
|
||||||
|
tfhd.set([0x00, 0x02, 0x00, 0x00]);
|
||||||
|
writeUint32(tfhd, 4, 1); // video track_ID
|
||||||
|
const trun = new Uint8Array(12);
|
||||||
|
trun[2] = 0x01; // data-offset present
|
||||||
|
const traf = MP4.box(
|
||||||
|
types.traf,
|
||||||
|
MP4.box(types.tfhd, tfhd),
|
||||||
|
MP4.box(types.trun, trun),
|
||||||
|
);
|
||||||
|
const moofSize = findBox(fragment, ['moof'])[0].byteLength + 8;
|
||||||
|
const result = new Uint8Array(fragment.byteLength + traf.byteLength);
|
||||||
|
result.set(fragment.subarray(0, moofSize), 0);
|
||||||
|
result.set(traf, moofSize);
|
||||||
|
result.set(fragment.subarray(moofSize), moofSize + traf.byteLength);
|
||||||
|
writeUint32(result, 0, moofSize + traf.byteLength);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mp4Fragment(samples: TrackFragmentSample[]): Uint8Array<ArrayBuffer> {
|
||||||
|
const mdatPayload = new Uint8Array(
|
||||||
|
samples.reduce((total, sample) => total + sample.size, 0),
|
||||||
|
);
|
||||||
|
return appendUint8Array(
|
||||||
|
MP4.moof(0, 0, { type: 'video', id: 1, samples }),
|
||||||
|
MP4.mdat(mdatPayload),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sample(
|
||||||
|
duration: number,
|
||||||
|
size: number,
|
||||||
|
cts: number,
|
||||||
|
): TrackFragmentSample {
|
||||||
|
return {
|
||||||
|
cts,
|
||||||
|
duration,
|
||||||
|
flags: {
|
||||||
|
degradPrio: 0,
|
||||||
|
dependsOn: 2,
|
||||||
|
hasRedundancy: 0,
|
||||||
|
isDependedOn: 0,
|
||||||
|
isLeading: 0,
|
||||||
|
isNonSync: 0,
|
||||||
|
paddingValue: 0,
|
||||||
|
},
|
||||||
|
size,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function passthroughTrack(samples: Uint8Array<ArrayBuffer>): PassthroughTrack {
|
||||||
|
return {
|
||||||
|
codec: 'avc1.42001e',
|
||||||
|
dropped: 0,
|
||||||
|
duration: 4,
|
||||||
|
id: 1,
|
||||||
|
inputTimeScale: 90000,
|
||||||
|
pid: -1,
|
||||||
|
sampleDuration: 3003,
|
||||||
|
samples,
|
||||||
|
sequenceNumber: 0,
|
||||||
|
supplemental: undefined,
|
||||||
|
timescale: 90000,
|
||||||
|
type: 'video',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function audioTrack(): DemuxedAudioTrack {
|
||||||
|
return {
|
||||||
|
dropped: 0,
|
||||||
|
id: 2,
|
||||||
|
inputTimeScale: 90000,
|
||||||
|
pid: -1,
|
||||||
|
samples: [],
|
||||||
|
segmentCodec: 'aac',
|
||||||
|
sequenceNumber: 0,
|
||||||
|
type: 'audio',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function metadataTrack(): DemuxedMetadataTrack {
|
||||||
|
return {
|
||||||
|
dropped: 0,
|
||||||
|
id: 3,
|
||||||
|
inputTimeScale: 90000,
|
||||||
|
pid: -1,
|
||||||
|
samples: [],
|
||||||
|
sequenceNumber: 0,
|
||||||
|
type: 'id3',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function userdataTrack(): DemuxedUserdataTrack {
|
||||||
|
return {
|
||||||
|
dropped: 0,
|
||||||
|
id: 4,
|
||||||
|
inputTimeScale: 90000,
|
||||||
|
pid: -1,
|
||||||
|
samples: [],
|
||||||
|
sequenceNumber: 0,
|
||||||
|
type: 'text',
|
||||||
|
};
|
||||||
|
}
|
||||||
Reference in new issue
Block a user