feat: Provide the binary data for the video header (box: ftyp, moov). #…

…235 #348
bilibili · Jan 24, 2025 · a55d12e · a55d12e
1 parent 1f4e7d5
commit a55d12e
Show file tree

Hide file tree

Showing 4 changed files with 89 additions and 28 deletions.
diff --git a/.changeset/healthy-garlics-thank.md b/.changeset/healthy-garlics-thank.md
@@ -0,0 +1,5 @@
+---
+'@webav/av-cliper': patch
+---
+
+feat: Provide the binary data for the video header (box: ftyp, moov). #235 #348
diff --git a/packages/av-cliper/src/clips/__tests__/mp4-clip.test.ts b/packages/av-cliper/src/clips/__tests__/mp4-clip.test.ts
@@ -1,6 +1,7 @@
-import { expect, test } from 'vitest';
+import { expect, test, vi } from 'vitest';
 import { MP4Clip } from '../mp4-clip';
 import { file, write } from 'opfs-tools';
+import mp4box, { MP4ArrayBuffer } from '@webav/mp4box.js';
 
 const mp4_123 = `//${location.host}/video/123.mp4`;
 
@@ -153,3 +154,18 @@ test('create instance by opfs file', async () => {
     f.remove();
   }
 });
+
+test('get file header data', async () => {
+  const clip = new MP4Clip((await fetch(mp4_123)).body!);
+  const boxfile = mp4box.createFile();
+  boxfile.onReady = vi.fn();
+  const buf = (await clip.getFileHeaderBinData()) as MP4ArrayBuffer;
+  buf.fileStart = 0;
+  boxfile.appendBuffer(buf);
+  expect(boxfile.onReady).toBeCalledWith(
+    expect.objectContaining({
+      hasMoov: true,
+    }),
+  );
+  expect(boxfile.moov?.mvhd.matrix.length).toBe(9);
+});
diff --git a/packages/av-cliper/src/clips/mp4-clip.ts b/packages/av-cliper/src/clips/mp4-clip.ts
@@ -85,6 +85,24 @@ export class MP4Clip implements IClip {
 
   #localFile: OPFSToolFile;
 
+  #headerBoxPos: Array<{ start: number; size: number }> = [];
+  /**
+   * 提供视频头（box: ftyp, moov）的二进制数据
+   * 使用任意 mp4 demxer 解析即可获得详细的视频信息
+   * 单元测试包含使用 mp4box.js 解析示例代码
+   */
+  async getFileHeaderBinData() {
+    await this.ready;
+    const oFile = await this.#localFile.getOriginFile();
+    if (oFile == null) throw Error('MP4Clip localFile is not origin file');
+
+    return await new Blob(
+      this.#headerBoxPos.map(({ start, size }) =>
+        oFile.slice(start, start + size),
+      ),
+    ).arrayBuffer();
+  }
+
   #volume = 1;
 
   #videoSamples: ExtMP4Sample[] = [];
@@ -141,34 +159,38 @@ export class MP4Clip implements IClip {
         : isOTFile(source)
           ? mp4FileToSamples(source, this.#opts)
           : Promise.resolve(source)
-    ).then(async ({ videoSamples, audioSamples, decoderConf }) => {
-      this.#videoSamples = videoSamples;
-      this.#audioSamples = audioSamples;
-      this.#decoderConf = decoderConf;
-      const { videoFrameFinder, audioFrameFinder } = genDecoder(
-        {
-          video:
-            decoderConf.video == null
-              ? null
-              : {
-                  ...decoderConf.video,
-                  hardwareAcceleration:
-                    this.#opts.__unsafe_hardwareAcceleration__,
-                },
-          audio: decoderConf.audio,
-        },
-        await this.#localFile.createReader(),
-        videoSamples,
-        audioSamples,
-        this.#opts.audio !== false ? this.#volume : 0,
-      );
-      this.#videoFrameFinder = videoFrameFinder;
-      this.#audioFrameFinder = audioFrameFinder;
+    ).then(
+      async ({ videoSamples, audioSamples, decoderConf, headerBoxPos }) => {
+        this.#videoSamples = videoSamples;
+        this.#audioSamples = audioSamples;
+        this.#decoderConf = decoderConf;
+        this.#headerBoxPos = headerBoxPos;
+
+        const { videoFrameFinder, audioFrameFinder } = genDecoder(
+          {
+            video:
+              decoderConf.video == null
+                ? null
+                : {
+                    ...decoderConf.video,
+                    hardwareAcceleration:
+                      this.#opts.__unsafe_hardwareAcceleration__,
+                  },
+            audio: decoderConf.audio,
+          },
+          await this.#localFile.createReader(),
+          videoSamples,
+          audioSamples,
+          this.#opts.audio !== false ? this.#volume : 0,
+        );
+        this.#videoFrameFinder = videoFrameFinder;
+        this.#audioFrameFinder = audioFrameFinder;
 
-      this.#meta = genMeta(decoderConf, videoSamples, audioSamples);
-      this.#log.info('MP4Clip meta:', this.#meta);
-      return { ...this.#meta };
-    });
+        this.#meta = genMeta(decoderConf, videoSamples, audioSamples);
+        this.#log.info('MP4Clip meta:', this.#meta);
+        return { ...this.#meta };
+      },
+    );
   }
 
   /**
@@ -332,6 +354,7 @@ export class MP4Clip implements IClip {
         videoSamples: preVideoSlice ?? [],
         audioSamples: preAudioSlice ?? [],
         decoderConf: this.#decoderConf,
+        headerBoxPos: this.#headerBoxPos,
       },
       this.#opts,
     );
@@ -341,6 +364,7 @@ export class MP4Clip implements IClip {
         videoSamples: postVideoSlice ?? [],
         audioSamples: postAudioSlice ?? [],
         decoderConf: this.#decoderConf,
+        headerBoxPos: this.#headerBoxPos,
       },
       this.#opts,
     );
@@ -357,6 +381,7 @@ export class MP4Clip implements IClip {
         videoSamples: [...this.#videoSamples],
         audioSamples: [...this.#audioSamples],
         decoderConf: this.#decoderConf,
+        headerBoxPos: this.#headerBoxPos,
       },
       this.#opts,
     );
@@ -382,6 +407,7 @@ export class MP4Clip implements IClip {
             video: this.#decoderConf.video,
             audio: null,
           },
+          headerBoxPos: this.#headerBoxPos,
         },
         this.#opts,
       );
@@ -399,6 +425,7 @@ export class MP4Clip implements IClip {
             audio: this.#decoderConf.audio,
             video: null,
           },
+          headerBoxPos: this.#headerBoxPos,
         },
         this.#opts,
       );
@@ -496,6 +523,7 @@ async function mp4FileToSamples(otFile: OPFSToolFile, opts: MP4ClipOpts = {}) {
   const decoderConf: MP4DecoderConf = { video: null, audio: null };
   let videoSamples: ExtMP4Sample[] = [];
   let audioSamples: ExtMP4Sample[] = [];
+  let headerBoxPos: Array<{ start: number; size: number }> = [];
 
   let videoDeltaTS = -1;
   let audioDeltaTS = -1;
@@ -504,6 +532,11 @@ async function mp4FileToSamples(otFile: OPFSToolFile, opts: MP4ClipOpts = {}) {
     reader,
     (data) => {
       mp4Info = data.info;
+      const ftyp = data.mp4boxFile.ftyp!;
+      headerBoxPos.push({ start: ftyp.start, size: ftyp.size });
+      const moov = data.mp4boxFile.moov!;
+      headerBoxPos.push({ start: moov.start, size: moov.size });
+
       let { videoDecoderConf: vc, audioDecoderConf: ac } = extractFileConfig(
         data.mp4boxFile,
         data.info,
@@ -553,6 +586,7 @@ async function mp4FileToSamples(otFile: OPFSToolFile, opts: MP4ClipOpts = {}) {
     videoSamples,
     audioSamples,
     decoderConf,
+    headerBoxPos,
   };
 
   function normalizeTimescale(

diff --git a/types/mp4box.d.ts b/types/mp4box.d.ts
@@ -188,10 +188,15 @@ declare module '@webav/mp4box.js' {
     mvhd: MVHDBoxParser;
   }
 
+  interface FTYPBoxParser extends BoxParser {
+    type: 'ftyp';
+  }
+
   interface MVHDBoxParser extends BoxParser {
     type: 'mvhd';
     duration: number;
     timescale: number;
+    matrix: Uint32Array;
   }
 
   interface TKHDBoxParser extends BoxParser {
@@ -259,6 +264,7 @@ declare module '@webav/mp4box.js' {
     boxes: BoxParser[];
     mdats: MDATBoxParser[];
     moofs: MOOFBoxParser[];
+    ftyp?: MOOVBoxParser;
     moov?: MOOVBoxParser;
 
     add: (name: string) => BoxParser;