Smaller frames (#3607)

src/core/builtins/builtins_ffmpeg_bitstream_filters.ml

@@ -83,9 +83,7 @@ let process (type a) ~put_data ~(mk_params : a mk_params)
           } ))
       packets
   in
-  let data =
-    { Ffmpeg_content_base.params = Some (mk_params params); data; length }
-  in
+  let data = { Content.Video.params = Some (mk_params params); data; length } in
   let data = Ffmpeg_copy_content.lift_data data in
   put_data generator data
 
@@ -121,7 +119,7 @@ let on_data (type a)
        a handler) ~put_data ~(get_params : a get_params)
     ~(get_packet : a get_packet) ~(mk_params : a mk_params)
     ~(mk_packet : a mk_packet) ~generator
-    ({ Ffmpeg_content_base.params; data } : Ffmpeg_copy_content.data) =
+    ({ Content.Video.params; data } : Ffmpeg_copy_content.data) =
   List.iter
     (fun (_, { Ffmpeg_copy_content.stream_idx; time_base; packet }) ->
       let handler =

src/core/builtins/builtins_ffmpeg_decoder.ml

@@ -98,7 +98,7 @@ let decode_audio_frame ~field ~mode generator =
     | `Frame frame ->
         let data, params =
           match Ffmpeg_copy_content.get_data frame with
-            | { Ffmpeg_content_base.data; params = Some (`Audio params) } ->
+            | { Content.Video.data; params = Some (`Audio params) } ->
                 (data, params)
             | _ -> assert false
         in
@@ -165,7 +165,7 @@ let decode_audio_frame ~field ~mode generator =
 
     function
     | `Frame frame ->
-        let { Ffmpeg_content_base.data; params } =
+        let { Content.Video.data; params } =
           Ffmpeg_raw_content.Audio.get_data frame
         in
         let data =
@@ -181,14 +181,14 @@ let decode_audio_frame ~field ~mode generator =
 
   let convert
         : 'a 'b.
-          get_data:(Content.data -> ('a, 'b) Ffmpeg_content_base.content) ->
+          get_data:(Content.data -> ('a, 'b) Content_video.Base.content) ->
           decoder:([ `Frame of Content.data | `Flush ] -> unit) ->
           [ `Frame of Frame.t | `Flush ] ->
           unit =
    fun ~get_data ~decoder -> function
     | `Frame frame ->
         let frame = Frame.get frame field in
-        let { Ffmpeg_content_base.data; _ } = get_data frame in
+        let { Content.Video.data; _ } = get_data frame in
         if data = [] then () else decoder (`Frame frame)
     | `Flush -> decoder `Flush
   in
@@ -253,8 +253,7 @@ let decode_video_frame ~field ~mode generator =
         Ffmpeg_utils.unpack_image ~width:internal_width ~height:internal_height
           (InternalScaler.convert scaler data)
       in
-      let data = Video.Canvas.single_image img in
-      let data = Content.Video.lift_data data in
+      let data = Content.Video.lift_image (Video.Canvas.Image.make img) in
       Generator.put generator field data
     in
 
@@ -318,7 +317,7 @@ let decode_video_frame ~field ~mode generator =
     | `Frame frame ->
         let data, params =
           match Ffmpeg_copy_content.get_data frame with
-            | { Ffmpeg_content_base.data; params = Some (`Video params) } ->
+            | { Content.Video.data; params = Some (`Video params) } ->
                 (data, params)
             | _ -> assert false
         in
@@ -364,7 +363,7 @@ let decode_video_frame ~field ~mode generator =
     let last_params = ref None in
     function
     | `Frame frame ->
-        let { Ffmpeg_content_base.data; _ } =
+        let { Content.Video.data; _ } =
           Ffmpeg_raw_content.Video.get_data frame
         in
         let data =
@@ -385,14 +384,14 @@ let decode_video_frame ~field ~mode generator =
 
   let convert
         : 'a 'b.
-          get_data:(Content.data -> ('a, 'b) Ffmpeg_content_base.content) ->
+          get_data:(Content.data -> ('a, 'b) Content_video.Base.content) ->
           decoder:([ `Frame of Content.data | `Flush ] -> unit) ->
           [ `Frame of Frame.t | `Flush ] ->
           unit =
    fun ~get_data ~decoder -> function
     | `Frame frame ->
         let frame = Frame.get frame field in
-        let { Ffmpeg_content_base.data; _ } = get_data frame in
+        let { Content.Video.data; _ } = get_data frame in
         if data = [] then () else decoder (`Frame frame)
     | `Flush -> decoder `Flush
   in

src/core/builtins/builtins_ffmpeg_encoder.ml

@@ -110,7 +110,7 @@ let encode_audio_frame ~source_idx ~type_t ~mode ~opts ?codec ~format
                           } ))
                       packets
                   in
-                  let data = { Ffmpeg_content_base.params; data; length } in
+                  let data = { Content.Video.params; data; length } in
                   let data = Ffmpeg_copy_content.lift_data data in
                   Generator.put generator field data
               | None -> ()
@@ -160,7 +160,7 @@ let encode_audio_frame ~source_idx ~type_t ~mode ~opts ?codec ~format
                               } ))
                           frames
                       in
-                      let data = { Ffmpeg_content_base.params; data; length } in
+                      let data = { Content.Video.params; data; length } in
                       let data = Ffmpeg_raw_content.Audio.lift_data data in
                       Generator.put generator field data
                   | None -> ())
@@ -294,7 +294,7 @@ let encode_video_frame ~source_idx ~type_t ~mode ~opts ?codec ~format ~field
                           } ))
                       packets
                   in
-                  let data = { Ffmpeg_content_base.params; data; length } in
+                  let data = { Content.Video.params; data; length } in
                   let data = Ffmpeg_copy_content.lift_data data in
                   Generator.put generator field data
               | None -> ()
@@ -350,7 +350,7 @@ let encode_video_frame ~source_idx ~type_t ~mode ~opts ?codec ~format ~field
                             } ))
                         frames
                     in
-                    let data = { Ffmpeg_content_base.params; data; length } in
+                    let data = { Content.Video.params; data; length } in
                     let data = Ffmpeg_raw_content.Video.lift_data data in
                     Generator.put generator field data
                 | None -> ())
@@ -368,17 +368,16 @@ let encode_video_frame ~source_idx ~type_t ~mode ~opts ?codec ~format ~field
 
   function
   | `Frame frame ->
-      let vstart = 0 in
-      let vstop = VFrame.position frame in
       let vbuf = VFrame.data frame in
-      for i = vstart to vstop - 1 do
-        let f = Video.Canvas.render vbuf i in
-        let vdata = Ffmpeg_utils.pack_image f in
-        let frame = InternalScaler.convert (Option.get !scaler) vdata in
-        Avutil.Frame.set_pts frame (Some !nb_frames);
-        nb_frames := Int64.succ !nb_frames;
-        encode_ffmpeg_frame frame
-      done
+      List.iter
+        (fun (_, img) ->
+          let f = Video.Canvas.Image.render img in
+          let vdata = Ffmpeg_utils.pack_image f in
+          let frame = InternalScaler.convert (Option.get !scaler) vdata in
+          Avutil.Frame.set_pts frame (Some !nb_frames);
+          nb_frames := Int64.succ !nb_frames;
+          encode_ffmpeg_frame frame)
+        vbuf.Content.Video.data
   | `Flush -> encode_frame `Flush
 
 let mk_encoder mode =

src/core/decoder/decoder.ml

@@ -62,7 +62,7 @@ type fps = Decoder_utils.fps = { num : int; den : int }
 type buffer = {
   generator : Generator.t;
   put_pcm : ?field:Frame.field -> samplerate:int -> Content.Audio.data -> unit;
-  put_yuva420p : ?field:Frame.field -> fps:fps -> Content.Video.data -> unit;
+  put_yuva420p : ?field:Frame.field -> fps:fps -> Video.Canvas.image -> unit;
 }
 
 type decoder = {
@@ -240,7 +240,7 @@ let test_file ?(log = log) ?mimes ?extensions fname =
     ext_ok || mime_ok)
 
 let channel_layout audio =
-  Lazy.force Content.(Audio.(get_params audio).Content.channel_layout)
+  Lazy.force Content.(Audio.(get_params audio).Content.Audio.channel_layout)
 
 let can_decode_type decoded_type target_type =
   let map_convertible cur (field, target_field) =
@@ -451,16 +451,28 @@ let mk_buffer ~ctype generator =
           let out_freq =
             Decoder_utils.{ num = Lazy.force Frame.video_rate; den = 1 }
           in
-          fun ~fps (data : Content.Video.data) ->
-            let data = Array.map video_scale data in
-            let data = video_resample ~in_freq:fps ~out_freq data in
-            let len = Video.Canvas.length data in
-            let data =
-              Content.Video.lift_data
-                ~length:(Frame_settings.main_of_video len)
-                data
-            in
-            Generator.put generator field data)
+          let params =
+            {
+              Content.Video.width = Some Frame.video_width;
+              height = Some Frame.video_height;
+            }
+          in
+          let interval = Frame.main_of_video 1 in
+          fun ~fps img ->
+            match video_resample ~in_freq:fps ~out_freq img with
+              | [] -> ()
+              | data ->
+                  let data =
+                    List.mapi
+                      (fun i img -> (i * interval, video_scale img))
+                      data
+                  in
+                  let length = List.length data * interval in
+                  let buf =
+                    Content.Video.lift_data
+                      { Content.Video.params; length; data }
+                  in
+                  Generator.put generator field buf)
         else fun ~fps:_ _ -> ()
       in
       Hashtbl.add video_handlers field handler;
@@ -471,8 +483,8 @@ let mk_buffer ~ctype generator =
     get_audio_handler ~field ~samplerate data
   in
 
-  let put_yuva420p ?(field = Frame.Fields.video) ~fps data =
-    get_video_handler ~field ~fps data
+  let put_yuva420p ?(field = Frame.Fields.video) ~fps img =
+    get_video_handler ~field ~fps img
   in
 
   { generator; put_pcm; put_yuva420p }

src/core/decoder/decoder.mli

@@ -49,8 +49,8 @@ type fps = { num : int; den : int }
     - Implicit content drop *)
 type buffer = {
   generator : Generator.t;
-  put_pcm : ?field:Frame.field -> samplerate:int -> Content.Audio.data -> unit;
-  put_yuva420p : ?field:Frame.field -> fps:fps -> Content.Video.data -> unit;
+  put_pcm : ?field:Frame.field -> samplerate:int -> Audio.t -> unit;
+  put_yuva420p : ?field:Frame.field -> fps:fps -> Video.Canvas.image -> unit;
 }
 
 type decoder = {

src/core/decoder/decoder_utils.ml

@@ -108,30 +108,29 @@ let video_resample ~in_freq ~out_freq =
    *     which o: nearest neighbour in the currently available buffer.
    *     This is not as good as nearest neighbour in the real stream.
    *
-   * Turns out the same code codes for when out_freq>in_freq too. *)
+   * Turns out the same code codes for when out_freq>in_freq works too. *)
   let in_pos = ref 0 in
   let in_freq = in_freq.num * out_freq.den
   and out_freq = out_freq.num * in_freq.num in
   let ratio = out_freq / in_freq in
-  fun input off len ->
-    let new_in_pos = !in_pos + len in
+  fun img ->
+    let new_in_pos = !in_pos + 1 in
     let already_out_len = !in_pos * ratio in
     let needed_out_len = new_in_pos * ratio in
     let out_len = needed_out_len - already_out_len in
     in_pos := new_in_pos mod in_freq;
-    Array.init out_len (fun i -> input.(off + (i * ratio)))
+    List.init out_len (fun _ -> img)
 
 let video_resample () =
   let state = ref None in
-  let exec resampler data = resampler data 0 (Video.Canvas.length data) in
-  fun ~in_freq ~out_freq (data : Content.Video.data) : Content.Video.data ->
-    if in_freq = out_freq then data
+  fun ~in_freq ~out_freq img ->
+    if in_freq = out_freq then [img]
     else (
       match !state with
         | Some (resampler, _in_freq, _out_freq)
           when in_freq = _in_freq && out_freq = _out_freq ->
-            exec resampler data
+            resampler img
         | _ ->
             let resampler = video_resample ~in_freq ~out_freq in
             state := Some (resampler, in_freq, out_freq);
-            exec resampler data)
+            resampler img)

src/core/decoder/decoder_utils.mli

@@ -59,5 +59,5 @@ val video_resample :
   unit ->
   in_freq:fps ->
   out_freq:fps ->
-  Content.Video.data ->
-  Content.Video.data
+  Video.Canvas.image ->
+  Video.Canvas.image list

src/core/decoder/ffmpeg_copy_decoder.ml

@@ -54,7 +54,7 @@ let mk_decoder ~stream_idx ~stream_time_base ~mk_packet ~put_data params =
               } ))
           packets
       in
-      let data = { Ffmpeg_content_base.params = Some params; data; length } in
+      let data = { Content.Video.params = Some params; data; length } in
       let data = Ffmpeg_copy_content.lift_data data in
       put_data buffer.Decoder.generator data
     with Empty | Corrupt (* Might want to change that later. *) -> ()

src/core/decoder/ffmpeg_internal_decoder.ml

@@ -159,10 +159,9 @@ let mk_video_decoder ~width ~height ~stream ~field codec =
   let pixel_aspect = Av.get_pixel_aspect stream in
   let cb ~buffer frame =
     let img = scale frame in
-    let content = Video.Canvas.single img in
     buffer.Decoder.put_yuva420p ~field
       ~fps:{ Decoder.num = target_fps; den = 1 }
-      content;
+      img;
     let metadata = Avutil.Frame.metadata frame in
     if metadata <> [] then
       Generator.add_metadata buffer.Decoder.generator

src/core/decoder/ffmpeg_raw_decoder.ml

@@ -45,7 +45,7 @@ let mk_decoder ~stream_idx ~stream_time_base ~mk_params ~lift_data ~put_data
                   frames
               in
               let data =
-                { Ffmpeg_content_base.params = mk_params params; data; length }
+                { Content.Video.params = mk_params params; data; length }
               in
               let data = lift_data data in
               put_data buffer.Decoder.generator data

src/core/decoder/gstreamer_decoder.ml

@@ -138,9 +138,8 @@ let create_decoder ?(merge_tracks = false) _ ~width ~height ~channels ~mode
       let y_stride = round4 width in
       let uv_stride = round4 (width / 2) in
       let img = Image.YUV420.make_data width height buf y_stride uv_stride in
-      let stream = Video.Canvas.single_image img in
       let fps = { Decoder.num = Lazy.force Frame.video_rate; den = 1 } in
-      buffer.Decoder.put_yuva420p ~fps stream);
+      buffer.Decoder.put_yuva420p ~fps (Video.Canvas.Image.make img));
     GU.flush ~log gst.bin
   in
   let seek off =

src/core/decoder/image_decoder.ml

@@ -119,16 +119,24 @@ let create_decoder ~ctype ~width ~height ~metadata img =
   let remaining () =
     if !duration = -1 then -1 else Frame.main_of_video !duration
   in
+  let generator =
+    Content.Video.make_generator
+      (Content.Video.get_params (Frame.Fields.find Frame.Fields.video ctype))
+  in
   let fread length =
-    let frame = Frame.create ~length ctype in
-    let video = Content.Video.get_data (Frame.get frame Frame.Fields.video) in
-    for i = 0 to Frame.video_of_main length - 1 do
-      Video.Canvas.set video i img
-    done;
-    match Frame.Fields.find_opt Frame.Fields.audio frame with
+    let frame = Frame.create ~length Frame.Fields.empty in
+    let video =
+      Content.Video.generate
+        ~create:(fun ~pos:_ ~width:_ ~height:_ () -> img)
+        generator length
+    in
+    let frame =
+      Frame.set_data frame Frame.Fields.video Content.Video.lift_data video
+    in
+    match Frame.Fields.find_opt Frame.Fields.audio ctype with
       | None -> frame
-      | Some data ->
-          let pcm = Content.Audio.get_data data in
+      | Some format ->
+          let pcm = Content.Audio.get_data (Content.make ~length format) in
           Audio.clear pcm 0 (Frame.audio_of_main length);
           Frame.set_data frame Frame.Fields.audio Content.Audio.lift_data pcm
   in

src/core/decoder/liq_ogg_decoder.ml

@@ -170,14 +170,14 @@ let create_decoder ?(merge_tracks = false) source input =
       in
       let video_feed track buf =
         let info, _ = Ogg_decoder.video_info decoder track in
-        let rgb = video_convert video_scale buf in
+        let img = video_convert video_scale buf in
         let fps =
           {
             Decoder.num = info.Ogg_decoder.fps_numerator;
             den = info.Ogg_decoder.fps_denominator;
           }
         in
-        buffer.Decoder.put_yuva420p ~fps (Video.Canvas.single_image rgb)
+        buffer.Decoder.put_yuva420p ~fps (Video.Canvas.Image.make img)
       in
       let decode_audio, decode_video =
         if decode_audio && decode_video then

src/core/decoder/midi_decoder.ml

@@ -69,7 +69,7 @@ let () =
         (fun ~metadata:_ ~ctype:_ _ ->
           Some
             (Frame.Fields.make
-               ~midi:Content.(Midi.lift_params { Content.channels = 16 })
+               ~midi:Content.(Midi.lift_params { Content.Midi.channels = 16 })
                ()));
       file_decoder =
         Some (fun ~metadata:_ ~ctype filename -> decoder ~ctype filename);

src/core/encoder/encoder.ml

@@ -96,7 +96,7 @@ let type_of_format f =
                     assert (channels > 0);
                     let params =
                       {
-                        Content.channel_layout =
+                        Content.Audio.channel_layout =
                           lazy
                             (Audio_converter.Channel_layout.layout_of_channels
                                channels);