HLS Playback (#233)

* HLS payload filtering * Unencrypted HLS Playback working on HLS native browsers; * Cleaned up and working for encrypted HLS data; * Added HLS; * Extend isVideo checks; * Added option to download payload whenever there is one; * Add support for DotYouClient headers on HlsSource; * Get HLS segmentation running on ffmpeg-wasm; * Updated segmenter to 6s segments; * Fix build errors; * Cleaned up the payloads; * Bit of type cleanup; * Last cleanup; * Disabled upload segmentation for now;
homebase-id · Sep 6, 2024 · 09f06c7 · 09f06c7
1 parent 6432e93
commit 09f06c7
Show file tree

Hide file tree

Showing 26 changed files with 633 additions and 172 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/chat-app/package.json b/packages/chat-app/package.json
@@ -26,6 +26,7 @@
     "@homebase-id/js-lib": "*",
     "@homebase-id/rich-text-editor": "*",
     "@homebase-id/ui-lib": "*",
+    "hls.js": "^1.5.15",
     "axios": "1.7.5",
     "emoji-picker-element": "^1.21.3",
     "idb-keyval": "^6.2.1",

diff --git a/packages/chat-app/src/components/Chat/Detail/Media/ChatMedia.tsx b/packages/chat-app/src/components/Chat/Detail/Media/ChatMedia.tsx
@@ -64,7 +64,9 @@ const MediaItem = ({
 }) => {
   const { isDarkMode } = useDarkMode();
   const dotYouClient = useDotYouClientContext();
-  const isVideo = payload.contentType?.startsWith('video');
+  const isVideo =
+    payload.contentType?.startsWith('video') ||
+    payload.contentType === 'application/vnd.apple.mpegurl';
   const isAudio = payload.contentType?.startsWith('audio');
   const isImage = payload.contentType?.startsWith('image');
   const isLink = payload.key === CHAT_LINKS_PAYLOAD_KEY;
@@ -88,7 +90,7 @@ const MediaItem = ({
                 lastModified={payload.lastModified || fileLastModified}
                 targetDrive={ChatDrive}
                 className={`w-full blur-sm`}
-                loadSize={{ pixelWidth: 1920, pixelHeight: 1080 }}
+                loadSize={{ pixelWidth: 200, pixelHeight: 200 }}
               />
               <div className="absolute inset-0 flex items-center justify-center">
                 <Triangle className="h-16 w-16 text-background" />

diff --git a/packages/chat-app/src/components/Chat/Detail/Media/ChatMediaGallery.tsx b/packages/chat-app/src/components/Chat/Detail/Media/ChatMediaGallery.tsx
@@ -84,7 +84,7 @@ export const ChatMediaGallery = ({ msg }: { msg: HomebaseFile<ChatMessage> }) =>
       <div className="inset-0 bg-black transition-opacity lg:fixed"></div>
       <div className="inset-0 z-10 lg:fixed lg:overflow-y-auto">
         <div className="relative flex h-full min-h-[100dvh] flex-row items-center justify-center">
-          {contentType?.startsWith('video') ? (
+          {contentType?.startsWith('video') || contentType === 'application/vnd.apple.mpegurl' ? (
             <VideoClickToLoad
               preload={true}
               fileId={msg.fileId}

diff --git a/packages/chat-app/src/providers/ChatProvider.ts b/packages/chat-app/src/providers/ChatProvider.ts
@@ -39,6 +39,7 @@ import {
   stringToUint8Array,
   makeGrid,
   base64ToUint8Array,
+  getRandom16ByteArray,
 } from '@homebase-id/js-lib/helpers';
 import { appId } from '../hooks/auth/useAuth';
 import {
@@ -267,6 +268,10 @@ export const uploadChatMessage = async (
   const payloads: PayloadFile[] = [];
   const thumbnails: ThumbnailFile[] = [];
   const previewThumbnails: EmbeddedThumb[] = [];
+  const keyHeader: KeyHeader = {
+    iv: getRandom16ByteArray(),
+    aesKey: getRandom16ByteArray(),
+  };
 
   if (!files?.length && linkPreviews?.length) {
     // We only support link previews when there is no media
@@ -307,13 +312,14 @@ export const uploadChatMessage = async (
     const payloadKey = `${CHAT_MESSAGE_PAYLOAD_KEY}${i}`;
     const newMediaFile = files[i];
     if (newMediaFile.file.type.startsWith('video/')) {
-      const { tinyThumb, additionalThumbnails, payload } = await processVideoFile(
-        newMediaFile,
-        payloadKey
-      );
+      const {
+        tinyThumb,
+        thumbnails: thumbnailsFromVideo,
+        payloads: payloadsFromVideo,
+      } = await processVideoFile(newMediaFile, payloadKey, keyHeader);
 
-      thumbnails.push(...additionalThumbnails);
-      payloads.push(payload);
+      thumbnails.push(...thumbnailsFromVideo);
+      payloads.push(...payloadsFromVideo);
 
       if (tinyThumb) previewThumbnails.push(tinyThumb);
     } else if (newMediaFile.file.type.startsWith('image/')) {
@@ -350,7 +356,10 @@ export const uploadChatMessage = async (
     payloads,
     thumbnails,
     undefined,
-    onVersionConflict
+    onVersionConflict,
+    {
+      keyHeader,
+    }
   );
 
   if (!uploadResult) return null;

diff --git a/packages/common-app/src/media/Video.tsx b/packages/common-app/src/media/Video.tsx
@@ -51,15 +51,15 @@ export const VideoClickToLoad = ({
 
   return (
     <div
-      className={`${props.className?.includes('absolute') ? '' : 'relative'} overflow-hidden ${props.className || ''}`}
+      className={`${props.className?.includes('absolute') ? '' : 'relative'} cursor-pointer overflow-hidden ${props.className || ''}`}
       onClick={(e) => {
         e.stopPropagation();
         setLoadVideo(true);
       }}
     >
       {shouldFallback ? (
         loadVideo ? null : (
-          <div className="bg-slate-200 dark:bg-slate-800 aspect-video w-full"></div>
+          <div className="bg-slate-200 dark:bg-slate-800 aspect-video w-full h-full"></div>
         )
       ) : (
         <>
@@ -73,7 +73,7 @@ export const VideoClickToLoad = ({
             dotYouClient={dotYouClient}
             {...props}
             className={`${props.className || ''} ${playingVideo ? 'opacity-0' : 'opacity-100'} absolute inset-0 blur-sm ${props.className?.includes('object-') ? '' : 'object-cover'}`}
-            loadSize={{ pixelWidth: 1920, pixelHeight: 1080 }}
+            loadSize={{ pixelWidth: 100, pixelHeight: 100 }}
             onLoad={() => setPreviewLoaded(true)}
             onError={() => setShouldFallback(true)}
             probablyEncrypted={probablyEncrypted}

diff --git a/packages/common-app/src/socialFeed/Blocks/Media/MediaGallery.tsx b/packages/common-app/src/socialFeed/Blocks/Media/MediaGallery.tsx
@@ -76,54 +76,60 @@ export const MediaGallery = ({
               tinyThumbUrl ? 'absolute inset-0' : ''
             } ${someLoaded || hasFirstInCache ? 'opacity-100' : 'opacity-0'} grid grid-cols-2 gap-1 bg-background`}
           >
-            {slicedFiles.map((file, index) => (
-              <div
-                className={slicedFiles.length === 3 && index === 2 ? 'col-span-2' : undefined}
-                key={file.key}
-              >
+            {slicedFiles.map((file, index) => {
+              const isVideo =
+                file.contentType.startsWith('video') ||
+                file.contentType.startsWith('application/vnd.apple.mpegurl');
+
+              return (
                 <div
-                  className={`relative ${
-                    slicedFiles.length === 3 && index === 2 ? 'aspect-[2/1]' : 'aspect-square'
-                  } h-auto w-full cursor-pointer overflow-hidden`}
-                  onClick={onClick ? (e) => onClick(e, index) : undefined}
+                  className={slicedFiles.length === 3 && index === 2 ? 'col-span-2' : undefined}
+                  key={file.key}
                 >
-                  {file.contentType.startsWith('image') || file.contentType.startsWith('video') ? (
-                    <Image
-                      odinId={odinId}
-                      className={`h-full w-auto ${file.contentType.startsWith('video') ? 'blur-sm' : ''}`}
-                      fileId={fileId}
-                      globalTransitId={globalTransitId}
-                      fileKey={file.key}
-                      lastModified={lastModified}
-                      targetDrive={targetDrive}
-                      fit="cover"
-                      probablyEncrypted={probablyEncrypted}
-                      avoidPayload={file.contentType.startsWith('video')}
-                      onLoad={() => setSomeLoaded(true)}
-                    />
-                  ) : (
-                    <BoringFile
-                      odinId={odinId}
-                      targetDrive={getChannelDrive(channelId)}
-                      fileId={fileId}
-                      file={file}
-                    />
-                  )}
-
-                  {index === maxVisible - 1 && countExcludedFromView > 0 ? (
-                    <div className="absolute inset-0 flex flex-col justify-center bg-black bg-opacity-40 text-6xl font-light text-white">
-                      <span className="block text-center">+{countExcludedFromView}</span>
-                    </div>
-                  ) : file.contentType.startsWith('video') ? (
-                    <div className="absolute inset-0 flex items-center justify-center">
-                      <div className="bg-background/40 rounded-full p-7 border border-foreground/20">
-                        <Triangle className="text-foreground h-12 w-12" />
+                  <div
+                    className={`relative ${
+                      slicedFiles.length === 3 && index === 2 ? 'aspect-[2/1]' : 'aspect-square'
+                    } h-auto w-full cursor-pointer overflow-hidden`}
+                    onClick={onClick ? (e) => onClick(e, index) : undefined}
+                  >
+                    {file.contentType.startsWith('image') || isVideo ? (
+                      <Image
+                        odinId={odinId}
+                        className={`h-full w-auto ${isVideo ? 'blur-sm' : ''}`}
+                        fileId={fileId}
+                        globalTransitId={globalTransitId}
+                        fileKey={file.key}
+                        lastModified={lastModified}
+                        targetDrive={targetDrive}
+                        fit="cover"
+                        probablyEncrypted={probablyEncrypted}
+                        avoidPayload={isVideo}
+                        onLoad={() => setSomeLoaded(true)}
+                      />
+                    ) : (
+                      <BoringFile
+                        odinId={odinId}
+                        targetDrive={getChannelDrive(channelId)}
+                        fileId={fileId}
+                        file={file}
+                      />
+                    )}
+
+                    {index === maxVisible - 1 && countExcludedFromView > 0 ? (
+                      <div className="absolute inset-0 flex flex-col justify-center bg-black bg-opacity-40 text-6xl font-light text-white">
+                        <span className="block text-center">+{countExcludedFromView}</span>
+                      </div>
+                    ) : isVideo ? (
+                      <div className="absolute inset-0 flex items-center justify-center">
+                        <div className="bg-background/40 rounded-full p-7 border border-foreground/20">
+                          <Triangle className="text-foreground h-12 w-12" />
+                        </div>
                       </div>
-                    </div>
-                  ) : null}
+                    ) : null}
+                  </div>
                 </div>
-              </div>
-            ))}
+              );
+            })}
           </div>
         ) : null}
       </div>

diff --git a/packages/common-app/src/socialFeed/Blocks/Media/PrimaryMedia.tsx b/packages/common-app/src/socialFeed/Blocks/Media/PrimaryMedia.tsx
@@ -39,7 +39,9 @@ export const PrimaryMedia = ({
     onClick && onClick(e);
   };
 
-  const isVideo = file.contentType?.startsWith('video');
+  const isVideo =
+    file.contentType?.startsWith('video') ||
+    file.contentType?.startsWith('application/vnd.apple.mpegurl');
   // const isAudio = file.contentType?.startsWith('audio');
   const isImage = file.contentType?.startsWith('image');
   const isLink = file.key === POST_LINKS_PAYLOAD_KEY;

diff --git a/packages/common-app/src/socialFeed/PostDetail/PostImageDetailCard.tsx b/packages/common-app/src/socialFeed/PostDetail/PostImageDetailCard.tsx
@@ -137,7 +137,8 @@ export const PostImageDetailCard = ({
                   probablyEncrypted={postFile.fileMetadata.isEncrypted}
                   key={(postFile.fileId || '') + (currentMediaFile?.key || currIndex)}
                 />
-              ) : currentMediaFile?.contentType.startsWith('video') ? (
+              ) : currentMediaFile?.contentType.startsWith('video') ||
+                currentMediaFile?.contentType === 'application/vnd.apple.mpegurl' ? (
                 <div className="absolute inset-0 flex items-center justify-center">
                   <Video
                     fileId={postFile.fileId}

diff --git a/packages/community-app/package.json b/packages/community-app/package.json
@@ -27,6 +27,7 @@
     "@homebase-id/js-lib": "*",
     "@homebase-id/rich-text-editor": "*",
     "@homebase-id/ui-lib": "*",
+    "hls.js": "^1.5.15",
     "axios": "1.7.5",
     "emoji-picker-element": "^1.21.3",
     "idb-keyval": "^6.2.1",

diff --git a/packages/community-app/src/components/Community/Message/CommunityMedia.tsx b/packages/community-app/src/components/Community/Message/CommunityMedia.tsx
@@ -73,7 +73,9 @@ const MediaItem = ({
 }) => {
   const { isDarkMode } = useDarkMode();
   const dotYouClient = useDotYouClientContext();
-  const isVideo = payload.contentType?.startsWith('video');
+  const isVideo =
+    payload.contentType?.startsWith('video') ||
+    payload.contentType === 'application/vnd.apple.mpegurl';
   const isAudio = payload.contentType?.startsWith('audio');
   const isImage = payload.contentType?.startsWith('image');
   const isLink = payload.key === COMMUNITY_LINKS_PAYLOAD_KEY;

diff --git a/packages/community-app/src/components/Community/Message/detail/CommunityMediaGallery.tsx b/packages/community-app/src/components/Community/Message/detail/CommunityMediaGallery.tsx
@@ -92,7 +92,7 @@ export const CommunityMediaGallery = ({
       <div className="inset-0 bg-black transition-opacity lg:fixed"></div>
       <div className="inset-0 z-10 lg:fixed lg:overflow-y-auto">
         <div className="relative flex h-full min-h-[100dvh] flex-row items-center justify-center">
-          {contentType?.startsWith('video') ? (
+          {contentType?.startsWith('video') || contentType === 'application/vnd.apple.mpegurl' ? (
             <VideoClickToLoad
               preload={true}
               fileId={msg.fileId}

diff --git a/packages/community-app/src/providers/CommunityMessageProvider.ts b/packages/community-app/src/providers/CommunityMessageProvider.ts
@@ -36,6 +36,7 @@ import {
   jsonStringify64,
   stringToUint8Array,
   makeGrid,
+  getRandom16ByteArray,
 } from '@homebase-id/js-lib/helpers';
 import {
   LinkPreview,
@@ -132,6 +133,10 @@ export const uploadCommunityMessage = async (
   const payloads: PayloadFile[] = [];
   const thumbnails: ThumbnailFile[] = [];
   const previewThumbnails: EmbeddedThumb[] = [];
+  const keyHeader: KeyHeader = {
+    iv: getRandom16ByteArray(),
+    aesKey: getRandom16ByteArray(),
+  };
 
   if (!files?.length && linkPreviews?.length) {
     // We only support link previews when there is no media
@@ -159,13 +164,14 @@ export const uploadCommunityMessage = async (
     const payloadKey = `${COMMUNITY_MESSAGE_PAYLOAD_KEY}${i}`;
     const newMediaFile = files[i];
     if (newMediaFile.file.type.startsWith('video/')) {
-      const { tinyThumb, additionalThumbnails, payload } = await processVideoFile(
-        newMediaFile,
-        payloadKey
-      );
+      const {
+        tinyThumb,
+        thumbnails: thumbnailsFromVideo,
+        payloads: payloadsFromVideo,
+      } = await processVideoFile(newMediaFile, payloadKey, keyHeader);
 
-      thumbnails.push(...additionalThumbnails);
-      payloads.push(payload);
+      thumbnails.push(...thumbnailsFromVideo);
+      payloads.push(...payloadsFromVideo);
 
       if (tinyThumb) previewThumbnails.push(tinyThumb);
     } else if (newMediaFile.file.type.startsWith('image/')) {
@@ -202,7 +208,10 @@ export const uploadCommunityMessage = async (
     payloads,
     thumbnails,
     undefined,
-    onVersionConflict
+    onVersionConflict,
+    {
+      keyHeader,
+    }
   );
 
   if (!uploadResult) return null;

diff --git a/packages/feed-app/package.json b/packages/feed-app/package.json
@@ -30,6 +30,7 @@
     "emoji-picker-element": "^1.21.3",
     "lodash-es": "^4.17.21",
     "mp4box": "^0.5.2",
+    "hls.js": "^1.5.15",
     "react": "^18.3.1",
     "react-cropper": "^2.3.3",
     "react-dom": "^18.3.1",