diff --git a/src/assets.ts b/src/assets.ts new file mode 100644 index 0000000..7a9cca9 --- /dev/null +++ b/src/assets.ts @@ -0,0 +1,23 @@ +import * as fs from "fs-extra"; +import * as Path from "path"; +import { verbose } from "./log"; + +export enum AssetType { + Image = "image", + Video = "video", +} + +export function writeAsset(path: string, buffer: Buffer): void { + // Note: it's tempting to not spend time writing this out if we already have + // it from a previous run. But we don't really know it's the same. A) it + // could just have the same name, B) it could have been previously + // unlocalized and thus filled with a copy of the primary language image + // while and now is localized. + if (fs.pathExistsSync(path)) { + verbose("Replacing asset " + path); + } else { + verbose("Adding asset " + path); + fs.mkdirsSync(Path.dirname(path)); + } + fs.createWriteStream(path).write(buffer); // async but we're not waiting +} diff --git a/src/images.ts b/src/images.ts index 0150f5b..aa42b8a 100644 --- a/src/images.ts +++ b/src/images.ts @@ -1,7 +1,6 @@ import * as fs from "fs-extra"; import FileType, { FileTypeResult } from "file-type"; import axios from "axios"; -import * as Path from "path"; import { makeImagePersistencePlan } from "./MakeImagePersistencePlan"; import { warning, logDebug, verbose, info } from "./log"; import { ListBlockChildrenResponseResult } from "notion-to-md/build/types"; @@ -10,6 +9,7 @@ import { IDocuNotionContextPageInfo, IPlugin, } from "./plugins/pluginTypes"; +import { writeAsset } from "./assets"; // We handle several things here: // 1) copy images locally instead of leaving them in Notion @@ -158,11 +158,13 @@ async function readPrimaryImage(imageSet: ImageSet) { } async function saveImage(imageSet: ImageSet): Promise { - writeImageIfNew(imageSet.primaryFileOutputPath!, imageSet.primaryBuffer!); + const path = imageSet.primaryFileOutputPath!; + imageWasSeen(path); + writeAsset(path, imageSet.primaryBuffer!); for (const localizedImage of imageSet.localizedUrls) { let buffer = imageSet.primaryBuffer!; - // if we have a urls for the localized screenshot, download it + // if we have a url for the localized screenshot, download it if (localizedImage?.url.length > 0) { verbose(`Retrieving ${localizedImage.iso632Code} version...`); const response = await fetch(localizedImage.url); @@ -180,30 +182,15 @@ async function saveImage(imageSet: ImageSet): Promise { imageSet.pageInfo!.relativeFilePathToFolderContainingPage }`; - writeImageIfNew( - (directory + "/" + imageSet.outputFileName!).replaceAll("//", "/"), - buffer + const newPath = (directory + "/" + imageSet.outputFileName!).replaceAll( + "//", + "/" ); + imageWasSeen(newPath); + writeAsset(newPath, buffer); } } -function writeImageIfNew(path: string, buffer: Buffer) { - imageWasSeen(path); - - // Note: it's tempting to not spend time writing this out if we already have - // it from a previous run. But we don't really know it's the same. A) it - // could just have the same name, B) it could have been previously - // unlocalized and thus filled with a copy of the primary language image - // while and now is localized. - if (fs.pathExistsSync(path)) { - verbose("Replacing image " + path); - } else { - verbose("Adding image " + path); - fs.mkdirsSync(Path.dirname(path)); - } - fs.createWriteStream(path).write(buffer); // async but we're not waiting -} - export function parseImageBlock(image: any): ImageSet { if (!locales) throw Error("Did you call initImageHandling()?"); const imageSet: ImageSet = { diff --git a/src/plugins/VideoTransformer.spec.ts b/src/plugins/VideoTransformer.spec.ts index 0cc0271..e6bdc9c 100644 --- a/src/plugins/VideoTransformer.spec.ts +++ b/src/plugins/VideoTransformer.spec.ts @@ -1,7 +1,28 @@ +import * as fs from "fs-extra"; import { setLogLevel } from "../log"; import { NotionBlock } from "../types"; import { standardVideoTransformer } from "./VideoTransformer"; -import { blocksToMarkdown } from "./pluginTestRun"; +import { blocksToMarkdown, kTemporaryTestDirectory } from "./pluginTestRun"; + +beforeAll(async () => { + try { + if (await fs.pathExists(kTemporaryTestDirectory)) { + await fs.emptyDir(kTemporaryTestDirectory); + } else { + await fs.mkdirp(kTemporaryTestDirectory); + } + } catch (err) { + console.error("Error in beforeAll:", err); + } +}); + +afterAll(async () => { + try { + await fs.remove(kTemporaryTestDirectory); + } catch (err) { + console.error("Error in afterAll:", err); + } +}); test("youtube embedded", async () => { const config = { plugins: [standardVideoTransformer] }; @@ -89,6 +110,9 @@ test("video link, not embedded", async () => { test("direct upload to to Notion (embedded)", async () => { setLogLevel("verbose"); const config = { plugins: [standardVideoTransformer] }; + + const fileName1 = "first_video.mp4"; + const fileName2 = "second_video.mp4"; const result = await blocksToMarkdown(config, [ { object: "block", @@ -103,13 +127,39 @@ test("direct upload to to Notion (embedded)", async () => { caption: [], type: "file", file: { - url: "https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/people_fre_motionAsset_p3.mp4?X-Blah-blah", + url: `https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/${fileName1}?X-Blah-blah`, + }, + }, + } as unknown as NotionBlock, + { + object: "block", + id: "12f7db3b-4412-4be9-a3f7-6ac423fee94b", + parent: { + type: "page_id", + page_id: "edaffeb2-ece8-4d44-976f-351e6b5757bb", + }, + + type: "video", + video: { + caption: [], + type: "file", + file: { + url: `https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/${fileName2}?X-Blah-blah`, }, }, } as unknown as NotionBlock, ]); + expect(result).toContain(`import ReactPlayer from "react-player";`); - expect(result).toContain( - `` - ); + expect(result).toContain(`import video1 from "./${fileName1}";`); + expect(result).toContain(`import video2 from "./${fileName2}";`); + expect(result).toContain(``); + expect(result).toContain(``); + + // Wait half a second for the files to be written + await new Promise(resolve => setTimeout(resolve, 500)); + + // We should have actually created files in "tempTestFileDir/" + expect(await fs.pathExists("tempTestFileDir/" + fileName1)).toBe(true); + expect(await fs.pathExists("tempTestFileDir/" + fileName2)).toBe(true); }); diff --git a/src/plugins/VideoTransformer.ts b/src/plugins/VideoTransformer.ts index a07c17d..76909db 100644 --- a/src/plugins/VideoTransformer.ts +++ b/src/plugins/VideoTransformer.ts @@ -1,42 +1,96 @@ +import * as Path from "path"; import { VideoBlockObjectResponse } from "@notionhq/client/build/src/api-endpoints"; +import { ListBlockChildrenResponseResult } from "notion-to-md/build/types"; import { IDocuNotionContext, IPlugin } from "./pluginTypes"; import { warning } from "../log"; import { NotionBlock } from "../types"; +import { writeAsset } from "../assets"; export const standardVideoTransformer: IPlugin = { name: "video", notionToMarkdownTransforms: [ { type: "video", - getStringFromBlock: ( - context: IDocuNotionContext, - block: NotionBlock - ): string => { - const video = (block as VideoBlockObjectResponse).video; - let url = ""; - switch (video.type) { - case "external": - url = video.external.url; - break; - case "file": - url = video.file.url; - break; - default: - // video.type can only be "external" or "file" as of the writing of this code, so typescript - // isn't happy trying to turn video.type into a string. But this default in our switch is - // just attempting some future-proofing. Thus the strange typing/stringifying below. - warning( - `[standardVideoTransformer] Found Notion "video" block with type ${JSON.stringify( - (video as any).type - )}. The best docu-notion can do for now is ignore it.` - ); - return ""; - break; - } - - context.imports.push(`import ReactPlayer from "react-player";`); - return ``; - }, + getStringFromBlock: (context: IDocuNotionContext, block: NotionBlock) => + markdownToMDVideoTransformer(block, context), }, ], }; + +async function markdownToMDVideoTransformer( + block: ListBlockChildrenResponseResult, + context: IDocuNotionContext +): Promise { + const videoBlock = block as VideoBlockObjectResponse; + const video = videoBlock.video; + let url = ""; + switch (video.type) { + case "external": + url = `"${video.external.url}"`; + break; + case "file": + // The url we get for a Notion-hosted asset expires after an hour, so we have to download it locally. + url = await downloadVideoAndConvertUrl( + context, + video.file.url, + videoBlock.id + ); + break; + default: + // video.type can only be "external" or "file" as of the writing of this code, so typescript + // isn't happy trying to turn video.type into a string. But this default in our switch is + // just attempting some future-proofing. Thus the strange typing/stringifying below. + warning( + `[standardVideoTransformer] Found Notion "video" block with type ${JSON.stringify( + (video as any).type + )}. The best docu-notion can do for now is ignore it.` + ); + return ""; + } + + context.imports.push(`import ReactPlayer from "react-player";`); + return ``; +} + +// ENHANCE: One day, we may want to allow for options of where to place the files, how +// to name them, etc. Or we could at least follow the image options. +// But for now, I'm just trying to fix the bug that Notion-hosted videos don't work at all. +async function downloadVideoAndConvertUrl( + context: IDocuNotionContext, + notionVideoUrl: string, + blockId: string +): Promise { + // Get the file name from the url. Ignore query parameters and fragments. + let newFileName = notionVideoUrl.split("?")[0].split("#")[0].split("/").pop(); + + if (!newFileName) { + // If something went wrong, fall back to the block ID. + // But at least try to get the extension from the url. + const extension = notionVideoUrl + .split("?")[0] + .split("#")[0] + .split(".") + .pop(); + newFileName = blockId + (extension ? "." + extension : ""); + } + + const newPath = Path.posix.join( + context.pageInfo.directoryContainingMarkdown, + newFileName + ); + + const response = await fetch(notionVideoUrl); + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + writeAsset(newPath, buffer); + + // Add an import statement for the video file. + // Otherwise, the docusaurus build won't include the video file in the build. + const countVideoImports = context.imports.filter(i => { + return /import video\d+/.exec(i); + }).length; + const importName = `video${countVideoImports + 1}`; + context.imports.push(`import ${importName} from "./${newFileName}";`); + + return `{${importName}}`; +} diff --git a/src/plugins/pluginTestRun.ts b/src/plugins/pluginTestRun.ts index 959b249..95dea98 100644 --- a/src/plugins/pluginTestRun.ts +++ b/src/plugins/pluginTestRun.ts @@ -10,6 +10,8 @@ import { NotionBlock } from "../types"; import { convertInternalUrl } from "./internalLinks"; import { numberChildrenIfNumberedList } from "../pull"; +export const kTemporaryTestDirectory = "tempTestFileDir"; + export async function blocksToMarkdown( config: IDocuNotionConfig, blocks: NotionBlock[], @@ -49,7 +51,7 @@ export async function blocksToMarkdown( //TODO might be needed for some tests, e.g. the image transformer... pageInfo: { - directoryContainingMarkdown: "not yet", + directoryContainingMarkdown: kTemporaryTestDirectory, relativeFilePathToFolderContainingPage: "not yet", slug: "not yet", },