feat: add ability to create cpio archives using the cli

packages/zip-it-and-ship-it/package.json

@@ -49,6 +49,7 @@
     "archiver": "^7.0.0",
     "common-path-prefix": "^3.0.0",
     "cp-file": "^10.0.0",
+    "cpio-stream": "^1.4.3",
     "es-module-lexer": "^1.0.0",
     "esbuild": "0.19.11",
     "execa": "^7.0.0",

packages/zip-it-and-ship-it/src/archive.ts

@@ -3,18 +3,75 @@ import { createWriteStream, Stats, readlinkSync } from 'fs'
 import { Writable } from 'stream'
 
 import archiver, { Archiver } from 'archiver'
+import cpio from 'cpio-stream'
 
 import { ObjectValues } from './types/utils.js'
 
 export { Archiver as ZipArchive } from 'archiver'
 
 export const ARCHIVE_FORMAT = {
+  CPIO: 'cpio',
   NONE: 'none',
   ZIP: 'zip',
 } as const
 
 export type ArchiveFormat = ObjectValues<typeof ARCHIVE_FORMAT>
 
+// Start cpio archive for files
+export const startCpio = function (destPath: string): { archive: any; output: Writable } {
+  const output = createWriteStream(destPath)
+  const archive = cpio.pack()
+
+  archive.pipe(output)
+
+  return { archive, output }
+}
+
+// Add new file to cpio
+export const addCpioFile = function (
+  archive: any,
+  file: string,
+  name: string,
+  stat?: Stats,
+  content?: Buffer | string,
+): void {
+  if (stat?.isSymbolicLink()) {
+    const linkContent = readlinkSync(file)
+
+    archive.entry({ name: name, linkname: linkContent, mode: stat.mode, type: 'symlink' })
+  } else {
+    archive.entry(
+      {
+        type: 'file',
+        name,
+        mode: stat?.mode,
+        // Ensure sha256 stability regardless of mtime
+        mtime: new Date(0),
+        dev: stat?.dev,
+        ino: stat?.ino,
+        nlink: stat?.nlink,
+        uid: stat?.uid,
+        gid: stat?.gid,
+        rdev: stat?.rdev,
+        size: stat?.size,
+      },
+      content,
+    )
+  }
+}
+
+// End cpioing files
+export const endCpio = async function (archive: any, output: Writable): Promise<void> {
+  const result = new Promise<void>((resolve, reject) => {
+    output.on('error', (error) => reject(error))
+    output.on('finish', () => resolve())
+  })
+
+  archive.finalize()
+
+  return result
+}
+
 // Start zipping files
 export const startZip = function (destPath: string): { archive: Archiver; output: Writable } {
   const output = createWriteStream(destPath)

packages/zip-it-and-ship-it/src/bin.ts

@@ -6,7 +6,7 @@ import yargs from 'yargs'
 import { hideBin } from 'yargs/helpers'
 
 import { ARCHIVE_FORMAT } from './archive.js'
-import { zipFunctions } from './main.js'
+import { cpioFunctions, zipFunctions } from './main.js'
 
 const packJson = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'))
 
@@ -22,10 +22,15 @@ const runCli = async function () {
 
   try {
     global.ZISI_CLI = true
-    // @ts-expect-error TODO: `options` is not getting the right types.
-    const zipped = await zipFunctions(srcFolder, destFolder, options)
-
-    console.log(JSON.stringify(zipped, null, 2))
+    let archive
+    if (options['archive-format'] === 'cpio') {
+      // @ts-expect-error TODO: `options` is not getting the right types.
+      archive = await cpioFunctions(srcFolder, destFolder, options)
+    } else {
+      // @ts-expect-error TODO: `options` is not getting the right types.
+      archive = await zipFunctions(srcFolder, destFolder, options)
+    }
+    console.log(JSON.stringify(archive, null, 2))
   } catch (error) {
     console.error(error.toString())
     exit(1)

packages/zip-it-and-ship-it/src/config.ts

@@ -23,6 +23,7 @@ export const functionConfig = z.object({
   rustTargetDirectory: z.string().optional().catch(undefined),
   schedule: z.string().optional().catch(undefined),
   timeout: z.number().optional().catch(undefined),
+  cpioGo: z.boolean().optional().catch(undefined),
   zipGo: z.boolean().optional().catch(undefined),
 
   // Temporary configuration property, only meant to be used by the deploy

packages/zip-it-and-ship-it/src/cpio.ts

@@ -0,0 +1,205 @@
+import { promises as fs } from 'fs'
+import { resolve } from 'path'
+
+import isPathInside from 'is-path-inside'
+import pMap from 'p-map'
+
+import { ArchiveFormat, ARCHIVE_FORMAT } from './archive.js'
+import { Config } from './config.js'
+import { FeatureFlags, getFlags } from './feature_flags.js'
+import { FunctionSource } from './function.js'
+import { createManifest } from './manifest.js'
+import { getFunctionsFromPaths } from './runtimes/index.js'
+import { MODULE_FORMAT } from './runtimes/node/utils/module_format.js'
+import { addArchiveSize } from './utils/archive_size.js'
+import { RuntimeCache } from './utils/cache.js'
+import { formatCpioResult, FunctionResult } from './utils/format_result.js'
+import { listFunctionsDirectories, resolveFunctionsDirectories } from './utils/fs.js'
+import { getLogger, LogFunction } from './utils/logger.js'
+import { nonNullable } from './utils/non_nullable.js'
+
+export interface CpioFunctionOptions {
+  archiveFormat?: ArchiveFormat
+  basePath?: string
+  branch?: string
+  config?: Config
+  featureFlags?: FeatureFlags
+  repositoryRoot?: string
+  systemLog?: LogFunction
+  debug?: boolean
+  internalSrcFolder?: string
+}
+
+export type CpioFunctionsOptions = CpioFunctionOptions & {
+  configFileDirectories?: string[]
+  manifest?: string
+  parallelLimit?: number
+  internalSrcFolder?: string
+}
+
+const DEFAULT_PARALLEL_LIMIT = 5
+
+const validateArchiveFormat = (archiveFormat: ArchiveFormat) => {
+  if (archiveFormat !== ARCHIVE_FORMAT.CPIO && archiveFormat !== ARCHIVE_FORMAT.NONE) {
+    throw new Error(`Invalid archive format: ${archiveFormat}`)
+  }
+}
+
+// Create a CPIO archive of `srcFolder/*` (Node.js or Go files) and write it to `destFolder/*.cpio`
+export const cpioFunctions = async function (
+  relativeSrcFolders: string | string[],
+  destFolder: string,
+  {
+    archiveFormat = ARCHIVE_FORMAT.CPIO,
+    basePath,
+    branch,
+    config = {},
+    configFileDirectories,
+    featureFlags: inputFeatureFlags,
+    manifest,
+    parallelLimit = DEFAULT_PARALLEL_LIMIT,
+    repositoryRoot = basePath,
+    systemLog,
+    debug,
+    internalSrcFolder,
+  }: CpioFunctionsOptions = {},
+): Promise<FunctionResult[]> {
+  validateArchiveFormat(archiveFormat)
+
+  const logger = getLogger(systemLog, debug)
+  const cache = new RuntimeCache()
+  const featureFlags = getFlags(inputFeatureFlags)
+  const srcFolders = resolveFunctionsDirectories(relativeSrcFolders)
+  const internalFunctionsPath = internalSrcFolder && resolve(internalSrcFolder)
+
+  const [paths] = await Promise.all([listFunctionsDirectories(srcFolders), fs.mkdir(destFolder, { recursive: true })])
+  const functions = await getFunctionsFromPaths(paths, {
+    cache,
+    config,
+    configFileDirectories,
+    dedupe: true,
+    featureFlags,
+  })
+  const results = await pMap(
+    functions.values(),
+    async (func) => {
+      const functionFlags = {
+        ...featureFlags,
+
+        // If there's a `nodeModuleFormat` configuration property set to `esm`,
+        // extend the feature flags with `zisi_pure_esm_mjs` enabled.
+        ...(func.config.nodeModuleFormat === MODULE_FORMAT.ESM ? { zisi_pure_esm_mjs: true } : {}),
+      }
+
+      const cpioResult = await func.runtime.cpioFunction({
+        archiveFormat,
+        basePath,
+        branch,
+        cache,
+        config: func.config,
+        destFolder,
+        extension: func.extension,
+        featureFlags: functionFlags,
+        filename: func.filename,
+        isInternal: Boolean(internalFunctionsPath && isPathInside(func.srcPath, internalFunctionsPath)),
+        logger,
+        mainFile: func.mainFile,
+        name: func.name,
+        repositoryRoot,
+        runtime: func.runtime,
+        srcDir: func.srcDir,
+        srcPath: func.srcPath,
+        stat: func.stat,
+      })
+
+      return { ...cpioResult, mainFile: func.mainFile, name: func.name, runtime: func.runtime }
+    },
+    {
+      concurrency: parallelLimit,
+    },
+  )
+  const formattedResults = await Promise.all(
+    results.filter(nonNullable).map(async (result) => {
+      const resultWithSize = await addArchiveSize(result)
+
+      return formatCpioResult(resultWithSize)
+    }),
+  )
+
+  if (manifest !== undefined) {
+    await createManifest({ functions: formattedResults, path: resolve(manifest) })
+  }
+
+  return formattedResults
+}
+
+export const cpioFunction = async function (
+  relativeSrcPath: string,
+  destFolder: string,
+  {
+    archiveFormat = ARCHIVE_FORMAT.CPIO,
+    basePath,
+    config: inputConfig = {},
+    featureFlags: inputFeatureFlags,
+    repositoryRoot = basePath,
+    systemLog,
+    debug,
+    internalSrcFolder,
+  }: CpioFunctionOptions = {},
+): Promise<FunctionResult | undefined> {
+  validateArchiveFormat(archiveFormat)
+
+  const logger = getLogger(systemLog, debug)
+  const featureFlags = getFlags(inputFeatureFlags)
+  const srcPath = resolve(relativeSrcPath)
+  const cache = new RuntimeCache()
+  const functions = await getFunctionsFromPaths([srcPath], { cache, config: inputConfig, dedupe: true, featureFlags })
+  const internalFunctionsPath = internalSrcFolder && resolve(internalSrcFolder)
+
+  if (functions.size === 0) {
+    return
+  }
+
+  const {
+    config,
+    extension,
+    filename,
+    mainFile,
+    name,
+    runtime,
+    srcDir,
+    stat: stats,
+  }: // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  FunctionSource = functions.values().next().value!
+
+  await fs.mkdir(destFolder, { recursive: true })
+
+  const functionFlags = {
+    ...featureFlags,
+
+    // If there's a `nodeModuleFormat` configuration property set to `esm`,
+    // extend the feature flags with `zisi_pure_esm_mjs` enabled.
+    ...(config.nodeModuleFormat === MODULE_FORMAT.ESM ? { zisi_pure_esm_mjs: true } : {}),
+  }
+  const cpioResult = await runtime.cpioFunction({
+    archiveFormat,
+    basePath,
+    cache,
+    config,
+    destFolder,
+    extension,
+    featureFlags: functionFlags,
+    filename,
+    isInternal: Boolean(internalFunctionsPath && isPathInside(srcPath, internalFunctionsPath)),
+    logger,
+    mainFile,
+    name,
+    repositoryRoot,
+    runtime,
+    srcDir,
+    srcPath,
+    stat: stats,
+  })
+
+  return formatCpioResult({ ...cpioResult, mainFile, name, runtime })
+}

packages/zip-it-and-ship-it/src/cpio_binary.ts

@@ -0,0 +1,26 @@
+import type { Stats } from 'fs'
+import { readFile } from 'fs/promises'
+
+import { startCpio, addCpioFile, endCpio } from './archive.js'
+import { Runtime } from './runtimes/runtime.js'
+
+// Create CPIO for a binary function file
+export const cpioBinary = async function ({
+  destPath,
+  filename,
+  runtime,
+  srcPath,
+  stat,
+}: {
+  destPath: string
+  filename: string
+  runtime: Runtime
+  srcPath: string
+  stat: Stats
+}) {
+  const { archive, output } = startCpio(destPath)
+
+  addCpioFile(archive, srcPath, filename, stat, await readFile(srcPath))
+  addCpioFile(archive, 'netlify-toolchain', 'netlify-toolchain', undefined, JSON.stringify({ runtime: runtime.name }))
+  await endCpio(archive, output)
+}

packages/zip-it-and-ship-it/src/main.ts

@@ -9,8 +9,10 @@ import { ModuleFormat } from './runtimes/node/utils/module_format.js'
 import { GetSrcFilesFunction, RuntimeName, RUNTIME } from './runtimes/runtime.js'
 import { RuntimeCache } from './utils/cache.js'
 import { listFunctionsDirectories, resolveFunctionsDirectories } from './utils/fs.js'
+export { cpioFunction, cpioFunctions } from './cpio.js'
 
 export { Config, FunctionConfig } from './config.js'
+
 export { zipFunction, zipFunctions, ZipFunctionOptions, ZipFunctionsOptions } from './zip.js'
 
 export { ArchiveFormat, ARCHIVE_FORMAT } from './archive.js'