GitDirectoryResource (#1858)

Related to
#1787, Follows
up on #1793

Implements GitDirectoryResource to enable loading files directly from
git repositories as follows:

```ts
{
	"landingPage": "/guides/for-plugin-developers.md",
	"steps": [
		{
			"step": "writeFiles",
			"writeToPath": "/wordpress/guides",
			"filesTree": {
				"resource": "git:directory",
				"url": "https://github.com/WordPress/wordpress-playground.git",
				"ref": "trunk",
				"path": "packages/docs/site/docs/main/guides"
			}
		}
	]
}
```

 ## Implementation details

Uses git client functions merged in
#1764 to sparse
checkout the requested files. It also leans on the PHP CORS proxy which
is now started as a part of the `npm run dev` command.

The CORS proxy URL is configurable per `compileBlueprint()` call so that
each Playground runtime may choose to either use it or not. For example,
it wouldn't be very useful in the CLI version of Playground.

 ## Testing plan

Go to

```
http://localhost:5400/website-server/#{%20%22landingPage%22:%20%22/guides/for-plugin-developers.md%22,%20%22steps%22:%20[%20{%20%22step%22:%20%22writeFiles%22,%20%22writeToPath%22:%20%22/wordpress/guides%22,%20%22filesTree%22:%20{%20%22resource%22:%20%22git:directory%22,%20%22url%22:%20%22https://github.com/WordPress/wordpress-playground.git%22,%20%22ref%22:%20%22trunk%22,%20%22path%22:%20%22packages/docs/site/docs/main/guides%22%20}%20}%20]%20}
```

And confirm the Playground loads a markdown file.

packages/playground/blueprints/src/lib/compile.ts

@@ -68,6 +68,14 @@ export interface CompileBlueprintOptions {
 	semaphore?: Semaphore;
 	/** Optional callback with step output */
 	onStepCompleted?: OnStepCompleted;
+	/**
+	 * Proxy URL to use for cross-origin requests.
+	 *
+	 * For example, if corsProxy is set to "https://cors.wordpress.net/proxy.php",
+	 * then the CORS requests to https://github.com/WordPress/gutenberg.git would actually
+	 * be made to https://cors.wordpress.net/proxy.php/https://github.com/WordPress/gutenberg.git.
+	 */
+	corsProxy?: string;
 }
 
 /**
@@ -84,6 +92,7 @@ export function compileBlueprint(
 		progress = new ProgressTracker(),
 		semaphore = new Semaphore({ concurrency: 3 }),
 		onStepCompleted = () => {},
+		corsProxy,
 	}: CompileBlueprintOptions = {}
 ): CompiledBlueprint {
 	// Deep clone the blueprint to avoid mutating the input
@@ -294,6 +303,7 @@ export function compileBlueprint(
 			semaphore,
 			rootProgressTracker: progress,
 			totalProgressWeight,
+			corsProxy,
 		})
 	);
 
@@ -481,6 +491,12 @@ interface CompileStepArgsOptions {
 	rootProgressTracker: ProgressTracker;
 	/** The total progress weight of all the steps in the blueprint */
 	totalProgressWeight: number;
+	/**
+	 * Proxy URL to use for cross-origin requests.
+	 *
+	 * @see CompileBlueprintOptions.corsProxy
+	 */
+	corsProxy?: string;
 }
 
 /**
@@ -497,6 +513,7 @@ function compileStep<S extends StepDefinition>(
 		semaphore,
 		rootProgressTracker,
 		totalProgressWeight,
+		corsProxy,
 	}: CompileStepArgsOptions
 ): { run: CompiledStep; step: S; resources: Array<Resource<any>> } {
 	const stepProgress = rootProgressTracker.stage(
@@ -509,6 +526,7 @@ function compileStep<S extends StepDefinition>(
 		if (isResourceReference(value)) {
 			value = Resource.create(value, {
 				semaphore,
+				corsProxy,
 			});
 		}
 		args[key] = value;

packages/playground/blueprints/src/lib/resources.ts

@@ -3,7 +3,12 @@ import {
 	ProgressTracker,
 } from '@php-wasm/progress';
 import { FileTree, UniversalPHP } from '@php-wasm/universal';
-import { Semaphore } from '@php-wasm/util';
+import { dirname, Semaphore } from '@php-wasm/util';
+import {
+	listDescendantFiles,
+	listGitFiles,
+	sparseCheckout,
+} from '@wp-playground/storage';
 import { zipNameToHumanName } from './utils/zip-name-to-human-name';
 
 export type { FileTree };
@@ -129,10 +134,12 @@ export abstract class Resource<T extends File | Directory> {
 		{
 			semaphore,
 			progress,
+			corsProxy,
 		}: {
 			/** Optional semaphore to limit concurrent downloads */
 			semaphore?: Semaphore;
 			progress?: ProgressTracker;
+			corsProxy?: string;
 		}
 	): Resource<File | Directory> {
 		let resource: Resource<File | Directory>;
@@ -153,7 +160,9 @@ export abstract class Resource<T extends File | Directory> {
 				resource = new UrlResource(ref, progress);
 				break;
 			case 'git:directory':
-				resource = new GitDirectoryResource(ref, progress);
+				resource = new GitDirectoryResource(ref, progress, {
+					corsProxy,
+				});
 				break;
 			case 'literal:directory':
 				resource = new LiteralDirectoryResource(ref, progress);
@@ -442,26 +451,35 @@ export class UrlResource extends FetchResource {
 export class GitDirectoryResource extends Resource<Directory> {
 	constructor(
 		private reference: GitDirectoryReference,
-		public override _progress?: ProgressTracker
+		public override _progress?: ProgressTracker,
+		private options?: { corsProxy?: string }
 	) {
 		super();
 	}
 
 	async resolve() {
-		// @TODO: Use the actual sparse checkout logic here once
-		//        https://github.com/WordPress/wordpress-playground/pull/1764 lands.
-		throw new Error('Not implemented yet');
+		const repoUrl = this.options?.corsProxy
+			? `${this.options.corsProxy}/${this.reference.url}`
+			: this.reference.url;
+		const ref = `refs/heads/${this.reference.ref}`;
+		const allFiles = await listGitFiles(repoUrl, ref);
+		const filesToClone = listDescendantFiles(allFiles, this.reference.path);
+		let files = await sparseCheckout(repoUrl, ref, filesToClone);
+		// Remove the path prefix from the cloned file names.
+		files = Object.fromEntries(
+			Object.entries(files).map(([name, contents]) => {
+				name = name.substring(this.reference.path.length);
+				name = name.replace(/^\/+/, '');
+				return [name, contents];
+			})
+		);
 		return {
-			name: 'hello-world',
-			files: {
-				'README.md': 'Hello, World!',
-				'index.php': `<?php
-/**
-* Plugin Name: Hello World
-* Description: A simple plugin that says hello world.
-*/
-			 	`,
-			},
+			name:
+				dirname(this.reference.path) ||
+				this.reference.url
+					.replaceAll(/[^a-zA-Z0-9-.]/g, '-')
+					.replaceAll(/-+/g, '-'),
+			files,
 		};
 	}
 

packages/playground/client/src/index.ts

@@ -75,6 +75,18 @@ export interface StartPlaygroundOptions {
 	 * for more details.
 	 */
 	scope?: string;
+	/**
+	 * Proxy URL to use for cross-origin requests.
+	 *
+	 * For example, if corsProxy is set to "https://cors.wordpress.net/proxy.php",
+	 * then the CORS requests to https://github.com/WordPress/wordpress-playground.git would actually
+	 * be made to https://cors.wordpress.net/proxy.php/https://github.com/WordPress/wordpress-playground.git.
+	 *
+	 * The Blueprints library will arbitrarily choose which requests to proxy. If you need
+	 * to proxy every single request, do not use this option. Instead, you should preprocess
+	 * your Blueprint to replace all cross-origin URLs with the proxy URL.
+	 */
+	corsProxy?: string;
 }
 
 /**
@@ -96,6 +108,7 @@ export async function startPlaygroundWeb({
 	onBeforeBlueprint,
 	mounts,
 	scope,
+	corsProxy,
 	shouldInstallWordPress,
 }: StartPlaygroundOptions): Promise<PlaygroundClient> {
 	assertValidRemote(remoteUrl);
@@ -116,6 +129,7 @@ export async function startPlaygroundWeb({
 	const compiled = compileBlueprint(blueprint, {
 		progress: progressTracker.stage(0.5),
 		onStepCompleted: onBlueprintStepCompleted,
+		corsProxy,
 	});
 
 	await new Promise((resolve) => {

packages/playground/components/src/demos/GitBrowserDemo.tsx

@@ -2,7 +2,7 @@ import React, { useEffect, useMemo } from 'react';
 import {
 	FileTree,
 	listDescendantFiles,
-	listFiles,
+	listGitFiles,
 	sparseCheckout,
 } from '@wp-playground/storage';
 import {
@@ -28,7 +28,7 @@ export default function GitBrowserDemo() {
 		() => Promise.resolve([])
 	);
 	const loadFiles = () => {
-		const promise = listFiles(repoUrl, branch);
+		const promise = listGitFiles(repoUrl, branch);
 		setFilesPromise(promise);
 	};
 	const files = usePromise(filesPromise);

packages/playground/storage/src/lib/git-sparse-checkout.spec.ts

@@ -1,8 +1,12 @@
-import { listRefs, sparseCheckout, listFiles } from './git-sparse-checkout';
+import {
+	listGitRefs,
+	sparseCheckout,
+	listGitFiles,
+} from './git-sparse-checkout';
 
 describe('listRefs', () => {
 	it('should return the latest commit hash for a given ref', async () => {
-		const refs = await listRefs(
+		const refs = await listGitRefs(
 			'https://github.com/WordPress/wordpress-playground',
 			'refs/heads/trunk'
 		);
@@ -26,9 +30,9 @@ describe('sparseCheckout', () => {
 	});
 });
 
-describe('listFiles', () => {
+describe('listGitFiles', () => {
 	it('should list the files in a git repo', async () => {
-		const files = await listFiles(
+		const files = await listGitFiles(
 			'https://github.com/WordPress/wordpress-playground.git',
 			'refs/heads/trunk'
 		);

packages/playground/storage/src/lib/git-sparse-checkout.ts

@@ -42,7 +42,7 @@ export async function sparseCheckout(
 	fullyQualifiedBranchName: string,
 	filesPaths: string[]
 ) {
-	const refs = await listRefs(repoUrl, fullyQualifiedBranchName);
+	const refs = await listGitRefs(repoUrl, fullyQualifiedBranchName);
 	const commitHash = refs[fullyQualifiedBranchName];
 	const treesIdx = await fetchWithoutBlobs(repoUrl, commitHash);
 	const objects = await resolveObjects(treesIdx, commitHash, filesPaths);
@@ -84,11 +84,11 @@ export type FileTree = FileTreeFile | FileTreeFolder;
  * @param fullyQualifiedBranchName The full name of the branch to fetch from (e.g., 'refs/heads/main').
  * @returns A list of all files in the repository.
  */
-export async function listFiles(
+export async function listGitFiles(
 	repoUrl: string,
 	fullyQualifiedBranchName: string
 ): Promise<FileTree[]> {
-	const refs = await listRefs(repoUrl, fullyQualifiedBranchName);
+	const refs = await listGitRefs(repoUrl, fullyQualifiedBranchName);
 	if (!(fullyQualifiedBranchName in refs)) {
 		throw new Error(`Branch ${fullyQualifiedBranchName} not found`);
 	}
@@ -131,7 +131,7 @@ function gitTreeToFileTree(tree: GitTree): FileTree[] {
  * @param fullyQualifiedBranchPrefix The prefix of the refs to fetch. For example: refs/heads/my-feature-branch
  * @returns A map of refs to their corresponding commit hashes.
  */
-export async function listRefs(
+export async function listGitRefs(
 	repoUrl: string,
 	fullyQualifiedBranchPrefix: string
 ) {

packages/playground/website/project.json

@@ -50,6 +50,7 @@
 			"executor": "nx:run-commands",
 			"options": {
 				"commands": [
+					"nx run playground-php-cors-proxy:start",
 					"nx dev playground-remote --configuration=development-for-website",
 					"sleep 1; nx dev:standalone playground-website --hmr --output-style=stream-without-prefixes"
 				],

packages/playground/website/src/lib/state/redux/boot-site-client.ts

@@ -20,6 +20,8 @@ import { getRemoteUrl } from '../../config';
 import { setActiveModal, setActiveSiteError } from './slice-ui';
 import { PlaygroundDispatch, PlaygroundReduxState } from './store';
 import { selectSiteBySlug } from './slice-sites';
+// @ts-ignore
+import { corsProxyUrl } from 'virtual:cors-proxy-url';
 
 export function bootSiteClient(
 	siteSlug: string,
@@ -131,6 +133,7 @@ export function bootSiteClient(
 					  ]
 					: [],
 				shouldInstallWordPress: !isWordPressInstalled,
+				corsProxy: corsProxyUrl,
 			});
 
 			// @TODO: Remove backcompat code after 2024-12-01.

packages/playground/website/vite.config.ts

@@ -1,6 +1,6 @@
 /// <reference types="vitest" />
 import { defineConfig } from 'vite';
-import type { Plugin, ViteDevServer } from 'vite';
+import type { CommonServerOptions, Plugin, ViteDevServer } from 'vite';
 import react from '@vitejs/plugin-react';
 // eslint-disable-next-line @nx/enforce-module-boundaries
 import { viteTsConfigPaths } from '../../vite-extensions/vite-ts-config-paths';
@@ -21,8 +21,9 @@ import { join } from 'node:path';
 import { buildVersionPlugin } from '../../vite-extensions/vite-build-version';
 import { listAssetsRequiredForOfflineMode } from '../../vite-extensions/vite-list-assets-required-for-offline-mode';
 import { addManifestJson } from '../../vite-extensions/vite-manifest';
+import virtualModule from '../../vite-extensions/vite-virtual-module';
 
-const proxy = {
+const proxy: CommonServerOptions['proxy'] = {
 	'^/plugin-proxy': {
 		target: 'https://playground.wordpress.net',
 		changeOrigin: true,
@@ -77,6 +78,15 @@ export default defineConfig(({ command, mode }) => {
 			}),
 			ignoreWasmImports(),
 			buildVersionPlugin('website-config'),
+			virtualModule({
+				name: 'cors-proxy-url',
+				content: `
+				export const corsProxyUrl = '${
+					mode === 'production'
+						? '/cors-proxy.php'
+						: 'http://127.0.0.1:5263/cors-proxy.php'
+				}';`,
+			}),
 			// GitHub OAuth flow
 			{
 				name: 'configure-server',