feat(fs-aws): add credential config for public and requester pays (#1409

)

#### Motivation

Requester pays and public buckets are pretty common ways of accessing s3
locations, access needs to be configured so that a bucket can be
accessed

with either:

- "assume this role then use requester pays"
- "use the current role but as requester pays"

or as a public bucket with no credentials.

#### Modification

allow credentials to be defined as requester pays or public

packages/fs-aws/src/__tests__/credentials.test.ts

@@ -0,0 +1,37 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { AwsS3CredentialProvider, getPublicS3 } from '../credentials.js';
+import { AwsCredentialConfig } from '../types.js';
+
+describe('AwsS3CredentialProvider', () => {
+  const baseConfig: AwsCredentialConfig = { type: 's3', prefix: 's3://' };
+  it('should set requester pays', () => {
+    const creds = new AwsS3CredentialProvider();
+
+    const fs = creds.createFileSystem({ ...baseConfig, access: 'public' });
+    assert.ok(fs.s3 === getPublicS3());
+    assert.ok(fs.requestPayer === 'public');
+  });
+
+  it('should support requester pays', () => {
+    const creds = new AwsS3CredentialProvider();
+
+    const fs = creds.createFileSystem({ ...baseConfig, roleArn: 'arn:...', access: 'requesterPays' });
+    assert.ok(fs.s3 !== getPublicS3());
+    assert.ok(fs.requestPayer === 'requester');
+  });
+
+  it('should support requester pays from the current role', () => {
+    const creds = new AwsS3CredentialProvider();
+
+    const fs = creds.createFileSystem({ ...baseConfig, roleArn: undefined, access: 'requesterPays' });
+    assert.ok(fs.s3 !== getPublicS3());
+    assert.ok(fs.requestPayer === 'requester');
+  });
+
+  it('should require a roleArn', () => {
+    const creds = new AwsS3CredentialProvider();
+    assert.throws(() => creds.createFileSystem({ ...baseConfig }));
+  });
+});

packages/fs-aws/src/credentials.ts

@@ -1,8 +1,9 @@
 import { S3Client } from '@aws-sdk/client-s3';
 import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
 import { FileSystem, FileSystemProvider } from '@chunkd/fs';
-import { AwsCredentialConfig, AwsCredentialProvider } from './types.js';
+
 import { FsAwsS3 } from './fs.s3.js';
+import { AwsCredentialConfig, AwsCredentialProvider } from './types.js';
 
 export function isPromise<T>(t: AwsCredentialProvider | Promise<T>): t is Promise<T> {
   return 'then' in t && typeof t['then'] === 'function';
@@ -48,6 +49,14 @@ export class FsConfigFetcher {
   }
 }
 
+let PublicClient: S3Client | undefined;
+/** Creating a public s3 client is somewhat hard, where the signing method needs to be overriden */
+export function getPublicS3(): S3Client {
+  if (PublicClient) return PublicClient;
+  PublicClient = new S3Client({ signer: { sign: async (req) => req } });
+  return PublicClient;
+}
+
 export type AwsCredentialProviderLoader = () => Promise<AwsCredentialProvider>;
 export class AwsS3CredentialProvider implements FileSystemProvider<FsAwsS3> {
   /**
@@ -63,6 +72,22 @@ export class AwsS3CredentialProvider implements FileSystemProvider<FsAwsS3> {
 
   /** Given a config create a file system */
   createFileSystem(cs: AwsCredentialConfig): FsAwsS3 {
+    // Public access
+    if (cs.access === 'public') {
+      const fs = new FsAwsS3(getPublicS3());
+      fs.requestPayer = 'public';
+      return fs;
+    }
+
+    // Requester pays off the current credentials
+    if (cs.access === 'requesterPays' && cs.roleArn == null) {
+      const fs = new FsAwsS3(new S3Client({}));
+      if (cs.access === 'requesterPays') fs.requestPayer = 'requester';
+      return fs;
+    }
+
+    // All other credentials need a role assumed
+    if (cs.roleArn == null) throw new Error('No roleArn is defined for prefix: ' + cs.prefix);
     const client = new S3Client({
       credentials: fromTemporaryCredentials({
         params: {
@@ -74,7 +99,9 @@ export class AwsS3CredentialProvider implements FileSystemProvider<FsAwsS3> {
       }),
     });
 
-    return new FsAwsS3(client);
+    const fs = new FsAwsS3(client);
+    if (cs.access === 'requesterPays') fs.requestPayer = 'requester';
+    return fs;
   }
   /** Version for session name generally v2 or v3 for aws-sdk versions */
   version = 'v3';

packages/fs-aws/src/fs.s3.ts

@@ -1,3 +1,6 @@
+import type { Readable } from 'node:stream';
+import { PassThrough } from 'node:stream';
+
 import {
   DeleteObjectCommand,
   GetObjectCommand,
@@ -7,10 +10,9 @@ import {
   S3Client,
 } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
-import { FileInfo, FileSystem, FileSystemAction, FsError, ListOptions, WriteOptions, isRecord } from '@chunkd/fs';
+import { FileInfo, FileSystem, FileSystemAction, FsError, isRecord, ListOptions, WriteOptions } from '@chunkd/fs';
 import { SourceAwsS3 } from '@chunkd/source-aws';
-import type { Readable } from 'node:stream';
-import { PassThrough } from 'node:stream';
+
 import { AwsS3CredentialProvider } from './credentials.js';
 
 function isReadable(r: any): r is Readable {
@@ -49,7 +51,7 @@ export class FsAwsS3 implements FileSystem {
   writeTests = new Map<string, Promise<void | FsAwsS3>>();
 
   /** Request Payment option */
-  requestPayer?: 'requester';
+  requestPayer?: 'requester' | 'public';
 
   /**
    * When testing write permissions add a suffix to the file name, this file will be deleted up after writing completes

packages/fs-aws/src/types.ts

@@ -1,20 +1,53 @@
 export interface AwsCredentialConfig {
-  /** Prefix type generally s3 */
+  /**
+   * Prefix type generally s3
+   */
   type: 's3';
-  /** Prefix should always start with `s3://${bucket}` */
+
+  /**
+   * Location that these credentials are valid for,
+   *
+   * prefixes should always start with `s3://${bucket}/`
+   *
+   * @example
+   * ```typescript
+   * "s3://example/" // Matches all files in "s3://example/**"
+   * "s3://example"  // Matches all files in all buckets starting with "s3://example*"
+   * ```
+   */
   prefix: string;
-  /** Role to use to access */
-  roleArn: string;
-  /** Aws account this bucket belongs to */
+
+  /**
+   * Role to use to access
+   *
+   * roleArn is not required if access is "public"
+   */
+  roleArn?: string;
+
+  /**
+   * Aws account this bucket belongs to
+   */
   accountId?: string;
-  /** Bucket name */
-  bucket?: string;
-  /** ExternalId if required */
+
+  /**
+   * ExternalId if required
+   */
   externalId?: string;
-  /** Max role session duration */
+
+  /**
+   * Max role session duration
+   */
   roleSessionDuration?: number;
-  /** Can these credentials be used for "read" or "read-write" access */
+
+  /**
+   * Can these credentials be used for "read" or "read-write" access
+   */
   flags?: 'r' | 'rw';
+
+  /**
+   * Can this prefix be accessed without credentials or as requesterPays
+   */
+  access?: 'public' | 'requesterPays';
 }
 
 export interface AwsCredentialProvider {