feat: complete implementation

packages/edge-gateway/package.json

@@ -22,7 +22,7 @@
     "ipfs-core-utils": "^0.15.0",
     "ipfs-gateway-race": "link:../ipfs-gateway-race",
     "itty-router": "^2.4.5",
-    "multiformats": "^9.6.4",
+    "multiformats": "^13.3.0",
     "p-any": "^4.0.0",
     "p-defer": "^4.0.0",
     "p-retry": "^5.0.0",

packages/edge-gateway/src/gateway.js

@@ -1,5 +1,5 @@
 /* eslint-env serviceworker, browser */
-/* global Response caches */
+/* global Response caches, IdentityTransformStream */
 
 import pAny, { AggregateError } from 'p-any'
 import pDefer from 'p-defer'
@@ -9,7 +9,8 @@ import { gatewayFetch } from 'ipfs-gateway-race'
 
 import {
   getCidFromSubdomainUrl,
-  toDenyListAnchor
+  toDenyListAnchor,
+  getCidFromEtag
 } from './utils/cid.js'
 import { getHeaders } from './utils/headers.js'
 import { getCidForbiddenResponse } from './utils/verification.js'
@@ -41,7 +42,7 @@ import {
  */
 
 /**
- * Handle gateway request.
+ * Handle gateway GET request.
  *
  * @param {Request} request
  * @param {Env} env
@@ -133,7 +134,7 @@ export async function gatewayGet (request, env, ctx) {
   } = await getFromGatewayRacer(cid, pathname, search, getHeaders(request), env, ctx)
 
   // Validation layer - resource CID
-  const resourceCid = pathname !== '/' ? getCidFromEtag(winnerGwResponse.headers.get('etag') || cid) : cid
+  const resourceCid = pathname !== '/' ? getCidFromEtag(winnerGwResponse.headers.get('etag') || `"${cid}"`) : cid
   if (winnerGwResponse && pathname !== '/' && resourceCid) {
     const resourceCidForbiddenResponse = await getCidForbiddenResponse(resourceCid, env)
     if (resourceCidForbiddenResponse) {
@@ -206,7 +207,7 @@ async function getFromCdn (request, env, cache) {
 /**
  * @param {Request} request
  * @param {Env} env
- * @param {string} cid
+ * @param {import('multiformats').UnknownLink} cid
  * @param {{ pathname?: string, search?: string }} [options]
  * @return {Promise<ProxiedCDNResponse | undefined>}
  */
@@ -276,7 +277,7 @@ async function getFromDotstorage (request, env, cid, options = {}) {
 
 /**
  *
- * @param {string} cid
+ * @param {import('multiformats').UnknownLink} cid
  * @param {string} pathname
  * @param {string} search
  * @param {Headers} headers
@@ -446,28 +447,6 @@ function getResponseWithCustomHeaders (
   return clonedResponse
 }
 
-/**
- * Extracting resource CID from etag based on
- * https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#etag-response-header
- *
- * @param {string} etag
- */
-function getCidFromEtag (etag) {
-  let resourceCid = decodeURIComponent(etag)
-
-  // Handle weak etag
-  resourceCid = resourceCid.replace('W/', '')
-  resourceCid = resourceCid.replaceAll('"', '')
-
-  // Handle directory index generated
-  if (etag.includes('DirIndex')) {
-    const split = resourceCid.split('-')
-    resourceCid = split[split.length - 1]
-  }
-
-  return resourceCid
-}
-
 /**
  * Async metrics for race.
  *

packages/edge-gateway/src/utils/cid.js

@@ -1,6 +1,6 @@
 import { Multibases } from 'ipfs-core-utils/multibases'
 import { bases } from 'multiformats/basics'
-import { CID } from 'multiformats/cid'
+import { parse } from 'multiformats/link'
 import * as uint8arrays from 'uint8arrays'
 import { sha256 } from 'multiformats/hashes/sha2'
 
@@ -31,14 +31,13 @@ export async function getCidFromSubdomainUrl (url) {
 }
 
 /**
- * Parse CID and return normalized b32 v1.
+ * Parse CID and return normalized v1.
  *
  * @param {string} cid
  */
 export async function normalizeCid (cid) {
   const baseDecoder = await getMultibaseDecoder(cid)
-  const c = CID.parse(cid, baseDecoder)
-  return c.toV1().toString()
+  return parse(cid, baseDecoder).toV1()
 }
 
 /**
@@ -61,10 +60,32 @@ async function getMultibaseDecoder (cid) {
 /**
  * Get denylist anchor with badbits format.
  *
- * @param {string} cid
+ * @param {import('multiformats').UnknownLink} cid
  */
 export async function toDenyListAnchor (cid) {
   const multihash = await sha256.digest(uint8arrays.fromString(`${cid}/`))
   const digest = multihash.bytes.subarray(2)
   return uint8arrays.toString(digest, 'hex')
 }
+
+/**
+ * Extracting resource CID from etag based on
+ * https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#etag-response-header
+ *
+ * @param {string} etag
+ */
+export function getCidFromEtag (etag) {
+  let resourceCid = decodeURIComponent(etag)
+
+  // Handle weak etag
+  resourceCid = resourceCid.replace('W/', '')
+  resourceCid = resourceCid.replaceAll('"', '')
+
+  // Handle directory index generated
+  if (etag.includes('DirIndex')) {
+    const split = resourceCid.split('-')
+    resourceCid = split[split.length - 1]
+  }
+
+  return parse(resourceCid)
+}

packages/edge-gateway/src/utils/verification.js

@@ -8,7 +8,7 @@ import {
 /**
  * Checks to see if denylist or cid-verifier forbid this CID from being served.
  *
- * @param {string} cid
+ * @param {import('multiformats').UnknownLink} cid
  * @param {import('../env').Env} env
  */
 export async function getCidForbiddenResponse (cid, env) {

packages/ipfs-gateway-race/lib/index.js

@@ -5,7 +5,7 @@ import { FilterError } from 'p-some'
 import pSettle from 'p-settle'
 import fetch, { Headers } from '@web-std/fetch'
 import * as UnixFSDownloader from '@storacha/unixfs-dl'
-
+import * as raw from 'multiformats/codecs/raw'
 import {
   NotFoundError,
   GatewayTimeoutError,
@@ -40,7 +40,7 @@ export class IpfsGatewayRacer {
   }
 
   /**
-   * @param {string} cid
+   * @param {import('multiformats').UnknownLink} cid
    * @param {IpfsGatewayRaceGetOptions} [options]
    * @return {Promise<Response>}
    */
@@ -132,7 +132,7 @@ export function createGatewayRacer (ipfsGateways, options = {}) {
  * Fetches given CID from given IPFS gateway URL.
  *
  * @param {string} gwUrl
- * @param {string} cid
+ * @param {import('multiformats').UnknownLink} cid
  * @param {string} pathname
  * @param {Object} [options]
  * @param {string} [options.method]
@@ -162,19 +162,20 @@ export async function gatewayFetch (
 
   let response
   try {
-    // If this is an atypical request, i.e. it's a HEAD request, a range request
-    // or a request for a different format to UnixFS, then just make the request
-    // to the upstream as usual.
+    // If this is an atypical request, i.e. a HEAD request, a range request, a
+    // request for a different format to UnixFS, or if the root CID is for a raw
+    // block then just make the request to the upstream as usual.
     if (
       method !== 'GET' ||
+      cid.code === raw.code ||
       isRangeRequest(headers) ||
       isAlternateFormatRequest(headers, url.searchParams)
     ) {
       response = await fetch(url, { signal, headers })
     } else {
       // Otherwise use the unixfs downloader to make byte range requests
-      // upstream allowing big files to be downloaded without exhausting
-      // the upstream worker's CPU budget.
+      // upstream allowing big files to be downloaded without exhausting the
+      // upstream worker's CPU budget.
       response = await UnixFSDownloader.fetch(url, { signal, TransformStream: options.TransformStream })
     }
   } catch (error) {

packages/ipfs-gateway-race/package.json

@@ -30,10 +30,10 @@
     "gateway"
   ],
   "dependencies": {
-    "@storacha/unixfs-dl": "^1.3.0",
+    "@storacha/unixfs-dl": "^1.4.0",
     "@web-std/fetch": "^4.1.0",
     "any-signal": "^3.0.1",
-    "multiformats": "^9.6.4",
+    "multiformats": "^13.3.0",
     "p-any": "^4.0.0",
     "p-map": "^5.3.0",
     "p-retry": "^5.0.0",