feat: new edge function upload

src/utils/deploy/deploy-site.ts

@@ -15,6 +15,7 @@ import {
   DEFAULT_SYNC_LIMIT,
 } from './constants.js'
 import { hashConfig } from './hash-config.js'
+import hashEdgeFunctions from './hash-edge-functions.js'
 import hashFiles from './hash-files.js'
 import hashFns from './hash-fns.js'
 import {
@@ -104,6 +105,7 @@ export const deploySite = async (
     { files: staticFiles, filesShaMap: staticShaMap },
     { fnConfig, fnShaMap, functionSchedules, functions, functionsWithNativeModules },
     configFile,
+    { edgeFunctions, edgeFnShaMap },
   ] = await Promise.all([
     hashFiles({
       assetType,
@@ -125,6 +127,7 @@ export const deploySite = async (
       rootDir: siteRoot,
     }),
     hashConfig({ config }),
+    hashEdgeFunctions(edgeFunctionsDistPath, { hashAlgorithm, statusCb }),
   ])
 
   const files = { ...staticFiles, [configFile.normalizedPath]: configFile.hash }
@@ -181,6 +184,7 @@ For more information, visit https://ntl.fyi/cli-native-modules.`)
     body: {
       files,
       functions,
+      edge_functions: edgeFunctions,
       function_schedules: functionSchedules,
       functions_config: fnConfig,
       async: Object.keys(files).length > syncFileLimit,
@@ -195,19 +199,20 @@ For more information, visit https://ntl.fyi/cli-native-modules.`)
 
   if (deployParams.body.async) deploy = await waitForDiff(api, deploy.id, siteId, deployTimeout)
 
-  const { required: requiredFiles, required_functions: requiredFns } = deploy
+  const { required: requiredFiles, required_functions: requiredFns, required_edge_functions: requiredEdgeFns } = deploy
 
   statusCb({
     type: 'create-deploy',
     msg: `CDN requesting ${requiredFiles.length} files${
       Array.isArray(requiredFns) ? ` and ${requiredFns.length} functions` : ''
-    }`,
+    }${Array.isArray(requiredEdgeFns) ? ` and ${requiredEdgeFns.length} edge functions` : ''}`,
     phase: 'stop',
   })
 
   const filesUploadList = getUploadList(requiredFiles, filesShaMap)
   const functionsUploadList = getUploadList(requiredFns, fnShaMap)
-  const uploadList = [...filesUploadList, ...functionsUploadList]
+  const edgeFunctionsUploadList = getUploadList(requiredEdgeFns, edgeFnShaMap)
+  const uploadList = [...filesUploadList, ...functionsUploadList, ...edgeFunctionsUploadList]
 
   await uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb, maxRetry })
 

src/utils/deploy/hash-edge-functions.ts

@@ -0,0 +1,73 @@
+import { createHash } from 'node:crypto'
+import { createReadStream } from 'node:fs'
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { pipeline } from 'node:stream/promises'
+
+import type { Manifest } from '@netlify/edge-bundler'
+
+import type { StatusCallback } from './status-cb.js'
+import type { EdgeFunctionUploadFile } from './upload-files.js'
+
+const hashBundle = async (filepath: string, hashAlgorithm: string): Promise<string> => {
+  const hasher = createHash(hashAlgorithm)
+  await pipeline([createReadStream(filepath), hasher])
+  return hasher.digest('hex')
+}
+
+// Reads the edge-bundler manifest from the dist directory and, for every bundle, computes its
+// `code_sha` (sha256 of the bundle bytes — the deploy identity, recomputed rather than trusting the
+// bundler's asset filename) so we can both declare it on deploy create and stream it on upload. We
+// declare every format; bitballoon decides which ones actually ride this path and returns them in
+// `required_edge_functions`.
+const hashEdgeFunctions = async (
+  edgeFunctionsDistPath: string | undefined,
+  { hashAlgorithm = 'sha256', statusCb }: { hashAlgorithm?: string; statusCb: StatusCallback },
+): Promise<{
+  // edge_functions: { format => code_sha } sent on deploy create
+  edgeFunctions: Record<string, string>
+  // code_sha => [fileObj] consumed by the upload arm
+  edgeFnShaMap: Record<string, EdgeFunctionUploadFile[]>
+}> => {
+  const edgeFunctions: Record<string, string> = {}
+  const edgeFnShaMap: Record<string, EdgeFunctionUploadFile[]> = {}
+
+  if (!edgeFunctionsDistPath) {
+    return { edgeFunctions, edgeFnShaMap }
+  }
+
+  // `Partial` because this is whatever happens to be on disk, not something we produced.
+  let manifest: Partial<Manifest>
+  try {
+    manifest = JSON.parse(await readFile(join(edgeFunctionsDistPath, 'manifest.json'), 'utf8')) as Partial<Manifest>
+  } catch {
+    // No manifest (or an unreadable one) means there are no edge functions to declare.
+    return { edgeFunctions, edgeFnShaMap }
+  }
+
+  const bundles = manifest.bundles ?? []
+  for (const bundle of bundles) {
+    const filepath = join(edgeFunctionsDistPath, bundle.asset)
+    const codeSha = await hashBundle(filepath, hashAlgorithm)
+
+    edgeFunctions[bundle.format] = codeSha
+
+    const fileObj: EdgeFunctionUploadFile = {
+      assetType: 'edge-function',
+      filepath,
+      normalizedPath: codeSha,
+      hash: codeSha,
+    }
+    if (Array.isArray(edgeFnShaMap[codeSha])) {
+      edgeFnShaMap[codeSha].push(fileObj)
+    } else {
+      edgeFnShaMap[codeSha] = [fileObj]
+    }
+
+    statusCb({ type: 'hashing', msg: `Hashing edge function bundle ${bundle.asset}`, phase: 'progress' })
+  }
+
+  return { edgeFunctions, edgeFnShaMap }
+}
+
+export default hashEdgeFunctions

src/utils/deploy/upload-files.ts

@@ -1,22 +1,72 @@
 import fs from 'fs'
 
+import type { NetlifyAPI } from '@netlify/api'
 import backoff from 'backoff'
 import pMap from 'p-map'
 
 import { UPLOAD_INITIAL_DELAY, UPLOAD_MAX_DELAY, UPLOAD_RANDOM_FACTOR } from './constants.js'
+import type { StatusCallback } from './status-cb.js'
 
-// @ts-expect-error TS(7006) FIXME: Parameter 'api' implicitly has an 'any' type.
-const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRetry, statusCb }) => {
-  if (!concurrentUpload || !statusCb || !maxRetry) throw new Error('Missing required option concurrentUpload')
+export type UploadApi = Pick<NetlifyAPI, 'uploadDeployFile' | 'uploadDeployFunction' | 'uploadDeployEdgeFunction'>
+
+// `@netlify/api` only models path and query parameters, so header parameters such as
+// `X-Nf-Retry-Count` have to be added on top of the generated parameter types.
+type WithRetryCount<T> = T & { xNfRetryCount?: number }
+
+type UploadDeployFunctionParams = WithRetryCount<Parameters<UploadApi['uploadDeployFunction']>[0]>
+type UploadDeployEdgeFunctionParams = WithRetryCount<Parameters<UploadApi['uploadDeployEdgeFunction']>[0]>
+
+interface UploadFileBase {
+  filepath: string
+  normalizedPath: string
+  body?: fs.ReadStream
+}
+
+export interface StaticUploadFile extends UploadFileBase {
+  assetType: 'file'
+}
+
+export interface FunctionUploadFile extends UploadFileBase {
+  assetType: 'function'
+  runtime?: string
+  invocationMode?: string
+  timeout?: number
+}
+
+export interface EdgeFunctionUploadFile extends UploadFileBase {
+  assetType: 'edge-function'
+  hash: string
+}
+
+export type UploadFile = StaticUploadFile | FunctionUploadFile | EdgeFunctionUploadFile
+
+class MissingAssetTypeError extends Error {
+  constructor(readonly fileObj: unknown) {
+    super('File Object missing assetType property')
+  }
+}
+
+interface UploadFilesOptions {
+  concurrentUpload: number
+  maxRetry: number
+  statusCb: StatusCallback
+}
+
+const uploadFiles = async (
+  api: UploadApi,
+  deployId: string,
+  uploadList: UploadFile[],
+  { concurrentUpload, maxRetry, statusCb }: UploadFilesOptions,
+) => {
+  if (!concurrentUpload || !maxRetry) throw new Error('Missing required option concurrentUpload')
   statusCb({
     type: 'upload',
     msg: `Uploading ${uploadList.length} files`,
     phase: 'start',
   })
 
-  // @ts-expect-error TS(7006) FIXME: Parameter 'fileObj' implicitly has an 'any' type.
-  const uploadFile = async (fileObj, index) => {
-    const { assetType, body, filepath, invocationMode, normalizedPath, runtime, timeout } = fileObj
+  const uploadFile = async (fileObj: UploadFile, index: number) => {
+    const { body, filepath, normalizedPath } = fileObj
 
     const readStreamCtor = () => body ?? fs.createReadStream(filepath)
 
@@ -25,10 +75,10 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet
       msg: `(${index}/${uploadList.length}) Uploading ${normalizedPath}...`,
       phase: 'progress',
     })
-    let response
-    switch (assetType) {
+
+    switch (fileObj.assetType) {
       case 'file': {
-        response = await retryUpload(
+        return await retryUpload(
           () =>
             api.uploadDeployFile({
               body: readStreamCtor,
@@ -37,12 +87,12 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet
             }),
           maxRetry,
         )
-        break
       }
       case 'function': {
-        // @ts-expect-error TS(7006) FIXME: Parameter 'retryCount' implicitly has an 'any' typ... Remove this comment to see the full error message
-        response = await retryUpload((retryCount) => {
-          const params = {
+        const { invocationMode, runtime, timeout } = fileObj
+
+        return await retryUpload((retryCount) => {
+          const params: UploadDeployFunctionParams = {
             body: readStreamCtor,
             deployId,
             invocationMode,
@@ -52,23 +102,31 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet
           }
 
           if (retryCount > 0) {
-            // @ts-expect-error TS(2339) FIXME: Property 'xNfRetryCount' does not exist on type '{... Remove this comment to see the full error message
             params.xNfRetryCount = retryCount
           }
 
           return api.uploadDeployFunction(params)
         }, maxRetry)
-        break
+      }
+      case 'edge-function': {
+        return await retryUpload((retryCount) => {
+          const params: UploadDeployEdgeFunctionParams = {
+            body: readStreamCtor,
+            deployId,
+            codeSha: normalizedPath,
+          }
+
+          if (retryCount > 0) {
+            params.xNfRetryCount = retryCount
+          }
+
+          return api.uploadDeployEdgeFunction(params)
+        }, maxRetry)
       }
       default: {
-        const error = new Error('File Object missing assetType property')
-        // @ts-expect-error TS(2339) FIXME: Property 'fileObj' does not exist on type 'Error'.
-        error.fileObj = fileObj
-        throw error
+        throw new MissingAssetTypeError(fileObj)
       }
     }
-
-    return response
   }
 
   const results = await pMap(uploadList, uploadFile, { concurrency: concurrentUpload })
@@ -80,11 +138,14 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet
   return results
 }
 
-// @ts-expect-error TS(7006) FIXME: Parameter 'uploadFn' implicitly has an 'any' type.
-const retryUpload = (uploadFn, maxRetry) =>
-  new Promise((resolve, reject) => {
-    // @ts-expect-error TS(7034) FIXME: Variable 'lastError' implicitly has type 'any' in ... Remove this comment to see the full error message
-    let lastError
+const getErrorStatus = (error: unknown): number | undefined =>
+  typeof error === 'object' && error !== null && 'status' in error && typeof error.status === 'number'
+    ? error.status
+    : undefined
+
+const retryUpload = <T>(uploadFn: (retryCount: number) => Promise<T>, maxRetry: number): Promise<T> =>
+  new Promise<T>((resolve, reject) => {
+    let lastError: unknown
 
     const fibonacciBackoff = backoff.fibonacci({
       randomisationFactor: UPLOAD_RANDOM_FACTOR,
@@ -101,16 +162,16 @@ const retryUpload = (uploadFn, maxRetry) =>
       } catch (error) {
         lastError = error
 
+        const status = getErrorStatus(error)
+
         // We don't need to retry for 400 or 422 errors
-        // @ts-expect-error TS(2571) FIXME: Object is of type 'unknown'.
-        if (error.status === 400 || error.status === 422) {
+        if (status === 400 || status === 422) {
           reject(error)
           return
         }
 
         // observed errors: 408, 401 (4** swallowed), 502
-        // @ts-expect-error TS(2571) FIXME: Object is of type 'unknown'.
-        if (error.status > 400 || error.name === 'FetchError') {
+        if ((status !== undefined && status > 400) || (error instanceof Error && error.name === 'FetchError')) {
           fibonacciBackoff.backoff()
           return
         }
@@ -130,7 +191,6 @@ const retryUpload = (uploadFn, maxRetry) =>
     fibonacciBackoff.on('ready', tryUpload)
 
     fibonacciBackoff.on('fail', () => {
-      // @ts-expect-error TS(7005) FIXME: Variable 'lastError' implicitly has an 'any' type.
       reject(lastError)
     })
 

tests/unit/utils/deploy/hash-edge-functions.test.ts

@@ -0,0 +1,57 @@
+import { createHash } from 'node:crypto'
+import { mkdir, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+
+import { expect, test } from 'vitest'
+
+import hashEdgeFunctions from '../../../../src/utils/deploy/hash-edge-functions.js'
+import { temporaryDirectory } from '../../../../src/utils/temporary-file.js'
+
+const sha256 = (contents: string) => createHash('sha256').update(contents).digest('hex')
+
+const writeManifest = async (dir: string, bundles: { asset: string; format: string; contents: string }[]) => {
+  await mkdir(dir, { recursive: true })
+  await Promise.all(bundles.map(({ asset, contents }) => writeFile(join(dir, asset), contents)))
+  await writeFile(
+    join(dir, 'manifest.json'),
+    JSON.stringify({ bundles: bundles.map(({ asset, format }) => ({ asset, format })) }),
+  )
+}
+
+test('declares every bundle format, keyed by the recomputed code_sha', async () => {
+  const dir = temporaryDirectory()
+  await writeManifest(dir, [
+    { asset: 'aaa.tar.gz', format: 'tar', contents: 'tar-bundle-bytes' },
+    { asset: 'bbb.eszip', format: 'eszip2', contents: 'eszip-bundle-bytes' },
+  ])
+
+  const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(dir, { statusCb() {} })
+
+  const tarSha = sha256('tar-bundle-bytes')
+  const eszipSha = sha256('eszip-bundle-bytes')
+  // We declare all formats; bitballoon filters and only asks for the ones that ride this path.
+  expect(edgeFunctions).toEqual({ tar: tarSha, eszip2: eszipSha })
+  expect(Object.keys(edgeFnShaMap).sort()).toEqual([tarSha, eszipSha].sort())
+  expect(edgeFnShaMap[tarSha][0]).toMatchObject({
+    assetType: 'edge-function',
+    filepath: join(dir, 'aaa.tar.gz'),
+    normalizedPath: tarSha,
+  })
+})
+
+test('returns empty maps when there is no dist path', async () => {
+  const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(undefined, { statusCb() {} })
+
+  expect(edgeFunctions).toEqual({})
+  expect(edgeFnShaMap).toEqual({})
+})
+
+test('returns empty maps when the manifest is missing', async () => {
+  const dir = temporaryDirectory()
+  await mkdir(dir, { recursive: true })
+
+  const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(dir, { statusCb() {} })
+
+  expect(edgeFunctions).toEqual({})
+  expect(edgeFnShaMap).toEqual({})
+})