diff --git a/src/utils/deploy/deploy-site.ts b/src/utils/deploy/deploy-site.ts index 18efd1969b1..a28b333e65a 100644 --- a/src/utils/deploy/deploy-site.ts +++ b/src/utils/deploy/deploy-site.ts @@ -15,6 +15,7 @@ import { DEFAULT_SYNC_LIMIT, } from './constants.js' import { hashConfig } from './hash-config.js' +import hashEdgeFunctions from './hash-edge-functions.js' import hashFiles from './hash-files.js' import hashFns from './hash-fns.js' import { @@ -104,6 +105,7 @@ export const deploySite = async ( { files: staticFiles, filesShaMap: staticShaMap }, { fnConfig, fnShaMap, functionSchedules, functions, functionsWithNativeModules }, configFile, + { edgeFunctions, edgeFnShaMap }, ] = await Promise.all([ hashFiles({ assetType, @@ -125,6 +127,7 @@ export const deploySite = async ( rootDir: siteRoot, }), hashConfig({ config }), + hashEdgeFunctions(edgeFunctionsDistPath, { hashAlgorithm, statusCb }), ]) const files = { ...staticFiles, [configFile.normalizedPath]: configFile.hash } @@ -181,6 +184,7 @@ For more information, visit https://ntl.fyi/cli-native-modules.`) body: { files, functions, + edge_functions: edgeFunctions, function_schedules: functionSchedules, functions_config: fnConfig, async: Object.keys(files).length > syncFileLimit, @@ -195,19 +199,20 @@ For more information, visit https://ntl.fyi/cli-native-modules.`) if (deployParams.body.async) deploy = await waitForDiff(api, deploy.id, siteId, deployTimeout) - const { required: requiredFiles, required_functions: requiredFns } = deploy + const { required: requiredFiles, required_functions: requiredFns, required_edge_functions: requiredEdgeFns } = deploy statusCb({ type: 'create-deploy', msg: `CDN requesting ${requiredFiles.length} files${ Array.isArray(requiredFns) ? ` and ${requiredFns.length} functions` : '' - }`, + }${Array.isArray(requiredEdgeFns) ? ` and ${requiredEdgeFns.length} edge functions` : ''}`, phase: 'stop', }) const filesUploadList = getUploadList(requiredFiles, filesShaMap) const functionsUploadList = getUploadList(requiredFns, fnShaMap) - const uploadList = [...filesUploadList, ...functionsUploadList] + const edgeFunctionsUploadList = getUploadList(requiredEdgeFns, edgeFnShaMap) + const uploadList = [...filesUploadList, ...functionsUploadList, ...edgeFunctionsUploadList] await uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb, maxRetry }) diff --git a/src/utils/deploy/hash-edge-functions.ts b/src/utils/deploy/hash-edge-functions.ts new file mode 100644 index 00000000000..f9057b95701 --- /dev/null +++ b/src/utils/deploy/hash-edge-functions.ts @@ -0,0 +1,73 @@ +import { createHash } from 'node:crypto' +import { createReadStream } from 'node:fs' +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { pipeline } from 'node:stream/promises' + +import type { Manifest } from '@netlify/edge-bundler' + +import type { StatusCallback } from './status-cb.js' +import type { EdgeFunctionUploadFile } from './upload-files.js' + +const hashBundle = async (filepath: string, hashAlgorithm: string): Promise => { + const hasher = createHash(hashAlgorithm) + await pipeline([createReadStream(filepath), hasher]) + return hasher.digest('hex') +} + +// Reads the edge-bundler manifest from the dist directory and, for every bundle, computes its +// `code_sha` (sha256 of the bundle bytes — the deploy identity, recomputed rather than trusting the +// bundler's asset filename) so we can both declare it on deploy create and stream it on upload. We +// declare every format; bitballoon decides which ones actually ride this path and returns them in +// `required_edge_functions`. +const hashEdgeFunctions = async ( + edgeFunctionsDistPath: string | undefined, + { hashAlgorithm = 'sha256', statusCb }: { hashAlgorithm?: string; statusCb: StatusCallback }, +): Promise<{ + // edge_functions: { format => code_sha } sent on deploy create + edgeFunctions: Record + // code_sha => [fileObj] consumed by the upload arm + edgeFnShaMap: Record +}> => { + const edgeFunctions: Record = {} + const edgeFnShaMap: Record = {} + + if (!edgeFunctionsDistPath) { + return { edgeFunctions, edgeFnShaMap } + } + + // `Partial` because this is whatever happens to be on disk, not something we produced. + let manifest: Partial + try { + manifest = JSON.parse(await readFile(join(edgeFunctionsDistPath, 'manifest.json'), 'utf8')) as Partial + } catch { + // No manifest (or an unreadable one) means there are no edge functions to declare. + return { edgeFunctions, edgeFnShaMap } + } + + const bundles = manifest.bundles ?? [] + for (const bundle of bundles) { + const filepath = join(edgeFunctionsDistPath, bundle.asset) + const codeSha = await hashBundle(filepath, hashAlgorithm) + + edgeFunctions[bundle.format] = codeSha + + const fileObj: EdgeFunctionUploadFile = { + assetType: 'edge-function', + filepath, + normalizedPath: codeSha, + hash: codeSha, + } + if (Array.isArray(edgeFnShaMap[codeSha])) { + edgeFnShaMap[codeSha].push(fileObj) + } else { + edgeFnShaMap[codeSha] = [fileObj] + } + + statusCb({ type: 'hashing', msg: `Hashing edge function bundle ${bundle.asset}`, phase: 'progress' }) + } + + return { edgeFunctions, edgeFnShaMap } +} + +export default hashEdgeFunctions diff --git a/src/utils/deploy/upload-files.ts b/src/utils/deploy/upload-files.ts index 7244c96ceb0..35845e07bcd 100644 --- a/src/utils/deploy/upload-files.ts +++ b/src/utils/deploy/upload-files.ts @@ -1,22 +1,72 @@ import fs from 'fs' +import type { NetlifyAPI } from '@netlify/api' import backoff from 'backoff' import pMap from 'p-map' import { UPLOAD_INITIAL_DELAY, UPLOAD_MAX_DELAY, UPLOAD_RANDOM_FACTOR } from './constants.js' +import type { StatusCallback } from './status-cb.js' -// @ts-expect-error TS(7006) FIXME: Parameter 'api' implicitly has an 'any' type. -const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRetry, statusCb }) => { - if (!concurrentUpload || !statusCb || !maxRetry) throw new Error('Missing required option concurrentUpload') +export type UploadApi = Pick + +// `@netlify/api` only models path and query parameters, so header parameters such as +// `X-Nf-Retry-Count` have to be added on top of the generated parameter types. +type WithRetryCount = T & { xNfRetryCount?: number } + +type UploadDeployFunctionParams = WithRetryCount[0]> +type UploadDeployEdgeFunctionParams = WithRetryCount[0]> + +interface UploadFileBase { + filepath: string + normalizedPath: string + body?: fs.ReadStream +} + +export interface StaticUploadFile extends UploadFileBase { + assetType: 'file' +} + +export interface FunctionUploadFile extends UploadFileBase { + assetType: 'function' + runtime?: string + invocationMode?: string + timeout?: number +} + +export interface EdgeFunctionUploadFile extends UploadFileBase { + assetType: 'edge-function' + hash: string +} + +export type UploadFile = StaticUploadFile | FunctionUploadFile | EdgeFunctionUploadFile + +class MissingAssetTypeError extends Error { + constructor(readonly fileObj: unknown) { + super('File Object missing assetType property') + } +} + +interface UploadFilesOptions { + concurrentUpload: number + maxRetry: number + statusCb: StatusCallback +} + +const uploadFiles = async ( + api: UploadApi, + deployId: string, + uploadList: UploadFile[], + { concurrentUpload, maxRetry, statusCb }: UploadFilesOptions, +) => { + if (!concurrentUpload || !maxRetry) throw new Error('Missing required option concurrentUpload') statusCb({ type: 'upload', msg: `Uploading ${uploadList.length} files`, phase: 'start', }) - // @ts-expect-error TS(7006) FIXME: Parameter 'fileObj' implicitly has an 'any' type. - const uploadFile = async (fileObj, index) => { - const { assetType, body, filepath, invocationMode, normalizedPath, runtime, timeout } = fileObj + const uploadFile = async (fileObj: UploadFile, index: number) => { + const { body, filepath, normalizedPath } = fileObj const readStreamCtor = () => body ?? fs.createReadStream(filepath) @@ -25,10 +75,10 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet msg: `(${index}/${uploadList.length}) Uploading ${normalizedPath}...`, phase: 'progress', }) - let response - switch (assetType) { + + switch (fileObj.assetType) { case 'file': { - response = await retryUpload( + return await retryUpload( () => api.uploadDeployFile({ body: readStreamCtor, @@ -37,12 +87,12 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet }), maxRetry, ) - break } case 'function': { - // @ts-expect-error TS(7006) FIXME: Parameter 'retryCount' implicitly has an 'any' typ... Remove this comment to see the full error message - response = await retryUpload((retryCount) => { - const params = { + const { invocationMode, runtime, timeout } = fileObj + + return await retryUpload((retryCount) => { + const params: UploadDeployFunctionParams = { body: readStreamCtor, deployId, invocationMode, @@ -52,23 +102,31 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet } if (retryCount > 0) { - // @ts-expect-error TS(2339) FIXME: Property 'xNfRetryCount' does not exist on type '{... Remove this comment to see the full error message params.xNfRetryCount = retryCount } return api.uploadDeployFunction(params) }, maxRetry) - break + } + case 'edge-function': { + return await retryUpload((retryCount) => { + const params: UploadDeployEdgeFunctionParams = { + body: readStreamCtor, + deployId, + codeSha: normalizedPath, + } + + if (retryCount > 0) { + params.xNfRetryCount = retryCount + } + + return api.uploadDeployEdgeFunction(params) + }, maxRetry) } default: { - const error = new Error('File Object missing assetType property') - // @ts-expect-error TS(2339) FIXME: Property 'fileObj' does not exist on type 'Error'. - error.fileObj = fileObj - throw error + throw new MissingAssetTypeError(fileObj) } } - - return response } const results = await pMap(uploadList, uploadFile, { concurrency: concurrentUpload }) @@ -80,11 +138,14 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet return results } -// @ts-expect-error TS(7006) FIXME: Parameter 'uploadFn' implicitly has an 'any' type. -const retryUpload = (uploadFn, maxRetry) => - new Promise((resolve, reject) => { - // @ts-expect-error TS(7034) FIXME: Variable 'lastError' implicitly has type 'any' in ... Remove this comment to see the full error message - let lastError +const getErrorStatus = (error: unknown): number | undefined => + typeof error === 'object' && error !== null && 'status' in error && typeof error.status === 'number' + ? error.status + : undefined + +const retryUpload = (uploadFn: (retryCount: number) => Promise, maxRetry: number): Promise => + new Promise((resolve, reject) => { + let lastError: unknown const fibonacciBackoff = backoff.fibonacci({ randomisationFactor: UPLOAD_RANDOM_FACTOR, @@ -101,16 +162,16 @@ const retryUpload = (uploadFn, maxRetry) => } catch (error) { lastError = error + const status = getErrorStatus(error) + // We don't need to retry for 400 or 422 errors - // @ts-expect-error TS(2571) FIXME: Object is of type 'unknown'. - if (error.status === 400 || error.status === 422) { + if (status === 400 || status === 422) { reject(error) return } // observed errors: 408, 401 (4** swallowed), 502 - // @ts-expect-error TS(2571) FIXME: Object is of type 'unknown'. - if (error.status > 400 || error.name === 'FetchError') { + if ((status !== undefined && status > 400) || (error instanceof Error && error.name === 'FetchError')) { fibonacciBackoff.backoff() return } @@ -130,7 +191,6 @@ const retryUpload = (uploadFn, maxRetry) => fibonacciBackoff.on('ready', tryUpload) fibonacciBackoff.on('fail', () => { - // @ts-expect-error TS(7005) FIXME: Variable 'lastError' implicitly has an 'any' type. reject(lastError) }) diff --git a/tests/unit/utils/deploy/hash-edge-functions.test.ts b/tests/unit/utils/deploy/hash-edge-functions.test.ts new file mode 100644 index 00000000000..9b6f01dee17 --- /dev/null +++ b/tests/unit/utils/deploy/hash-edge-functions.test.ts @@ -0,0 +1,57 @@ +import { createHash } from 'node:crypto' +import { mkdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' + +import { expect, test } from 'vitest' + +import hashEdgeFunctions from '../../../../src/utils/deploy/hash-edge-functions.js' +import { temporaryDirectory } from '../../../../src/utils/temporary-file.js' + +const sha256 = (contents: string) => createHash('sha256').update(contents).digest('hex') + +const writeManifest = async (dir: string, bundles: { asset: string; format: string; contents: string }[]) => { + await mkdir(dir, { recursive: true }) + await Promise.all(bundles.map(({ asset, contents }) => writeFile(join(dir, asset), contents))) + await writeFile( + join(dir, 'manifest.json'), + JSON.stringify({ bundles: bundles.map(({ asset, format }) => ({ asset, format })) }), + ) +} + +test('declares every bundle format, keyed by the recomputed code_sha', async () => { + const dir = temporaryDirectory() + await writeManifest(dir, [ + { asset: 'aaa.tar.gz', format: 'tar', contents: 'tar-bundle-bytes' }, + { asset: 'bbb.eszip', format: 'eszip2', contents: 'eszip-bundle-bytes' }, + ]) + + const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(dir, { statusCb() {} }) + + const tarSha = sha256('tar-bundle-bytes') + const eszipSha = sha256('eszip-bundle-bytes') + // We declare all formats; bitballoon filters and only asks for the ones that ride this path. + expect(edgeFunctions).toEqual({ tar: tarSha, eszip2: eszipSha }) + expect(Object.keys(edgeFnShaMap).sort()).toEqual([tarSha, eszipSha].sort()) + expect(edgeFnShaMap[tarSha][0]).toMatchObject({ + assetType: 'edge-function', + filepath: join(dir, 'aaa.tar.gz'), + normalizedPath: tarSha, + }) +}) + +test('returns empty maps when there is no dist path', async () => { + const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(undefined, { statusCb() {} }) + + expect(edgeFunctions).toEqual({}) + expect(edgeFnShaMap).toEqual({}) +}) + +test('returns empty maps when the manifest is missing', async () => { + const dir = temporaryDirectory() + await mkdir(dir, { recursive: true }) + + const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(dir, { statusCb() {} }) + + expect(edgeFunctions).toEqual({}) + expect(edgeFnShaMap).toEqual({}) +}) diff --git a/tests/unit/utils/deploy/upload-files.test.ts b/tests/unit/utils/deploy/upload-files.test.ts index d3c1c089b83..876c73d9b73 100644 --- a/tests/unit/utils/deploy/upload-files.test.ts +++ b/tests/unit/utils/deploy/upload-files.test.ts @@ -1,7 +1,7 @@ import crypto from 'crypto' import { afterAll, expect, test, vi } from 'vitest' -import uploadFiles from '../../../../src/utils/deploy/upload-files.js' +import uploadFiles, { type UploadApi, type UploadFile } from '../../../../src/utils/deploy/upload-files.js' vi.mock('../../../../src/utils/deploy/constants.js', async () => { const actual = await vi.importActual('../../../../src/utils/deploy/constants.js') @@ -18,8 +18,7 @@ test('Adds a retry count to function upload requests', async () => { const uploadDeployFunction = vi.fn() const mockError = new Error('Uh-oh') - // @ts-expect-error TS(2339) FIXME: Property 'status' does not exist on type 'Error'. - mockError.status = 500 + Object.assign(mockError, { status: 500 }) uploadDeployFunction.mockRejectedValueOnce(mockError) uploadDeployFunction.mockRejectedValueOnce(mockError) @@ -27,9 +26,9 @@ test('Adds a retry count to function upload requests', async () => { const mockApi = { uploadDeployFunction, - } + } as unknown as UploadApi const deployId = crypto.randomUUID() - const files = [ + const files: UploadFile[] = [ { assetType: 'function', filepath: '/some/path/func1.zip', @@ -51,20 +50,54 @@ test('Adds a retry count to function upload requests', async () => { expect(uploadDeployFunction).toHaveBeenNthCalledWith(3, expect.objectContaining({ xNfRetryCount: 2 })) }) +test('Adds a retry count to edge function upload requests', async () => { + const uploadDeployEdgeFunction = vi.fn() + const mockError = new Error('Uh-oh') + + Object.assign(mockError, { status: 500 }) + + uploadDeployEdgeFunction.mockRejectedValueOnce(mockError) + uploadDeployEdgeFunction.mockResolvedValueOnce(undefined) + + const mockApi = { + uploadDeployEdgeFunction, + } as unknown as UploadApi + const deployId = crypto.randomUUID() + const files: UploadFile[] = [ + { + assetType: 'edge-function', + filepath: '/some/path/abc123.tar.gz', + normalizedPath: 'abc123', + hash: 'abc123', + }, + ] + const options = { + concurrentUpload: 1, + maxRetry: 3, + statusCb: vi.fn(), + } + + await uploadFiles(mockApi, deployId, files, options) + + expect(uploadDeployEdgeFunction).toHaveBeenCalledTimes(2) + expect(uploadDeployEdgeFunction).toHaveBeenNthCalledWith(1, expect.objectContaining({ codeSha: 'abc123' })) + expect(uploadDeployEdgeFunction).toHaveBeenNthCalledWith(1, expect.not.objectContaining({ xNfRetryCount: 1 })) + expect(uploadDeployEdgeFunction).toHaveBeenNthCalledWith(2, expect.objectContaining({ xNfRetryCount: 1 })) +}) + test('Does not retry on 400 response from function upload requests', async () => { const uploadDeployFunction = vi.fn() const mockError = new Error('Uh-oh') - // @ts-expect-error TS(2339) FIXME: Property 'status' does not exist on type 'Error'. - mockError.status = 400 + Object.assign(mockError, { status: 400 }) uploadDeployFunction.mockRejectedValue(mockError) const mockApi = { uploadDeployFunction, - } + } as unknown as UploadApi const deployId = crypto.randomUUID() - const files = [ + const files: UploadFile[] = [ { assetType: 'function', filepath: '/some/path/func1.zip',