feat: Update to latest blob client (7.3.0) (#398)

* chore: Add buildVersion and useRegionalBlobs to PluginContext

* chore: Centralize deploy store configuration

* chore: Extract FixtureTestContext and BLOB_TOKEN into their own files

* chore: Prepare getBlobServerGets to handle regions

* chore: Set and make use of shared build/run USE_REGIONAL_BLOBS environment variable

* chore: Use latest @netlify/blobs version

* chore: Pin regional blob functionality to a higher version of the cli

* chore: mark all runtime modules as external

* fix: Ensure ts files are compiled in unit tests

* chore: linting

* maybe win slash?

* test: add fixture using CLI before regional blobs support

* test: use createRequestContext in tests instead of manually creating request context objects

* Update tests/e2e/page-router.test.ts

Co-authored-by: Philippe Serhal <[email protected]>

* test: rename unit test for blobs directory

---------

Co-authored-by: Michal Piechowiak <[email protected]>
Co-authored-by: Philippe Serhal <[email protected]>

Author: 3 people

package-lock.json

@@ -49,7 +49,7 @@
   "homepage": "https://github.com/netlify/next-runtime-minimal#readme",
   "devDependencies": {
     "@fastly/http-compute-js": "1.1.4",
-    "@netlify/blobs": "^7.0.1",
+    "@netlify/blobs": "^7.3.0",
     "@netlify/build": "^29.37.2",
     "@netlify/edge-bundler": "^11.4.0",
     "@netlify/edge-functions": "^2.5.1",

package.json

@@ -7,7 +7,8 @@ import glob from 'fast-glob'
 import { Mock, beforeEach, describe, expect, test, vi } from 'vitest'
 
 import { mockFileSystem } from '../../../tests/index.js'
-import { FixtureTestContext, createFsFixture } from '../../../tests/utils/fixture.js'
+import { type FixtureTestContext } from '../../../tests/utils/contexts.js'
+import { createFsFixture } from '../../../tests/utils/fixture.js'
 import { PluginContext, RequiredServerFilesManifest } from '../plugin-context.js'
 
 import { copyStaticAssets, copyStaticContent } from './static.js'

src/build/content/static.test.ts

@@ -97,6 +97,7 @@ const getHandlerFile = async (ctx: PluginContext): Promise<string> => {
     return template
       .replaceAll('{{cwd}}', posixJoin(ctx.lambdaWorkingDirectory))
       .replace('{{nextServerHandler}}', posixJoin(ctx.nextServerHandler))
+      .replace('{{useRegionalBlobs}}', ctx.useRegionalBlobs.toString())
   }
 
   return await readFile(join(templatesDir, 'handler.tmpl.js'), 'utf-8')

src/build/functions/server.ts

@@ -195,3 +195,18 @@ test('nx monorepo with package path and different distDir', () => {
   expect(ctx.relPublishDir).toBe('dist/apps/my-app/.next')
   expect(ctx.publishDir).toBe(join(cwd, 'dist/apps/my-app/.next'))
 })
+
+test('should use deploy configuration blobs directory when @netlify/build version supports regional blob awareness', () => {
+  const { cwd } = mockFileSystem({
+    '.next/required-server-files.json': JSON.stringify({
+      config: { distDir: '.next' },
+      relativeAppDir: '',
+    } as RequiredServerFilesManifest),
+  })
+
+  const ctx = new PluginContext({
+    constants: { NETLIFY_BUILD_VERSION: '29.39.1' },
+  } as NetlifyPluginOptions)
+
+  expect(ctx.blobDir).toBe(join(cwd, '.netlify/deploy/v1/blobs/deploy'))
+})

src/build/plugin-context.test.ts

@@ -13,6 +13,7 @@ import type {
 import type { PrerenderManifest, RoutesManifest } from 'next/dist/build/index.js'
 import type { MiddlewareManifest } from 'next/dist/build/webpack/plugins/middleware-plugin.js'
 import type { NextConfigComplete } from 'next/dist/server/config-shared.js'
+import { satisfies } from 'semver'
 
 const MODULE_DIR = fileURLToPath(new URL('.', import.meta.url))
 const PLUGIN_DIR = join(MODULE_DIR, '../..')
@@ -135,12 +136,27 @@ export class PluginContext {
 
   /**
    * Absolute path of the directory that will be deployed to the blob store
-   * `.netlify/blobs/deploy`
+   * region aware: `.netlify/deploy/v1/blobs/deploy`
+   * default: `.netlify/blobs/deploy`
    */
   get blobDir(): string {
+    if (this.useRegionalBlobs) {
+      return this.resolveFromPackagePath('.netlify/deploy/v1/blobs/deploy')
+    }
+
     return this.resolveFromPackagePath('.netlify/blobs/deploy')
   }
 
+  get buildVersion(): string {
+    return this.constants.NETLIFY_BUILD_VERSION || 'v0.0.0'
+  }
+
+  get useRegionalBlobs(): boolean {
+    // Region-aware blobs are only available as of CLI v17.22.1 (i.e. Build v29.39.1)
+    const REQUIRED_BUILD_VERSION = '>=29.39.1'
+    return satisfies(this.buildVersion, REQUIRED_BUILD_VERSION, { includePrerelease: true })
+  }
+
   /**
    * Absolute path of the directory containing the files for the serverless lambda function
    * `.netlify/functions-internal`

src/build/plugin-context.ts

@@ -7,6 +7,9 @@ import tracing from '{{cwd}}/.netlify/dist/run/handlers/tracing.js'
 
 process.chdir('{{cwd}}')
 
+// Set feature flag for regional blobs
+process.env.USE_REGIONAL_BLOBS = '{{useRegionalBlobs}}'
+
 let cachedHandler
 export default async function (req, context) {
   if (process.env.NETLIFY_OTLP_TRACE_EXPORTER_URL) {

src/build/templates/handler-monorepo.tmpl.js

@@ -6,6 +6,9 @@ import serverHandler from './.netlify/dist/run/handlers/server.js'
 import { getTracer } from './.netlify/dist/run/handlers/tracer.cjs'
 import tracing from './.netlify/dist/run/handlers/tracing.js'
 
+// Set feature flag for regional blobs
+process.env.USE_REGIONAL_BLOBS = '{{useRegionalBlobs}}'
+
 export default async function handler(req, context) {
   if (process.env.NETLIFY_OTLP_TRACE_EXPORTER_URL) {
     tracing.start()

src/build/templates/handler.tmpl.js

@@ -3,7 +3,7 @@
 //
 import { Buffer } from 'node:buffer'
 
-import { getDeployStore, Store } from '@netlify/blobs'
+import { Store } from '@netlify/blobs'
 import { purgeCache } from '@netlify/functions'
 import { type Span } from '@opentelemetry/api'
 import { NEXT_CACHE_TAGS_HEADER } from 'next/dist/lib/constants.js'
@@ -16,6 +16,7 @@ import type {
   NetlifyCacheHandlerValue,
   NetlifyIncrementalCacheValue,
 } from '../../shared/cache-types.cjs'
+import { getRegionalBlobStore } from '../regional-blob-store.cjs'
 
 import { getRequestContext } from './request-context.cjs'
 import { getTracer } from './tracer.cjs'
@@ -24,8 +25,6 @@ type TagManifest = { revalidatedAt: number }
 
 type TagManifestBlobCache = Record<string, Promise<TagManifest>>
 
-const fetchBeforeNextPatchedIt = globalThis.fetch
-
 export class NetlifyCacheHandler implements CacheHandler {
   options: CacheHandlerContext
   revalidatedTags: string[]
@@ -36,7 +35,7 @@ export class NetlifyCacheHandler implements CacheHandler {
   constructor(options: CacheHandlerContext) {
     this.options = options
     this.revalidatedTags = options.revalidatedTags
-    this.blobStore = getDeployStore({ fetch: fetchBeforeNextPatchedIt, consistency: 'strong' })
+    this.blobStore = getRegionalBlobStore({ consistency: 'strong' })
     this.tagManifestsFetchedFromBlobStoreInCurrentRequest = {}
   }
 

src/run/handlers/cache.cts

@@ -1,11 +1,12 @@
 import type { NextConfigComplete } from 'next/dist/server/config-shared.js'
 import { v4 } from 'uuid'
-import { afterEach, describe, expect, test, vi, beforeEach } from 'vitest'
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'
 
-import { FixtureTestContext } from '../../tests/utils/fixture.js'
+import { type FixtureTestContext } from '../../tests/utils/contexts.js'
 import { generateRandomObjectID, startMockBlobStore } from '../../tests/utils/helpers.js'
 
-import { setVaryHeaders, setCacheControlHeaders } from './headers.js'
+import { createRequestContext } from './handlers/request-context.cjs'
+import { setCacheControlHeaders, setVaryHeaders } from './headers.js'
 
 beforeEach<FixtureTestContext>(async (ctx) => {
   // set for each test a new deployID and siteID
@@ -198,7 +199,7 @@ describe('headers', () => {
       const request = new Request(defaultUrl)
       vi.spyOn(headers, 'set')
 
-      setCacheControlHeaders(headers, request, {})
+      setCacheControlHeaders(headers, request, createRequestContext())
 
       expect(headers.set).toHaveBeenCalledTimes(0)
     })
@@ -208,7 +209,10 @@ describe('headers', () => {
       const request = new Request(defaultUrl)
       vi.spyOn(headers, 'set')
 
-      setCacheControlHeaders(headers, request, { usedFsRead: true })
+      const requestContext = createRequestContext()
+      requestContext.usedFsRead = true
+
+      setCacheControlHeaders(headers, request, requestContext)
 
       expect(headers.set).toHaveBeenNthCalledWith(
         1,
@@ -231,7 +235,7 @@ describe('headers', () => {
       const request = new Request(defaultUrl)
       vi.spyOn(headers, 'set')
 
-      setCacheControlHeaders(headers, request, {})
+      setCacheControlHeaders(headers, request, createRequestContext())
 
       expect(headers.set).toHaveBeenCalledTimes(0)
     })
@@ -245,7 +249,7 @@ describe('headers', () => {
       const request = new Request(defaultUrl)
       vi.spyOn(headers, 'set')
 
-      setCacheControlHeaders(headers, request, {})
+      setCacheControlHeaders(headers, request, createRequestContext())
 
       expect(headers.set).toHaveBeenCalledTimes(0)
     })
@@ -258,7 +262,7 @@ describe('headers', () => {
       const request = new Request(defaultUrl)
       vi.spyOn(headers, 'set')
 
-      setCacheControlHeaders(headers, request, {})
+      setCacheControlHeaders(headers, request, createRequestContext())
 
       expect(headers.set).toHaveBeenNthCalledWith(
         1,
@@ -280,7 +284,7 @@ describe('headers', () => {
       const request = new Request(defaultUrl, { method: 'HEAD' })
       vi.spyOn(headers, 'set')
 
-      setCacheControlHeaders(headers, request, {})
+      setCacheControlHeaders(headers, request, createRequestContext())
 
       expect(headers.set).toHaveBeenNthCalledWith(
         1,
@@ -302,7 +306,7 @@ describe('headers', () => {
       const request = new Request(defaultUrl, { method: 'POST' })
       vi.spyOn(headers, 'set')
 
-      setCacheControlHeaders(headers, request, {})
+      setCacheControlHeaders(headers, request, createRequestContext())
 
       expect(headers.set).toHaveBeenCalledTimes(0)
     })
@@ -315,7 +319,7 @@ describe('headers', () => {
       const request = new Request(defaultUrl)
       vi.spyOn(headers, 'set')
 
-      setCacheControlHeaders(headers, request, {})
+      setCacheControlHeaders(headers, request, createRequestContext())
 
       expect(headers.set).toHaveBeenNthCalledWith(1, 'cache-control', 'public')
       expect(headers.set).toHaveBeenNthCalledWith(
@@ -333,7 +337,7 @@ describe('headers', () => {
       const request = new Request(defaultUrl)
       vi.spyOn(headers, 'set')
 
-      setCacheControlHeaders(headers, request, {})
+      setCacheControlHeaders(headers, request, createRequestContext())
 
       expect(headers.set).toHaveBeenNthCalledWith(1, 'cache-control', 'max-age=604800')
       expect(headers.set).toHaveBeenNthCalledWith(
@@ -351,7 +355,7 @@ describe('headers', () => {
       const request = new Request(defaultUrl)
       vi.spyOn(headers, 'set')
 
-      setCacheControlHeaders(headers, request, {})
+      setCacheControlHeaders(headers, request, createRequestContext())
 
       expect(headers.set).toHaveBeenNthCalledWith(
         1,

src/run/headers.test.ts

@@ -1,4 +1,3 @@
-import { getDeployStore } from '@netlify/blobs'
 import type { Span } from '@opentelemetry/api'
 import type { NextConfigComplete } from 'next/dist/server/config-shared.js'
 
@@ -7,6 +6,7 @@ import { encodeBlobKey } from '../shared/blobkey.js'
 import type { TagsManifest } from './config.js'
 import type { RequestContext } from './handlers/request-context.cjs'
 import type { RuntimeTracer } from './handlers/tracer.cjs'
+import { getRegionalBlobStore } from './regional-blob-store.cjs'
 
 const ALL_VARIATIONS = Symbol.for('ALL_VARIATIONS')
 interface NetlifyVaryValues {
@@ -121,8 +121,6 @@ export const setVaryHeaders = (
   headers.set(`netlify-vary`, generateNetlifyVaryValues(netlifyVaryValues))
 }
 
-const fetchBeforeNextPatchedIt = globalThis.fetch
-
 /**
  * Change the date header to be the last-modified date of the blob. This means the CDN
  * will use the correct expiry time for the response. e.g. if the blob was last modified
@@ -173,8 +171,8 @@ export const adjustDateHeader = async ({
       warning: true,
     })
 
+    const blobStore = getRegionalBlobStore({ consistency: 'strong' })
     const blobKey = await encodeBlobKey(key)
-    const blobStore = getDeployStore({ fetch: fetchBeforeNextPatchedIt, consistency: 'strong' })
 
     // TODO: use metadata for this
     lastModified = await tracer.withActiveSpan(

src/run/headers.ts

@@ -1,12 +1,12 @@
 import fs from 'fs/promises'
 import { relative, resolve } from 'path'
 
-import { getDeployStore } from '@netlify/blobs'
 // @ts-expect-error no types installed
 import { patchFs } from 'fs-monkey'
 
 import { getRequestContext } from './handlers/request-context.cjs'
 import { getTracer } from './handlers/tracer.cjs'
+import { getRegionalBlobStore } from './regional-blob-store.cjs'
 
 console.time('import next server')
 
@@ -17,8 +17,6 @@ console.timeEnd('import next server')
 
 type FS = typeof import('fs')
 
-const fetchBeforeNextPatchedIt = globalThis.fetch
-
 export async function getMockedRequestHandlers(...args: Parameters<typeof getRequestHandlers>) {
   const tracer = getTracer()
   return tracer.withActiveSpan('mocked request handler', async () => {
@@ -35,7 +33,7 @@ export async function getMockedRequestHandlers(...args: Parameters<typeof getReq
       } catch (error) {
         // only try to get .html files from the blob store
         if (typeof path === 'string' && path.endsWith('.html')) {
-          const store = getDeployStore({ fetch: fetchBeforeNextPatchedIt })
+          const store = getRegionalBlobStore()
           const relPath = relative(resolve('.next/server/pages'), path)
           const file = await store.get(await encodeBlobKey(relPath))
           if (file !== null) {

src/run/next.cts

@@ -0,0 +1,12 @@
+import { getDeployStore, Store } from '@netlify/blobs'
+
+const fetchBeforeNextPatchedIt = globalThis.fetch
+
+export const getRegionalBlobStore = (args: Parameters<typeof getDeployStore>[0] = {}): Store => {
+  return getDeployStore({
+    ...args,
+    fetch: fetchBeforeNextPatchedIt,
+    experimentalRegion:
+      process.env.USE_REGIONAL_BLOBS?.toUpperCase() === 'TRUE' ? 'context' : undefined,
+  })
+}

src/run/regional-blob-store.cts

@@ -0,0 +1,25 @@
+import { expect } from '@playwright/test'
+import { test } from '../utils/playwright-helpers.js'
+
+test('should serve 404 page when requesting non existing page (no matching route) if site is deployed with CLI not supporting regional blobs', async ({
+  page,
+  cliBeforeRegionalBlobsSupport,
+}) => {
+  // 404 page is built and uploaded to blobs at build time
+  // when Next.js serves 404 it will try to fetch it from the blob store
+  // if request handler function is unable to get from blob store it will
+  // fail request handling and serve 500 error.
+  // This implicitly tests that request handler function is able to read blobs
+  // that are uploaded as part of site deploy.
+
+  const response = await page.goto(new URL('non-existing', cliBeforeRegionalBlobsSupport.url).href)
+  const headers = response?.headers() || {}
+  expect(response?.status()).toBe(404)
+
+  expect(await page.textContent('h1')).toBe('404')
+
+  expect(headers['netlify-cdn-cache-control']).toBe(
+    'no-cache, no-store, max-age=0, must-revalidate',
+  )
+  expect(headers['cache-control']).toBe('no-cache,no-store,max-age=0,must-revalidate')
+})