From 7992232c27412840c04fe24f50c87bbf4423cb9a Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Wed, 13 Nov 2024 13:27:25 +0100
Subject: [PATCH 01/22] ci: Only run E2E test apps that are affected on PRs

---
 .github/workflows/build.yml                   | 47 ++++++++----
 .../lib/ciShouldSkipTestApplication.ts        | 72 +++++++++++++++++++
 dev-packages/e2e-tests/package.json           |  1 +
 3 files changed, 106 insertions(+), 14 deletions(-)
 create mode 100644 dev-packages/e2e-tests/lib/ciShouldSkipTestApplication.ts

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e80e2e869905..1c61c467b5f5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -963,11 +963,6 @@ jobs:
           - test-application: 'nextjs-app-dir'
             build-command: 'test:build-13'
             label: 'nextjs-app-dir (next@13)'
-        exclude:
-          - is_dependabot: true
-            test-application: 'cloudflare-astro'
-          - is_dependabot: true
-            test-application: 'cloudflare-workers'
 
     steps:
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
@@ -989,18 +984,26 @@ jobs:
         with:
           dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }}
 
+      - name: Determine if test app should be run
+        id: should-skip
+        if: github.event.pull_request.base.sha
+        run: yarn test:should-skip ${{ matrix.test-application }} --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
+        working-directory: dev-packages/e2e-tests
+
       - name: Restore tarball cache
         uses: actions/cache/restore@v4
         id: restore-tarball-cache
+        if: steps.should-skip.outputs.SKIP != 'true'
         with:
           path: ${{ github.workspace }}/packages/*/*.tgz
           key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
 
       - name: Build tarballs if not cached
-        if: steps.restore-tarball-cache.outputs.cache-hit != 'true'
+        if: steps.restore-tarball-cache.outputs.cache-hit != 'true' && steps.should-skip.outputs.SKIP != 'true'
         run: yarn build:tarball
 
       - name: Install Playwright
+        if: steps.should-skip.outputs.SKIP != 'true'
         uses: ./.github/actions/install-playwright
         with:
           browsers: chromium
@@ -1011,28 +1014,32 @@ jobs:
           echo "echo node=$(jq -r '.volta.node' dev-packages/e2e-tests/package.json)" >> $GITHUB_OUTPUT
 
       - name: Validate Verdaccio
+        if: steps.should-skip.outputs.SKIP != 'true'
         run: yarn test:validate
         working-directory: dev-packages/e2e-tests
 
       - name: Prepare Verdaccio
+        if: steps.should-skip.outputs.SKIP != 'true'
         run: yarn test:prepare
         working-directory: dev-packages/e2e-tests
         env:
           E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
 
       - name: Build E2E app
+        if: steps.should-skip.outputs.SKIP != 'true'
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 7
         run: pnpm ${{ matrix.build-command || 'test:build' }}
 
       - name: Run E2E test
+        if: steps.should-skip.outputs.SKIP != 'true'
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 10
         run: pnpm test:assert
 
       - name: Upload Playwright Traces
         uses: actions/upload-artifact@v4
-        if: failure()
+        if: failure() &&  steps.should-skip.outputs.SKIP != 'true'
         with:
           name: playwright-traces-job_e2e_playwright_tests-${{ matrix.test-application}}
           path: dev-packages/e2e-tests/test-applications/${{ matrix.test-application}}/test-results
@@ -1040,13 +1047,13 @@ jobs:
           retention-days: 7
 
       - name: Pre-process E2E Test Dumps
-        if: always()
+        if: always() && steps.should-skip.outputs.SKIP != 'true'
         run: |
           node ./scripts/normalize-e2e-test-dump-transaction-events.js
 
       - name: Upload E2E Test Event Dumps
         uses: actions/upload-artifact@v4
-        if: always()
+        if: always() && steps.should-skip.outputs.SKIP != 'true'
         with:
           name: E2E Test Dump (${{ matrix.label || matrix.test-application }})
           path: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/event-dumps
@@ -1055,7 +1062,7 @@ jobs:
           if-no-files-found: ignore
 
       - name: Upload test results to Codecov
-        if: cancelled() == false
+        if: cancelled() == false && steps.should-skip.outputs.SKIP != 'true'
         continue-on-error: true
         uses: codecov/test-results-action@v1
         with:
@@ -1155,7 +1162,14 @@ jobs:
         with:
           dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }}
 
+      - name: Determine if test app should be run
+        id: should-skip
+        if: github.event.pull_request.base.sha
+        run: yarn test:should-skip ${{ matrix.test-application }} --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
+        working-directory: dev-packages/e2e-tests
+
       - name: Restore tarball cache
+        if: steps.should-skip.outputs.SKIP != 'true'
         uses: actions/cache/restore@v4
         id: restore-tarball-cache
         with:
@@ -1163,10 +1177,11 @@ jobs:
           key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
 
       - name: Build tarballs if not cached
-        if: steps.restore-tarball-cache.outputs.cache-hit != 'true'
+        if: steps.restore-tarball-cache.outputs.cache-hit != 'true' &&  steps.should-skip.outputs.SKIP != 'true'
         run: yarn build:tarball
 
       - name: Install Playwright
+        if: steps.should-skip.outputs.SKIP != 'true'
         uses: ./.github/actions/install-playwright
         with:
           browsers: chromium
@@ -1177,33 +1192,37 @@ jobs:
           echo "echo node=$(jq -r '.volta.node' dev-packages/e2e-tests/package.json)" >> $GITHUB_OUTPUT
 
       - name: Validate Verdaccio
+        if: steps.should-skip.outputs.SKIP != 'true'
         run: yarn test:validate
         working-directory: dev-packages/e2e-tests
 
       - name: Prepare Verdaccio
+        if: steps.should-skip.outputs.SKIP != 'true'
         run: yarn test:prepare
         working-directory: dev-packages/e2e-tests
         env:
           E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
 
       - name: Build E2E app
+        if: steps.should-skip.outputs.SKIP != 'true'
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 7
         run: pnpm ${{ matrix.build-command || 'test:build' }}
 
       - name: Run E2E test
+        if: steps.should-skip.outputs.SKIP != 'true'
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 10
         run: pnpm ${{ matrix.assert-command || 'test:assert' }}
 
       - name: Pre-process E2E Test Dumps
-        if: always()
+        if: always() && steps.should-skip.outputs.SKIP != 'true'
         run: |
           node ./scripts/normalize-e2e-test-dump-transaction-events.js
 
       - name: Upload E2E Test Event Dumps
         uses: actions/upload-artifact@v4
-        if: always()
+        if: always() && steps.should-skip.outputs.SKIP != 'true'
         with:
           name: E2E Test Dump (${{ matrix.label || matrix.test-application }})
           path: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/event-dumps
@@ -1213,7 +1232,7 @@ jobs:
 
       - name: Deploy Astro to Cloudflare
         uses: cloudflare/pages-action@v1
-        if: matrix.test-application == 'cloudflare-astro'
+        if: matrix.test-application == 'cloudflare-astro' && steps.should-skip.outputs.SKIP != 'true'
         with:
           apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
diff --git a/dev-packages/e2e-tests/lib/ciShouldSkipTestApplication.ts b/dev-packages/e2e-tests/lib/ciShouldSkipTestApplication.ts
new file mode 100644
index 000000000000..bf112b4919f6
--- /dev/null
+++ b/dev-packages/e2e-tests/lib/ciShouldSkipTestApplication.ts
@@ -0,0 +1,72 @@
+import { execSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+
+/**
+ * This logs SKIP=true or SKIP=false,
+ * which can be used by GHA to determine if the given test application should be skipped or not.
+ * This uses nx to check if any of the sentry dependencies of the test-application have been affected.
+ */
+function ciShouldSkipTestApplication(): void {
+  // Allow to run a single app only via `yarn test:run <app-name>`
+  const appName = process.argv[2];
+
+  const args: Record<string, string> = {};
+  process.argv
+    .slice(2)
+    .filter(arg => arg.startsWith('--') && arg.includes('='))
+    .forEach(arg => {
+      const [part1, part2] = arg.split('=') as [string, string];
+      const argName = part1.replace('--', '');
+      const argValue = part2;
+      args[argName] = argValue;
+    });
+
+  // We default to `develop` as base, if none is specified
+  // head has a correct default value anyhow
+  const { base = 'develop', head } = args;
+
+  if (!appName) {
+    throw new Error('Please provide the app name as the first argument');
+  }
+
+  const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json');
+
+  if (!fs.existsSync(fullPath)) {
+    throw new Error(`The app ${appName} does not exist`);
+  }
+
+  const packageJson = JSON.parse(fs.readFileSync(fullPath, 'utf8')) as {
+    dependencies?: { [key: string]: string };
+    devDependencies?: { [key: string]: string };
+  };
+
+  const dependencies = {
+    ...packageJson.devDependencies,
+    ...packageJson.dependencies,
+  };
+
+  const sentryDependencies = Object.keys(dependencies).filter(key => key.startsWith('@sentry/'));
+
+  const additionalArgs = [];
+  if (base) {
+    additionalArgs.push(`--base=${base}`);
+  }
+  if (head) {
+    additionalArgs.push(`--head=${head}`);
+  }
+
+  const affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`)
+    .toString()
+    .split('\n')
+    .map(line => line.trim());
+
+  // If one of the sentry dependencies is affected, this test should be run
+  const affected = sentryDependencies.some(dep => affectedProjects.includes(dep));
+
+  // This is used by CI to determine if steps should be skipped or not
+  // eslint-disable-next-line no-console
+  console.log(`SKIP=${affected ? 'false' : 'true'}`);
+}
+
+ciShouldSkipTestApplication();
diff --git a/dev-packages/e2e-tests/package.json b/dev-packages/e2e-tests/package.json
index fdb5958462ee..d982aad9d9fa 100644
--- a/dev-packages/e2e-tests/package.json
+++ b/dev-packages/e2e-tests/package.json
@@ -14,6 +14,7 @@
     "test:prepare": "ts-node prepare.ts",
     "test:validate": "run-s test:validate-configuration test:validate-test-app-setups",
     "clean": "rimraf tmp node_modules pnpm-lock.yaml && yarn clean:test-applications",
+    "test:should-skip": "ts-node ./lib/ciShouldSkipTestApplication.ts",
     "clean:test-applications": "rimraf --glob test-applications/**/{node_modules,dist,build,.next,.sveltekit,pnpm-lock.yaml} .last-run.json && pnpm store prune"
   },
   "devDependencies": {

From 45f9e67c4b99950ca1c262523ecd226475f7287c Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Wed, 13 Nov 2024 14:47:50 +0100
Subject: [PATCH 02/22] fix checkout

---
 .github/workflows/build.yml | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 1c61c467b5f5..16d21264f35e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -965,6 +965,12 @@ jobs:
             label: 'nextjs-app-dir (next@13)'
 
     steps:
+      - name: Check out base commit (${{ github.event.pull_request.base.sha }})
+        uses: actions/checkout@v4
+        if: github.event_name == 'pull_request'
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v4
         with:
@@ -986,8 +992,8 @@ jobs:
 
       - name: Determine if test app should be run
         id: should-skip
-        if: github.event.pull_request.base.sha
-        run: yarn test:should-skip ${{ matrix.test-application }} --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
+        if: github.event_name == 'pull_request'
+        run: yarn --silent test:should-skip ${{ matrix.test-application }} --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
         working-directory: dev-packages/e2e-tests
 
       - name: Restore tarball cache
@@ -1146,6 +1152,11 @@ jobs:
             label: 'nextjs-turbo (latest)'
 
     steps:
+      - name: Check out base commit (${{ github.event.pull_request.base.sha }})
+        uses: actions/checkout@v4
+        if: github.event_name == 'pull_request'
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v4
         with:
@@ -1164,8 +1175,8 @@ jobs:
 
       - name: Determine if test app should be run
         id: should-skip
-        if: github.event.pull_request.base.sha
-        run: yarn test:should-skip ${{ matrix.test-application }} --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
+        if: github.event_name == 'pull_request'
+        run: yarn --silent test:should-skip ${{ matrix.test-application }} --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
         working-directory: dev-packages/e2e-tests
 
       - name: Restore tarball cache

From 83be5c6c360115b96528e2840d9d866b26e15801 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Wed, 13 Nov 2024 16:14:27 +0100
Subject: [PATCH 03/22] add missing skip

---
 .github/workflows/build.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 16d21264f35e..1e223cb424f9 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -1015,6 +1015,7 @@ jobs:
           browsers: chromium
 
       - name: Get node version
+        if: steps.should-skip.outputs.SKIP != 'true'
         id: versions
         run: |
           echo "echo node=$(jq -r '.volta.node' dev-packages/e2e-tests/package.json)" >> $GITHUB_OUTPUT
@@ -1199,6 +1200,7 @@ jobs:
 
       - name: Get node version
         id: versions
+        if: steps.should-skip.outputs.SKIP != 'true'
         run: |
           echo "echo node=$(jq -r '.volta.node' dev-packages/e2e-tests/package.json)" >> $GITHUB_OUTPUT
 

From c92f5d88167f3980a7fcbbf365696e7a0756874a Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Wed, 13 Nov 2024 17:15:32 +0100
Subject: [PATCH 04/22] mark optionals as optional

---
 .../e2e-tests/test-applications/cloudflare-astro/package.json  | 3 +++
 .../test-applications/cloudflare-workers/package.json          | 3 +++
 .../test-applications/debug-id-sourcemaps/package.json         | 3 +++
 .../test-applications/node-express-send-to-sentry/package.json | 3 +++
 .../test-applications/react-send-to-sentry/package.json        | 3 +++
 5 files changed, 15 insertions(+)

diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-astro/package.json b/dev-packages/e2e-tests/test-applications/cloudflare-astro/package.json
index fd636122d590..d2fc66736b4f 100644
--- a/dev-packages/e2e-tests/test-applications/cloudflare-astro/package.json
+++ b/dev-packages/e2e-tests/test-applications/cloudflare-astro/package.json
@@ -26,5 +26,8 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optional": true
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-workers/package.json b/dev-packages/e2e-tests/test-applications/cloudflare-workers/package.json
index bb01c0b8a8ad..f7fd08df85f9 100644
--- a/dev-packages/e2e-tests/test-applications/cloudflare-workers/package.json
+++ b/dev-packages/e2e-tests/test-applications/cloudflare-workers/package.json
@@ -24,5 +24,8 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optional": true
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/debug-id-sourcemaps/package.json b/dev-packages/e2e-tests/test-applications/debug-id-sourcemaps/package.json
index 9295b7997ee6..6451610ffe86 100644
--- a/dev-packages/e2e-tests/test-applications/debug-id-sourcemaps/package.json
+++ b/dev-packages/e2e-tests/test-applications/debug-id-sourcemaps/package.json
@@ -19,5 +19,8 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optional": true
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/package.json
index 96f61837c597..5a1b9b7d8300 100644
--- a/dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/package.json
+++ b/dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/package.json
@@ -24,5 +24,8 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optional": true
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/react-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/react-send-to-sentry/package.json
index 95b9c3bd78b4..a6ba509bc09a 100644
--- a/dev-packages/e2e-tests/test-applications/react-send-to-sentry/package.json
+++ b/dev-packages/e2e-tests/test-applications/react-send-to-sentry/package.json
@@ -47,5 +47,8 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optional": true
   }
 }

From 376b1a8e6ebdba31a634209bd2b1560e907f1dee Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 09:03:31 +0100
Subject: [PATCH 05/22] Generate E2E matrix programmatically

---
 .github/workflows/build.yml                   | 196 +++---------------
 .../lib/ciShouldSkipTestApplication.ts        |  72 -------
 dev-packages/e2e-tests/lib/getTestMatrix.ts   | 123 +++++++++++
 dev-packages/e2e-tests/package.json           |   3 +-
 .../create-next-app/package.json              |   8 +
 .../create-react-app/package.json             |   8 +
 .../create-remix-app-legacy/package.json      |   8 +
 .../create-remix-app/package.json             |   8 +
 .../test-applications/nextjs-13/package.json  |  12 ++
 .../test-applications/nextjs-14/package.json  |  12 ++
 .../test-applications/nextjs-15/package.json  |  12 ++
 .../nextjs-app-dir/package.json               |  18 ++
 .../nextjs-turbo/package.json                 |  12 ++
 .../react-router-6/package.json               |   8 +
 14 files changed, 260 insertions(+), 240 deletions(-)
 delete mode 100644 dev-packages/e2e-tests/lib/ciShouldSkipTestApplication.ts
 create mode 100644 dev-packages/e2e-tests/lib/getTestMatrix.ts

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 1e223cb424f9..5ad024ebbc35 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -801,7 +801,15 @@ jobs:
     needs: [job_get_metadata, job_build, job_compile_bindings_profiling_node]
     runs-on: ubuntu-20.04-large-js
     timeout-minutes: 15
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+      matrix-optional: ${{ steps.matrix-optional.outputs.matrix }}
     steps:
+      - name: Check out base commit (${{ github.event.pull_request.base.sha }})
+        uses: actions/checkout@v4
+        if: github.event_name == 'pull_request'
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v4
         with:
@@ -851,6 +859,18 @@ jobs:
           path: ${{ github.workspace }}/packages/*/*.tgz
           key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
 
+      - name: Determine which E2E test applications should be run
+        id: matrix
+        if: github.event_name == 'pull_request'
+        run: yarn --silent test:build-matrix --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
+        working-directory: dev-packages/e2e-tests
+
+      - name: Determine which optional E2E test applications should be run
+        id: matrix-optional
+        if: github.event_name == 'pull_request'
+        run: yarn --silent test:build-matrix-optional --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
+        working-directory: dev-packages/e2e-tests
+
   job_e2e_tests:
     name: E2E ${{ matrix.label || matrix.test-application }} Test
     # We need to add the `always()` check here because the previous step has this as well :(
@@ -870,100 +890,7 @@ jobs:
       E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
     strategy:
       fail-fast: false
-      matrix:
-        is_dependabot:
-          - ${{ github.actor == 'dependabot[bot]' }}
-        test-application:
-          [
-            'angular-17',
-            'angular-18',
-            'astro-4',
-            'aws-lambda-layer-cjs',
-            'aws-serverless-esm',
-            'node-express',
-            'create-react-app',
-            'create-next-app',
-            'create-remix-app',
-            'create-remix-app-legacy',
-            'create-remix-app-v2',
-            'create-remix-app-v2-legacy',
-            'create-remix-app-express',
-            'create-remix-app-express-legacy',
-            'create-remix-app-express-vite-dev',
-            'default-browser',
-            'node-express-esm-loader',
-            'node-express-esm-preload',
-            'node-express-esm-without-loader',
-            'node-express-cjs-preload',
-            'node-otel-sdk-node',
-            'node-otel-custom-sampler',
-            'node-otel-without-tracing',
-            'ember-classic',
-            'ember-embroider',
-            'nextjs-app-dir',
-            'nextjs-13',
-            'nextjs-14',
-            'nextjs-15',
-            'nextjs-turbo',
-            'nextjs-t3',
-            'react-17',
-            'react-19',
-            'react-create-hash-router',
-            'react-router-6-use-routes',
-            'react-router-5',
-            'react-router-6',
-            'solid',
-            'solidstart',
-            'solidstart-spa',
-            'svelte-5',
-            'sveltekit',
-            'sveltekit-2',
-            'sveltekit-2-svelte-5',
-            'sveltekit-2-twp',
-            'tanstack-router',
-            'generic-ts3.8',
-            'node-fastify',
-            'node-fastify-5',
-            'node-hapi',
-            'node-nestjs-basic',
-            'node-nestjs-distributed-tracing',
-            'nestjs-basic',
-            'nestjs-8',
-            'nestjs-distributed-tracing',
-            'nestjs-with-submodules',
-            'nestjs-with-submodules-decorator',
-            'nestjs-basic-with-graphql',
-            'nestjs-graphql',
-            'node-exports-test-app',
-            'node-koa',
-            'node-connect',
-            'nuxt-3',
-            'nuxt-3-min',
-            'nuxt-4',
-            'vue-3',
-            'webpack-4',
-            'webpack-5'
-          ]
-        build-command:
-          - false
-        label:
-          - false
-        # Add any variations of a test app here
-        # You should provide an alternate build-command as well as a matching label
-        include:
-          - test-application: 'create-react-app'
-            build-command: 'test:build-ts3.8'
-            label: 'create-react-app (TS 3.8)'
-          - test-application: 'react-router-6'
-            build-command: 'test:build-ts3.8'
-            label: 'react-router-6 (TS 3.8)'
-          - test-application: 'create-next-app'
-            build-command: 'test:build-13'
-            label: 'create-next-app (next@13)'
-          - test-application: 'nextjs-app-dir'
-            build-command: 'test:build-13'
-            label: 'nextjs-app-dir (next@13)'
-
+      matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix) }}
     steps:
       - name: Check out base commit (${{ github.event.pull_request.base.sha }})
         uses: actions/checkout@v4
@@ -999,47 +926,40 @@ jobs:
       - name: Restore tarball cache
         uses: actions/cache/restore@v4
         id: restore-tarball-cache
-        if: steps.should-skip.outputs.SKIP != 'true'
         with:
           path: ${{ github.workspace }}/packages/*/*.tgz
           key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
 
       - name: Build tarballs if not cached
-        if: steps.restore-tarball-cache.outputs.cache-hit != 'true' && steps.should-skip.outputs.SKIP != 'true'
+        if: steps.restore-tarball-cache.outputs.cache-hit != 'true'
         run: yarn build:tarball
 
       - name: Install Playwright
-        if: steps.should-skip.outputs.SKIP != 'true'
         uses: ./.github/actions/install-playwright
         with:
           browsers: chromium
 
       - name: Get node version
-        if: steps.should-skip.outputs.SKIP != 'true'
         id: versions
         run: |
           echo "echo node=$(jq -r '.volta.node' dev-packages/e2e-tests/package.json)" >> $GITHUB_OUTPUT
 
       - name: Validate Verdaccio
-        if: steps.should-skip.outputs.SKIP != 'true'
         run: yarn test:validate
         working-directory: dev-packages/e2e-tests
 
       - name: Prepare Verdaccio
-        if: steps.should-skip.outputs.SKIP != 'true'
         run: yarn test:prepare
         working-directory: dev-packages/e2e-tests
         env:
           E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
 
       - name: Build E2E app
-        if: steps.should-skip.outputs.SKIP != 'true'
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 7
         run: pnpm ${{ matrix.build-command || 'test:build' }}
 
       - name: Run E2E test
-        if: steps.should-skip.outputs.SKIP != 'true'
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 10
         run: pnpm test:assert
@@ -1054,13 +974,13 @@ jobs:
           retention-days: 7
 
       - name: Pre-process E2E Test Dumps
-        if: always() && steps.should-skip.outputs.SKIP != 'true'
+        if: always()
         run: |
           node ./scripts/normalize-e2e-test-dump-transaction-events.js
 
       - name: Upload E2E Test Event Dumps
         uses: actions/upload-artifact@v4
-        if: always() && steps.should-skip.outputs.SKIP != 'true'
+        if: always()
         with:
           name: E2E Test Dump (${{ matrix.label || matrix.test-application }})
           path: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/event-dumps
@@ -1069,7 +989,7 @@ jobs:
           if-no-files-found: ignore
 
       - name: Upload test results to Codecov
-        if: cancelled() == false && steps.should-skip.outputs.SKIP != 'true'
+        if: cancelled() == false
         continue-on-error: true
         uses: codecov/test-results-action@v1
         with:
@@ -1099,58 +1019,7 @@ jobs:
       E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
     strategy:
       fail-fast: false
-      matrix:
-        test-application:
-          [
-            'cloudflare-astro',
-            'cloudflare-workers',
-            'react-send-to-sentry',
-            'node-express-send-to-sentry',
-            'debug-id-sourcemaps',
-          ]
-        build-command:
-          - false
-        assert-command:
-          - false
-        label:
-          - false
-        include:
-          - test-application: 'create-remix-app'
-            assert-command: 'test:assert-sourcemaps'
-            label: 'create-remix-app (sourcemaps)'
-          - test-application: 'create-remix-app-legacy'
-            assert-command: 'test:assert-sourcemaps'
-            label: 'create-remix-app-legacy (sourcemaps)'
-          - test-application: 'nextjs-app-dir'
-            build-command: 'test:build-canary'
-            label: 'nextjs-app-dir (canary)'
-          - test-application: 'nextjs-app-dir'
-            build-command: 'test:build-latest'
-            label: 'nextjs-app-dir (latest)'
-          - test-application: 'nextjs-13'
-            build-command: 'test:build-canary'
-            label: 'nextjs-13 (canary)'
-          - test-application: 'nextjs-13'
-            build-command: 'test:build-latest'
-            label: 'nextjs-13 (latest)'
-          - test-application: 'nextjs-14'
-            build-command: 'test:build-canary'
-            label: 'nextjs-14 (canary)'
-          - test-application: 'nextjs-14'
-            build-command: 'test:build-latest'
-            label: 'nextjs-14 (latest)'
-          - test-application: 'nextjs-15'
-            build-command: 'test:build-canary'
-            label: 'nextjs-15 (canary)'
-          - test-application: 'nextjs-15'
-            build-command: 'test:build-latest'
-            label: 'nextjs-15 (latest)'
-          - test-application: 'nextjs-turbo'
-            build-command: 'test:build-canary'
-            label: 'nextjs-turbo (canary)'
-          - test-application: 'nextjs-turbo'
-            build-command: 'test:build-latest'
-            label: 'nextjs-turbo (latest)'
+      matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix-optional) }}
 
     steps:
       - name: Check out base commit (${{ github.event.pull_request.base.sha }})
@@ -1181,7 +1050,6 @@ jobs:
         working-directory: dev-packages/e2e-tests
 
       - name: Restore tarball cache
-        if: steps.should-skip.outputs.SKIP != 'true'
         uses: actions/cache/restore@v4
         id: restore-tarball-cache
         with:
@@ -1193,49 +1061,43 @@ jobs:
         run: yarn build:tarball
 
       - name: Install Playwright
-        if: steps.should-skip.outputs.SKIP != 'true'
         uses: ./.github/actions/install-playwright
         with:
           browsers: chromium
 
       - name: Get node version
         id: versions
-        if: steps.should-skip.outputs.SKIP != 'true'
         run: |
           echo "echo node=$(jq -r '.volta.node' dev-packages/e2e-tests/package.json)" >> $GITHUB_OUTPUT
 
       - name: Validate Verdaccio
-        if: steps.should-skip.outputs.SKIP != 'true'
         run: yarn test:validate
         working-directory: dev-packages/e2e-tests
 
       - name: Prepare Verdaccio
-        if: steps.should-skip.outputs.SKIP != 'true'
         run: yarn test:prepare
         working-directory: dev-packages/e2e-tests
         env:
           E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
 
       - name: Build E2E app
-        if: steps.should-skip.outputs.SKIP != 'true'
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 7
         run: pnpm ${{ matrix.build-command || 'test:build' }}
 
       - name: Run E2E test
-        if: steps.should-skip.outputs.SKIP != 'true'
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 10
         run: pnpm ${{ matrix.assert-command || 'test:assert' }}
 
       - name: Pre-process E2E Test Dumps
-        if: always() && steps.should-skip.outputs.SKIP != 'true'
+        if: always()
         run: |
           node ./scripts/normalize-e2e-test-dump-transaction-events.js
 
       - name: Upload E2E Test Event Dumps
         uses: actions/upload-artifact@v4
-        if: always() && steps.should-skip.outputs.SKIP != 'true'
+        if: always()
         with:
           name: E2E Test Dump (${{ matrix.label || matrix.test-application }})
           path: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/event-dumps
@@ -1245,7 +1107,7 @@ jobs:
 
       - name: Deploy Astro to Cloudflare
         uses: cloudflare/pages-action@v1
-        if: matrix.test-application == 'cloudflare-astro' && steps.should-skip.outputs.SKIP != 'true'
+        if: matrix.test-application == 'cloudflare-astro'
         with:
           apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
diff --git a/dev-packages/e2e-tests/lib/ciShouldSkipTestApplication.ts b/dev-packages/e2e-tests/lib/ciShouldSkipTestApplication.ts
deleted file mode 100644
index bf112b4919f6..000000000000
--- a/dev-packages/e2e-tests/lib/ciShouldSkipTestApplication.ts
+++ /dev/null
@@ -1,72 +0,0 @@
-import { execSync } from 'child_process';
-import * as fs from 'fs';
-import * as path from 'path';
-
-/**
- * This logs SKIP=true or SKIP=false,
- * which can be used by GHA to determine if the given test application should be skipped or not.
- * This uses nx to check if any of the sentry dependencies of the test-application have been affected.
- */
-function ciShouldSkipTestApplication(): void {
-  // Allow to run a single app only via `yarn test:run <app-name>`
-  const appName = process.argv[2];
-
-  const args: Record<string, string> = {};
-  process.argv
-    .slice(2)
-    .filter(arg => arg.startsWith('--') && arg.includes('='))
-    .forEach(arg => {
-      const [part1, part2] = arg.split('=') as [string, string];
-      const argName = part1.replace('--', '');
-      const argValue = part2;
-      args[argName] = argValue;
-    });
-
-  // We default to `develop` as base, if none is specified
-  // head has a correct default value anyhow
-  const { base = 'develop', head } = args;
-
-  if (!appName) {
-    throw new Error('Please provide the app name as the first argument');
-  }
-
-  const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json');
-
-  if (!fs.existsSync(fullPath)) {
-    throw new Error(`The app ${appName} does not exist`);
-  }
-
-  const packageJson = JSON.parse(fs.readFileSync(fullPath, 'utf8')) as {
-    dependencies?: { [key: string]: string };
-    devDependencies?: { [key: string]: string };
-  };
-
-  const dependencies = {
-    ...packageJson.devDependencies,
-    ...packageJson.dependencies,
-  };
-
-  const sentryDependencies = Object.keys(dependencies).filter(key => key.startsWith('@sentry/'));
-
-  const additionalArgs = [];
-  if (base) {
-    additionalArgs.push(`--base=${base}`);
-  }
-  if (head) {
-    additionalArgs.push(`--head=${head}`);
-  }
-
-  const affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`)
-    .toString()
-    .split('\n')
-    .map(line => line.trim());
-
-  // If one of the sentry dependencies is affected, this test should be run
-  const affected = sentryDependencies.some(dep => affectedProjects.includes(dep));
-
-  // This is used by CI to determine if steps should be skipped or not
-  // eslint-disable-next-line no-console
-  console.log(`SKIP=${affected ? 'false' : 'true'}`);
-}
-
-ciShouldSkipTestApplication();
diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.ts b/dev-packages/e2e-tests/lib/getTestMatrix.ts
new file mode 100644
index 000000000000..a2e283fe629a
--- /dev/null
+++ b/dev-packages/e2e-tests/lib/getTestMatrix.ts
@@ -0,0 +1,123 @@
+import { sync as globSync } from 'glob';
+import { execSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+
+interface MatrixInclude {
+  'test-application': string;
+  'build-command'?: string;
+  'assert-command'?: string;
+  label?: string;
+}
+
+function run(): void {
+  const args: Record<string, string> = {};
+  process.argv
+    .slice(2)
+    .filter(arg => arg.startsWith('--') && arg.includes('='))
+    .forEach(arg => {
+      const [part1, part2] = arg.split('=') as [string, string];
+      const argName = part1.replace('--', '');
+      const argValue = part2;
+      args[argName] = argValue;
+    });
+
+  // We default to `develop` as base, if none is specified
+  // head has a correct default value anyhow
+  const { base = 'develop', head, optional = 'false' } = args;
+
+  const testApplications = globSync('*', { cwd: `${__dirname}/../test-applications/` });
+
+  const additionalArgs = [];
+  if (base) {
+    additionalArgs.push(`--base=${base}`);
+  }
+  if (head) {
+    additionalArgs.push(`--head=${head}`);
+  }
+
+  const affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`)
+    .toString()
+    .split('\n')
+    .map(line => line.trim())
+    .filter(Boolean);
+
+  const includedTestApplications = testApplications.filter(testApp => {
+    const sentryDependencies = getSentryDependencies(testApp);
+    return sentryDependencies.some(dep => affectedProjects.includes(dep));
+  });
+
+  const optionalMode = optional === 'true';
+  const includes: MatrixInclude[] = [];
+
+  includedTestApplications.forEach(testApp => {
+    addIncludesForTestApp(testApp, includes, { optionalMode });
+  });
+
+  // We print this to the output, so the GHA can use it for the matrix
+  // eslint-disable-next-line no-console
+  console.log(`matrix=${JSON.stringify({ include: includes })}`);
+}
+
+function addIncludesForTestApp(
+  testApp: string,
+  includes: MatrixInclude[],
+  { optionalMode }: { optionalMode: boolean },
+): void {
+  const packageJson = getPackageJson(testApp);
+
+  // this means something went wrong
+  if (!packageJson.name) {
+    return;
+  }
+
+  const isOptional = packageJson.sentryTest?.optional || false;
+  const variants = (optionalMode ? packageJson.sentryTest?.optionalVariants : packageJson.sentryTest?.variants) || [];
+
+  // Add the basic test-application itself, if it is in the current mode
+  if (optionalMode === isOptional) {
+    includes.push({
+      'test-application': testApp,
+    });
+  }
+
+  variants.forEach(variant => {
+    includes.push({
+      'test-application': testApp,
+      ...variant,
+    });
+  });
+}
+
+function getSentryDependencies(appName: string): string[] {
+  const packageJson = getPackageJson(appName);
+
+  const dependencies = {
+    ...packageJson.devDependencies,
+    ...packageJson.dependencies,
+  };
+
+  return Object.keys(dependencies).filter(key => key.startsWith('@sentry/'));
+}
+
+function getPackageJson(appName: string): {
+  name?: string;
+  dependencies?: { [key: string]: string };
+  devDependencies?: { [key: string]: string };
+  sentryTest?: {
+    optional?: boolean;
+    variants?: Partial<MatrixInclude>[];
+    optionalVariants?: Partial<MatrixInclude>[];
+  };
+} {
+  const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json');
+
+  // This can happen if you e.g. have a leftover directory in test-applications
+  if (!fs.existsSync(fullPath)) {
+    return {};
+  }
+
+  return JSON.parse(fs.readFileSync(fullPath, 'utf8'));
+}
+
+run();
diff --git a/dev-packages/e2e-tests/package.json b/dev-packages/e2e-tests/package.json
index d982aad9d9fa..3858fe5b011d 100644
--- a/dev-packages/e2e-tests/package.json
+++ b/dev-packages/e2e-tests/package.json
@@ -14,7 +14,8 @@
     "test:prepare": "ts-node prepare.ts",
     "test:validate": "run-s test:validate-configuration test:validate-test-app-setups",
     "clean": "rimraf tmp node_modules pnpm-lock.yaml && yarn clean:test-applications",
-    "test:should-skip": "ts-node ./lib/ciShouldSkipTestApplication.ts",
+    "ci:build-matrix": "ts-node ./lib/ciBuildMatrix.ts",
+    "ci:build-matrix-optional": "ts-node ./lib/ciBuildMatrix.ts --optional=true",
     "clean:test-applications": "rimraf --glob test-applications/**/{node_modules,dist,build,.next,.sveltekit,pnpm-lock.yaml} .last-run.json && pnpm store prune"
   },
   "devDependencies": {
diff --git a/dev-packages/e2e-tests/test-applications/create-next-app/package.json b/dev-packages/e2e-tests/test-applications/create-next-app/package.json
index 9c240942b3b7..316fb561cdf3 100644
--- a/dev-packages/e2e-tests/test-applications/create-next-app/package.json
+++ b/dev-packages/e2e-tests/test-applications/create-next-app/package.json
@@ -27,5 +27,13 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "variants": [
+      {
+        "build-command": "test:build-13",
+        "label": "create-next-app (next@13)"
+      }
+    ]
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/create-react-app/package.json b/dev-packages/e2e-tests/test-applications/create-react-app/package.json
index ce3471d2a7d1..916a17260a2a 100644
--- a/dev-packages/e2e-tests/test-applications/create-react-app/package.json
+++ b/dev-packages/e2e-tests/test-applications/create-react-app/package.json
@@ -43,5 +43,13 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "variants": [
+      {
+        "build-command": "test:build-ts3.8",
+        "label": "create-react-app (TS 3.8)"
+      }
+    ]
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-legacy/package.json b/dev-packages/e2e-tests/test-applications/create-remix-app-legacy/package.json
index 4b7c2c162b86..6b50ddc96b4a 100644
--- a/dev-packages/e2e-tests/test-applications/create-remix-app-legacy/package.json
+++ b/dev-packages/e2e-tests/test-applications/create-remix-app-legacy/package.json
@@ -36,5 +36,13 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optionalVariants": [
+      {
+        "assert-command": "test:assert-sourcemaps",
+        "label": "create-remix-app-legacy (sourcemaps)"
+      }
+    ]
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app/package.json b/dev-packages/e2e-tests/test-applications/create-remix-app/package.json
index db5c5b474ef0..4850fedf1e5d 100644
--- a/dev-packages/e2e-tests/test-applications/create-remix-app/package.json
+++ b/dev-packages/e2e-tests/test-applications/create-remix-app/package.json
@@ -36,5 +36,13 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optionalVariants": [
+      {
+        "assert-command": "test:assert-sourcemaps",
+        "label": "create-remix-app (sourcemaps)"
+      }
+    ]
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/nextjs-13/package.json b/dev-packages/e2e-tests/test-applications/nextjs-13/package.json
index 3e7a0ac88266..c56d7c6ed204 100644
--- a/dev-packages/e2e-tests/test-applications/nextjs-13/package.json
+++ b/dev-packages/e2e-tests/test-applications/nextjs-13/package.json
@@ -41,5 +41,17 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optionalVariants": [
+      {
+        "build-command": "test:build-canary",
+        "label": "nextjs-13 (canary)"
+      },
+      {
+        "build-command": "test:build-latest",
+        "label": "nextjs-13 (latest)"
+      }
+    ]
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/nextjs-14/package.json b/dev-packages/e2e-tests/test-applications/nextjs-14/package.json
index bbda1b0144cc..c8fcba03410d 100644
--- a/dev-packages/e2e-tests/test-applications/nextjs-14/package.json
+++ b/dev-packages/e2e-tests/test-applications/nextjs-14/package.json
@@ -41,5 +41,17 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optionalVariants": [
+      {
+        "build-command": "test:build-canary",
+        "label": "nextjs-14 (canary)"
+      },
+      {
+        "build-command": "test:build-latest",
+        "label": "nextjs-14 (latest)"
+      }
+    ]
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/nextjs-15/package.json b/dev-packages/e2e-tests/test-applications/nextjs-15/package.json
index 04033e0362b2..1c5754bd66da 100644
--- a/dev-packages/e2e-tests/test-applications/nextjs-15/package.json
+++ b/dev-packages/e2e-tests/test-applications/nextjs-15/package.json
@@ -42,5 +42,17 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optionalVariants": [
+      {
+        "build-command": "test:build-canary",
+        "label": "nextjs-15 (canary)"
+      },
+      {
+        "build-command": "test:build-latest",
+        "label": "nextjs-15 (latest)"
+      }
+    ]
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/nextjs-app-dir/package.json b/dev-packages/e2e-tests/test-applications/nextjs-app-dir/package.json
index 8ccad25e6ab4..b0bc898d9bd1 100644
--- a/dev-packages/e2e-tests/test-applications/nextjs-app-dir/package.json
+++ b/dev-packages/e2e-tests/test-applications/nextjs-app-dir/package.json
@@ -44,5 +44,23 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "variants": [
+      {
+        "build-command": "test:build-13",
+        "label": "nextjs-app-dir (next@13)"
+      }
+    ],
+    "optionalVariants": [
+      {
+        "build-command": "test:build-canary",
+        "label": "nextjs-app-dir (canary)"
+      },
+      {
+        "build-command": "test:build-latest",
+        "label": "nextjs-app-dir (latest)"
+      }
+    ]
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/nextjs-turbo/package.json b/dev-packages/e2e-tests/test-applications/nextjs-turbo/package.json
index 900e0b5b2efc..9cf05720fc28 100644
--- a/dev-packages/e2e-tests/test-applications/nextjs-turbo/package.json
+++ b/dev-packages/e2e-tests/test-applications/nextjs-turbo/package.json
@@ -45,5 +45,17 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optionalVariants": [
+      {
+        "build-command": "test:build-canary",
+        "label": "nextjs-turbo (canary)"
+      },
+      {
+        "build-command": "test:build-latest",
+        "label": "nextjs-turbo (latest)"
+      }
+    ]
   }
 }
diff --git a/dev-packages/e2e-tests/test-applications/react-router-6/package.json b/dev-packages/e2e-tests/test-applications/react-router-6/package.json
index 5171a89eadb3..b3ef37f6bc4a 100644
--- a/dev-packages/e2e-tests/test-applications/react-router-6/package.json
+++ b/dev-packages/e2e-tests/test-applications/react-router-6/package.json
@@ -51,5 +51,13 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "variants": [
+      {
+        "build-command": "test:build-ts3.8",
+        "label": "react-router-6 (TS 3.8)"
+      }
+    ]
   }
 }

From 5023b05412e756476bf3f0e5dbc370d24d1850ab Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 09:03:52 +0100
Subject: [PATCH 06/22] WIP test browser change

---
 packages/browser/src/index.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/browser/src/index.ts b/packages/browser/src/index.ts
index 56724b926c5b..e59cc8b50c24 100644
--- a/packages/browser/src/index.ts
+++ b/packages/browser/src/index.ts
@@ -75,3 +75,5 @@ export type { Span } from '@sentry/types';
 export { makeBrowserOfflineTransport } from './transports/offline';
 export { browserProfilingIntegration } from './profiling/integration';
 export { spotlightBrowserIntegration } from './integrations/spotlight';
+
+export const testChange = 'ahaha';

From c89506529ab0db65509f967b148d56d2c072db89 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 09:07:55 +0100
Subject: [PATCH 07/22] fix linting

---
 dev-packages/e2e-tests/lib/getTestMatrix.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.ts b/dev-packages/e2e-tests/lib/getTestMatrix.ts
index a2e283fe629a..f905e9266fbb 100644
--- a/dev-packages/e2e-tests/lib/getTestMatrix.ts
+++ b/dev-packages/e2e-tests/lib/getTestMatrix.ts
@@ -1,7 +1,7 @@
-import { sync as globSync } from 'glob';
 import { execSync } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
+import { sync as globSync } from 'glob';
 
 interface MatrixInclude {
   'test-application': string;

From 62c1d68e28c65def6cf66c15c8927cfce83cf1b8 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 09:18:34 +0100
Subject: [PATCH 08/22] handle non-PRs

---
 .github/workflows/build.yml                 | 12 +----
 dev-packages/e2e-tests/lib/getTestMatrix.ts | 58 +++++++++++++--------
 2 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5ad024ebbc35..9162ecfaf78b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -861,13 +861,11 @@ jobs:
 
       - name: Determine which E2E test applications should be run
         id: matrix
-        if: github.event_name == 'pull_request'
         run: yarn --silent test:build-matrix --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
         working-directory: dev-packages/e2e-tests
 
       - name: Determine which optional E2E test applications should be run
         id: matrix-optional
-        if: github.event_name == 'pull_request'
         run: yarn --silent test:build-matrix-optional --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
         working-directory: dev-packages/e2e-tests
 
@@ -966,7 +964,7 @@ jobs:
 
       - name: Upload Playwright Traces
         uses: actions/upload-artifact@v4
-        if: failure() &&  steps.should-skip.outputs.SKIP != 'true'
+        if: failure()
         with:
           name: playwright-traces-job_e2e_playwright_tests-${{ matrix.test-application}}
           path: dev-packages/e2e-tests/test-applications/${{ matrix.test-application}}/test-results
@@ -1043,12 +1041,6 @@ jobs:
         with:
           dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }}
 
-      - name: Determine if test app should be run
-        id: should-skip
-        if: github.event_name == 'pull_request'
-        run: yarn --silent test:should-skip ${{ matrix.test-application }} --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
-        working-directory: dev-packages/e2e-tests
-
       - name: Restore tarball cache
         uses: actions/cache/restore@v4
         id: restore-tarball-cache
@@ -1057,7 +1049,7 @@ jobs:
           key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
 
       - name: Build tarballs if not cached
-        if: steps.restore-tarball-cache.outputs.cache-hit != 'true' &&  steps.should-skip.outputs.SKIP != 'true'
+        if: steps.restore-tarball-cache.outputs.cache-hit != 'true'
         run: yarn build:tarball
 
       - name: Install Playwright
diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.ts b/dev-packages/e2e-tests/lib/getTestMatrix.ts
index f905e9266fbb..22679c271e25 100644
--- a/dev-packages/e2e-tests/lib/getTestMatrix.ts
+++ b/dev-packages/e2e-tests/lib/getTestMatrix.ts
@@ -10,6 +10,15 @@ interface MatrixInclude {
   label?: string;
 }
 
+/**
+ * This methods generates a matrix for the GitHub Actions workflow to run the E2E tests.
+ * It checks which test applications are affected by the current changes in the PR and then generates a matrix
+ * including all test apps that have at least one dependency that was changed in the PR.
+ * If no `--base=xxx` is provided, it will output all test applications.
+ *
+ * If `--optional=true` is set, it will generate a matrix of optional test applications only.
+ * Otherwise, these will be skipped.
+ */
 function run(): void {
   const args: Record<string, string> = {};
   process.argv
@@ -22,30 +31,15 @@ function run(): void {
       args[argName] = argValue;
     });
 
-  // We default to `develop` as base, if none is specified
-  // head has a correct default value anyhow
-  const { base = 'develop', head, optional = 'false' } = args;
+  const { base, head, optional = 'false' } = args;
 
   const testApplications = globSync('*', { cwd: `${__dirname}/../test-applications/` });
 
-  const additionalArgs = [];
-  if (base) {
-    additionalArgs.push(`--base=${base}`);
-  }
-  if (head) {
-    additionalArgs.push(`--head=${head}`);
-  }
-
-  const affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`)
-    .toString()
-    .split('\n')
-    .map(line => line.trim())
-    .filter(Boolean);
-
-  const includedTestApplications = testApplications.filter(testApp => {
-    const sentryDependencies = getSentryDependencies(testApp);
-    return sentryDependencies.some(dep => affectedProjects.includes(dep));
-  });
+  // If `--base=xxx` is defined, we only want to get test applications changed since that base
+  // Else, we take all test applications (e.g. on push)
+  const includedTestApplications = base
+    ? getAffectedTestApplications(testApplications, { base, head })
+    : testApplications;
 
   const optionalMode = optional === 'true';
   const includes: MatrixInclude[] = [];
@@ -121,3 +115,25 @@ function getPackageJson(appName: string): {
 }
 
 run();
+
+function getAffectedTestApplications(
+  testApplications: string[],
+  { base = 'develop', head }: { base?: string; head?: string },
+): string[] {
+  const additionalArgs = [`--base=${base}`];
+
+  if (head) {
+    additionalArgs.push(`--head=${head}`);
+  }
+
+  const affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`)
+    .toString()
+    .split('\n')
+    .map(line => line.trim())
+    .filter(Boolean);
+
+  return testApplications.filter(testApp => {
+    const sentryDependencies = getSentryDependencies(testApp);
+    return sentryDependencies.some(dep => affectedProjects.includes(dep));
+  });
+}

From b424ba4b7a720cfae39c88becab5d02b65b93aae Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 09:22:33 +0100
Subject: [PATCH 09/22] fix it

---
 .github/workflows/build.yml         | 4 ++--
 dev-packages/e2e-tests/package.json | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 9162ecfaf78b..79bf3f381a08 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -861,12 +861,12 @@ jobs:
 
       - name: Determine which E2E test applications should be run
         id: matrix
-        run: yarn --silent test:build-matrix --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
+        run: yarn --silent ci:build-matrix --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
         working-directory: dev-packages/e2e-tests
 
       - name: Determine which optional E2E test applications should be run
         id: matrix-optional
-        run: yarn --silent test:build-matrix-optional --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
+        run: yarn --silent ci:build-matrix-optional --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
         working-directory: dev-packages/e2e-tests
 
   job_e2e_tests:
diff --git a/dev-packages/e2e-tests/package.json b/dev-packages/e2e-tests/package.json
index 3858fe5b011d..54b0388807c7 100644
--- a/dev-packages/e2e-tests/package.json
+++ b/dev-packages/e2e-tests/package.json
@@ -14,8 +14,8 @@
     "test:prepare": "ts-node prepare.ts",
     "test:validate": "run-s test:validate-configuration test:validate-test-app-setups",
     "clean": "rimraf tmp node_modules pnpm-lock.yaml && yarn clean:test-applications",
-    "ci:build-matrix": "ts-node ./lib/ciBuildMatrix.ts",
-    "ci:build-matrix-optional": "ts-node ./lib/ciBuildMatrix.ts --optional=true",
+    "ci:build-matrix": "ts-node ./lib/getTestMatrix.ts",
+    "ci:build-matrix-optional": "ts-node ./lib/getTestMatrix.ts --optional=true",
     "clean:test-applications": "rimraf --glob test-applications/**/{node_modules,dist,build,.next,.sveltekit,pnpm-lock.yaml} .last-run.json && pnpm store prune"
   },
   "devDependencies": {

From 07c67355981c620f3844dc5fa8b34eaa353fcf9c Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 10:58:03 +0100
Subject: [PATCH 10/22] fix remaining thing, oops

---
 .github/workflows/build.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 79bf3f381a08..fd5ef9698893 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -915,12 +915,6 @@ jobs:
         with:
           dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }}
 
-      - name: Determine if test app should be run
-        id: should-skip
-        if: github.event_name == 'pull_request'
-        run: yarn --silent test:should-skip ${{ matrix.test-application }} --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
-        working-directory: dev-packages/e2e-tests
-
       - name: Restore tarball cache
         uses: actions/cache/restore@v4
         id: restore-tarball-cache

From 0d161a7d256b2d4fc580e3b7886d2a2aed23e539 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 11:20:29 +0100
Subject: [PATCH 11/22] fix test apps without name

---
 dev-packages/e2e-tests/lib/getTestMatrix.ts | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.ts b/dev-packages/e2e-tests/lib/getTestMatrix.ts
index 22679c271e25..f9e4293d66f1 100644
--- a/dev-packages/e2e-tests/lib/getTestMatrix.ts
+++ b/dev-packages/e2e-tests/lib/getTestMatrix.ts
@@ -61,7 +61,7 @@ function addIncludesForTestApp(
   const packageJson = getPackageJson(testApp);
 
   // this means something went wrong
-  if (!packageJson.name) {
+  if (!packageJson) {
     return;
   }
 
@@ -84,18 +84,17 @@ function addIncludesForTestApp(
 }
 
 function getSentryDependencies(appName: string): string[] {
-  const packageJson = getPackageJson(appName);
+  const packageJson = getPackageJson(appName) || {};
 
   const dependencies = {
     ...packageJson.devDependencies,
     ...packageJson.dependencies,
   };
 
-  return Object.keys(dependencies).filter(key => key.startsWith('@sentry/'));
+  return Object.keys(dependencies).filter(key => key.startsWith('@sentry'));
 }
 
 function getPackageJson(appName: string): {
-  name?: string;
   dependencies?: { [key: string]: string };
   devDependencies?: { [key: string]: string };
   sentryTest?: {
@@ -103,12 +102,12 @@ function getPackageJson(appName: string): {
     variants?: Partial<MatrixInclude>[];
     optionalVariants?: Partial<MatrixInclude>[];
   };
-} {
+} | undefined {
   const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json');
 
   // This can happen if you e.g. have a leftover directory in test-applications
   if (!fs.existsSync(fullPath)) {
-    return {};
+    return undefined;
   }
 
   return JSON.parse(fs.readFileSync(fullPath, 'utf8'));

From 9324aded4872b0595bfc17b070896b810bab8075 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 11:30:40 +0100
Subject: [PATCH 12/22] fix linting

---
 dev-packages/e2e-tests/lib/getTestMatrix.ts | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.ts b/dev-packages/e2e-tests/lib/getTestMatrix.ts
index f9e4293d66f1..9c5b582acd73 100644
--- a/dev-packages/e2e-tests/lib/getTestMatrix.ts
+++ b/dev-packages/e2e-tests/lib/getTestMatrix.ts
@@ -94,15 +94,17 @@ function getSentryDependencies(appName: string): string[] {
   return Object.keys(dependencies).filter(key => key.startsWith('@sentry'));
 }
 
-function getPackageJson(appName: string): {
-  dependencies?: { [key: string]: string };
-  devDependencies?: { [key: string]: string };
-  sentryTest?: {
-    optional?: boolean;
-    variants?: Partial<MatrixInclude>[];
-    optionalVariants?: Partial<MatrixInclude>[];
-  };
-} | undefined {
+function getPackageJson(appName: string):
+  | {
+      dependencies?: { [key: string]: string };
+      devDependencies?: { [key: string]: string };
+      sentryTest?: {
+        optional?: boolean;
+        variants?: Partial<MatrixInclude>[];
+        optionalVariants?: Partial<MatrixInclude>[];
+      };
+    }
+  | undefined {
   const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json');
 
   // This can happen if you e.g. have a leftover directory in test-applications

From 64165eb933c309f17de76c2a99df2fa2a4e26912 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 11:31:03 +0100
Subject: [PATCH 13/22] Revert "WIP test browser change"

This reverts commit e91d64d3a5415ec7dac7fc5dc33580f9c2b330df.
---
 packages/browser/src/index.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/browser/src/index.ts b/packages/browser/src/index.ts
index e59cc8b50c24..56724b926c5b 100644
--- a/packages/browser/src/index.ts
+++ b/packages/browser/src/index.ts
@@ -75,5 +75,3 @@ export type { Span } from '@sentry/types';
 export { makeBrowserOfflineTransport } from './transports/offline';
 export { browserProfilingIntegration } from './profiling/integration';
 export { spotlightBrowserIntegration } from './integrations/spotlight';
-
-export const testChange = 'ahaha';

From 067bb111f1b69952a10d56c8131460193f4b0187 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 11:40:40 +0100
Subject: [PATCH 14/22] better handling for pushes

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fd5ef9698893..a15e983187da 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -861,12 +861,12 @@ jobs:
 
       - name: Determine which E2E test applications should be run
         id: matrix
-        run: yarn --silent ci:build-matrix --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
+        run: yarn --silent ci:build-matrix --base=${{ (github.event_name == 'pull_request' && github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT
         working-directory: dev-packages/e2e-tests
 
       - name: Determine which optional E2E test applications should be run
         id: matrix-optional
-        run: yarn --silent ci:build-matrix-optional --base=${{ github.event.pull_request.base.sha }} >> $GITHUB_OUTPUT
+        run: yarn --silent ci:build-matrix-optional --base=${{ (github.event_name == 'pull_request' && github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT
         working-directory: dev-packages/e2e-tests
 
   job_e2e_tests:

From 20cd6ef02e980a9c9ad528332d9529af8079f267 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 11:58:41 +0100
Subject: [PATCH 15/22] small fixes

---
 dev-packages/e2e-tests/lib/getTestMatrix.ts           | 11 +++++++++++
 .../test-applications/node-profiling/package.json     |  3 +++
 2 files changed, 14 insertions(+)

diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.ts b/dev-packages/e2e-tests/lib/getTestMatrix.ts
index 9c5b582acd73..80ee3cc12b65 100644
--- a/dev-packages/e2e-tests/lib/getTestMatrix.ts
+++ b/dev-packages/e2e-tests/lib/getTestMatrix.ts
@@ -65,9 +65,14 @@ function addIncludesForTestApp(
     return;
   }
 
+  const shouldSkip = packageJson.sentryTest?.skip || false;
   const isOptional = packageJson.sentryTest?.optional || false;
   const variants = (optionalMode ? packageJson.sentryTest?.optionalVariants : packageJson.sentryTest?.variants) || [];
 
+  if (shouldSkip) {
+    return;
+  }
+
   // Add the basic test-application itself, if it is in the current mode
   if (optionalMode === isOptional) {
     includes.push({
@@ -102,6 +107,7 @@ function getPackageJson(appName: string):
         optional?: boolean;
         variants?: Partial<MatrixInclude>[];
         optionalVariants?: Partial<MatrixInclude>[];
+        skip?: boolean;
       };
     }
   | undefined {
@@ -133,6 +139,11 @@ function getAffectedTestApplications(
     .map(line => line.trim())
     .filter(Boolean);
 
+  // If something in e2e tests themselves are changed, just run everything
+  if (affectedProjects.includes('@sentry-internal/e2e-tests')) {
+    return testApplications;
+  }
+
   return testApplications.filter(testApp => {
     const sentryDependencies = getSentryDependencies(testApp);
     return sentryDependencies.some(dep => affectedProjects.includes(dep));
diff --git a/dev-packages/e2e-tests/test-applications/node-profiling/package.json b/dev-packages/e2e-tests/test-applications/node-profiling/package.json
index cfe4e136b1c1..8aede827a1f3 100644
--- a/dev-packages/e2e-tests/test-applications/node-profiling/package.json
+++ b/dev-packages/e2e-tests/test-applications/node-profiling/package.json
@@ -22,5 +22,8 @@
   "devDependencies": {},
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "skip": true
   }
 }

From 9827c389602728de4b5aa697bb026636e26bc87c Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 11:44:38 +0100
Subject: [PATCH 16/22] remove unneeded

---
 .github/workflows/build.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index a15e983187da..e6c484b5a4f8 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -895,7 +895,6 @@ jobs:
         if: github.event_name == 'pull_request'
         with:
           ref: ${{ github.event.pull_request.base.sha }}
-
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v4
         with:
@@ -1014,11 +1013,6 @@ jobs:
       matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix-optional) }}
 
     steps:
-      - name: Check out base commit (${{ github.event.pull_request.base.sha }})
-        uses: actions/checkout@v4
-        if: github.event_name == 'pull_request'
-        with:
-          ref: ${{ github.event.pull_request.base.sha }}
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v4
         with:

From dc7dd913a43b16fcf7b18c8a7d0ac9d5e18dd65d Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 12:57:00 +0100
Subject: [PATCH 17/22] skip when no tests ???

---
 .github/workflows/build.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e6c484b5a4f8..441a5b0008c1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -873,7 +873,7 @@ jobs:
     name: E2E ${{ matrix.label || matrix.test-application }} Test
     # We need to add the `always()` check here because the previous step has this as well :(
     # See: https://github.com/actions/runner/issues/2205
-    if: always() && needs.job_e2e_prepare.result == 'success'
+    if: always() && needs.job_e2e_prepare.result == 'success' && fromJSON(needs.job_e2e_prepare.outputs.matrix).include.length > 0
     needs: [job_get_metadata, job_build, job_e2e_prepare]
     runs-on: ubuntu-20.04
     timeout-minutes: 15
@@ -994,6 +994,7 @@ jobs:
     # See: https://github.com/actions/runner/issues/2205
     if:
       always() && needs.job_e2e_prepare.result == 'success' &&
+      fromJson(needs.job_e2e_prepare.outputs.matrix-optional).include.length > 0 &&
       (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
       github.actor != 'dependabot[bot]'
     needs: [job_get_metadata, job_build, job_e2e_prepare]

From 2ab5a789314574953d004e265331586c20031edf Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 14:32:12 +0100
Subject: [PATCH 18/22] fix check

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 441a5b0008c1..8f46c02cbd6c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -873,7 +873,7 @@ jobs:
     name: E2E ${{ matrix.label || matrix.test-application }} Test
     # We need to add the `always()` check here because the previous step has this as well :(
     # See: https://github.com/actions/runner/issues/2205
-    if: always() && needs.job_e2e_prepare.result == 'success' && fromJSON(needs.job_e2e_prepare.outputs.matrix).include.length > 0
+    if: always() && needs.job_e2e_prepare.result == 'success' && needs.job_e2e_prepare.outputs.matrix != '{"include":[]}'
     needs: [job_get_metadata, job_build, job_e2e_prepare]
     runs-on: ubuntu-20.04
     timeout-minutes: 15
@@ -994,7 +994,7 @@ jobs:
     # See: https://github.com/actions/runner/issues/2205
     if:
       always() && needs.job_e2e_prepare.result == 'success' &&
-      fromJson(needs.job_e2e_prepare.outputs.matrix-optional).include.length > 0 &&
+      needs.job_e2e_prepare.outputs.matrix-optional != '{"include":[]}' &&
       (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
       github.actor != 'dependabot[bot]'
     needs: [job_get_metadata, job_build, job_e2e_prepare]

From 6fdad14d8e0fd3ed64a3bbb52132a45593eb3a93 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Thu, 14 Nov 2024 14:59:59 +0100
Subject: [PATCH 19/22] remove unneeded

---
 .github/workflows/build.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8f46c02cbd6c..72d572c01e5c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -890,11 +890,6 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix) }}
     steps:
-      - name: Check out base commit (${{ github.event.pull_request.base.sha }})
-        uses: actions/checkout@v4
-        if: github.event_name == 'pull_request'
-        with:
-          ref: ${{ github.event.pull_request.base.sha }}
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v4
         with:

From ec2a1a752979d18afb0ac1de7aa1ec1a95365f75 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Fri, 15 Nov 2024 09:38:48 +0100
Subject: [PATCH 20/22] PR feedback

---
 dev-packages/e2e-tests/lib/getTestMatrix.ts | 66 +++++++++++----------
 dev-packages/e2e-tests/package.json         |  2 +-
 yarn.lock                                   | 39 ++++--------
 3 files changed, 49 insertions(+), 58 deletions(-)

diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.ts b/dev-packages/e2e-tests/lib/getTestMatrix.ts
index 80ee3cc12b65..342f20cf9820 100644
--- a/dev-packages/e2e-tests/lib/getTestMatrix.ts
+++ b/dev-packages/e2e-tests/lib/getTestMatrix.ts
@@ -1,15 +1,32 @@
 import { execSync } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
+import { dirname } from 'path';
+import { parseArgs } from 'util';
 import { sync as globSync } from 'glob';
 
 interface MatrixInclude {
+  /** The test application (directory) name. */
   'test-application': string;
+  /** Optional override for the build command to run. */
   'build-command'?: string;
+  /** Optional override for the assert command to run. */
   'assert-command'?: string;
+  /** Optional label for the test run. If not set, defaults to value of `test-application`. */
   label?: string;
 }
 
+interface PackageJsonSentryTestConfig {
+  /** If this is true, the test app is optional. */
+  optional?: boolean;
+  /** Variant configs that should be run in non-optional test runs. */
+  variants?: Partial<MatrixInclude>[];
+  /** Variant configs that should be run in optional test runs. */
+  optionalVariants?: Partial<MatrixInclude>[];
+  /** Skip this test app for matrix generation. */
+  skip?: boolean;
+}
+
 /**
  * This methods generates a matrix for the GitHub Actions workflow to run the E2E tests.
  * It checks which test applications are affected by the current changes in the PR and then generates a matrix
@@ -20,20 +37,20 @@ interface MatrixInclude {
  * Otherwise, these will be skipped.
  */
 function run(): void {
-  const args: Record<string, string> = {};
-  process.argv
-    .slice(2)
-    .filter(arg => arg.startsWith('--') && arg.includes('='))
-    .forEach(arg => {
-      const [part1, part2] = arg.split('=') as [string, string];
-      const argName = part1.replace('--', '');
-      const argValue = part2;
-      args[argName] = argValue;
-    });
+  const { values } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      base: { type: 'string' },
+      head: { type: 'string' },
+      optional: { type: 'string', default: 'false' },
+    },
+  });
 
-  const { base, head, optional = 'false' } = args;
+  const { base, head, optional } = values;
 
-  const testApplications = globSync('*', { cwd: `${__dirname}/../test-applications/` });
+  const testApplications = globSync('*/package.json', {
+    cwd: `${__dirname}/../test-applications`,
+  }).map(filePath => dirname(filePath));
 
   // If `--base=xxx` is defined, we only want to get test applications changed since that base
   // Else, we take all test applications (e.g. on push)
@@ -60,11 +77,6 @@ function addIncludesForTestApp(
 ): void {
   const packageJson = getPackageJson(testApp);
 
-  // this means something went wrong
-  if (!packageJson) {
-    return;
-  }
-
   const shouldSkip = packageJson.sentryTest?.skip || false;
   const isOptional = packageJson.sentryTest?.optional || false;
   const variants = (optionalMode ? packageJson.sentryTest?.optionalVariants : packageJson.sentryTest?.variants) || [];
@@ -99,23 +111,15 @@ function getSentryDependencies(appName: string): string[] {
   return Object.keys(dependencies).filter(key => key.startsWith('@sentry'));
 }
 
-function getPackageJson(appName: string):
-  | {
-      dependencies?: { [key: string]: string };
-      devDependencies?: { [key: string]: string };
-      sentryTest?: {
-        optional?: boolean;
-        variants?: Partial<MatrixInclude>[];
-        optionalVariants?: Partial<MatrixInclude>[];
-        skip?: boolean;
-      };
-    }
-  | undefined {
+function getPackageJson(appName: string): {
+  dependencies?: { [key: string]: string };
+  devDependencies?: { [key: string]: string };
+  sentryTest?: PackageJsonSentryTestConfig;
+} {
   const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json');
 
-  // This can happen if you e.g. have a leftover directory in test-applications
   if (!fs.existsSync(fullPath)) {
-    return undefined;
+    throw new Error(`Could not find package.json for ${appName}`);
   }
 
   return JSON.parse(fs.readFileSync(fullPath, 'utf8'));
diff --git a/dev-packages/e2e-tests/package.json b/dev-packages/e2e-tests/package.json
index 54b0388807c7..ccf59ef38f9d 100644
--- a/dev-packages/e2e-tests/package.json
+++ b/dev-packages/e2e-tests/package.json
@@ -20,7 +20,7 @@
   },
   "devDependencies": {
     "@types/glob": "8.0.0",
-    "@types/node": "^14.18.0",
+    "@types/node": "^18.0.0",
     "dotenv": "16.0.3",
     "esbuild": "0.20.0",
     "glob": "8.0.3",
diff --git a/yarn.lock b/yarn.lock
index ee2e6542a2e0..d51069a84a0c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -10093,6 +10093,13 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.63.tgz#1788fa8da838dbb5f9ea994b834278205db6ca2b"
   integrity sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==
 
+"@types/node@^18.0.0":
+  version "18.19.64"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.64.tgz#122897fb79f2a9ec9c979bded01c11461b2b1478"
+  integrity sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==
+  dependencies:
+    undici-types "~5.26.4"
+
 "@types/normalize-package-data@^2.4.0":
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e"
@@ -12922,25 +12929,7 @@ bluebird@^3.4.6, bluebird@^3.7.2:
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
   integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
 
-body-parser@1.20.3, body-parser@^1.18.3, body-parser@^1.19.0:
-  version "1.20.3"
-  resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.3.tgz#1953431221c6fb5cd63c4b36d53fab0928e548c6"
-  integrity sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==
-  dependencies:
-    bytes "3.1.2"
-    content-type "~1.0.5"
-    debug "2.6.9"
-    depd "2.0.0"
-    destroy "1.2.0"
-    http-errors "2.0.0"
-    iconv-lite "0.4.24"
-    on-finished "2.4.1"
-    qs "6.13.0"
-    raw-body "2.5.2"
-    type-is "~1.6.18"
-    unpipe "1.0.0"
-
-body-parser@^1.20.3:
+body-parser@1.20.3, body-parser@^1.18.3, body-parser@^1.19.0, body-parser@^1.20.3:
   version "1.20.3"
   resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.3.tgz#1953431221c6fb5cd63c4b36d53fab0928e548c6"
   integrity sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==
@@ -28321,13 +28310,6 @@ qs@^6.4.0:
   dependencies:
     side-channel "^1.0.4"
 
-qs@6.13.0:
-  version "6.13.0"
-  resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906"
-  integrity sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==
-  dependencies:
-    side-channel "^1.0.6"
-
 query-string@^4.2.2:
   version "4.3.4"
   resolved "https://registry.yarnpkg.com/query-string/-/query-string-4.3.4.tgz#bbb693b9ca915c232515b228b1a02b609043dbeb"
@@ -32419,6 +32401,11 @@ underscore@>=1.8.3:
   resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.12.1.tgz#7bb8cc9b3d397e201cf8553336d262544ead829e"
   integrity sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==
 
+undici-types@~5.26.4:
+  version "5.26.5"
+  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
+  integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
+
 undici-types@~6.19.2:
   version "6.19.8"
   resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02"

From e68dc713577c9a75946d7fbc9614db1daa250fc0 Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Fri, 15 Nov 2024 09:41:17 +0100
Subject: [PATCH 21/22] fix node types version

---
 yarn.lock | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index d51069a84a0c..ab60214fdc7a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -10061,10 +10061,12 @@
   dependencies:
     "@types/node" "*"
 
-"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.7.0":
-  version "17.0.38"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.38.tgz#f8bb07c371ccb1903f3752872c89f44006132947"
-  integrity sha512-5jY9RhV7c0Z4Jy09G+NIDTsCZ5G0L5n+Z+p+Y7t5VJHM30bgwzSjVtlcBxqAj+6L/swIlvtOSzr8rBk/aNyV2g==
+"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=18":
+  version "22.9.0"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-22.9.0.tgz#b7f16e5c3384788542c72dc3d561a7ceae2c0365"
+  integrity sha512-vuyHg81vvWA1Z1ELfvLko2c8f34gyA0zaic0+Rllc5lbCnbSyuvb2Oxpm6TAUAC/2xZN3QGqxBNggD1nNR2AfQ==
+  dependencies:
+    undici-types "~6.19.8"
 
 "@types/node@16.18.70":
   version "16.18.70"
@@ -10076,13 +10078,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.2.tgz#d76fb80d87d0d8abfe334fc6d292e83e5524efc4"
   integrity sha512-Vvycsc9FQdwhxE3y3DzeIxuEJbWGDsnrxvMADzTDF/lcdR9/K+AQIeAghTQsHtotg/q0j3WEOYS/jQgSdWue3w==
 
-"@types/node@>=18":
-  version "22.7.4"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-22.7.4.tgz#e35d6f48dca3255ce44256ddc05dee1c23353fcc"
-  integrity sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==
-  dependencies:
-    undici-types "~6.19.2"
-
 "@types/node@^10.1.0":
   version "10.17.60"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.60.tgz#35f3d6213daed95da7f0f73e75bcc6980e90597b"
@@ -32406,7 +32401,7 @@ undici-types@~5.26.4:
   resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
   integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
 
-undici-types@~6.19.2:
+undici-types@~6.19.8:
   version "6.19.8"
   resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02"
   integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==

From 2ba6776b9d6366d1ae556b46d40134e3c9fe0fce Mon Sep 17 00:00:00 2001
From: Francesco Novy <francesco.novy@sentry.io>
Date: Fri, 15 Nov 2024 11:46:49 +0100
Subject: [PATCH 22/22] explicit add types

---
 .../e2e-tests/test-applications/generic-ts3.8/package.json     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dev-packages/e2e-tests/test-applications/generic-ts3.8/package.json b/dev-packages/e2e-tests/test-applications/generic-ts3.8/package.json
index d13bf86e7c64..80875e5a2d0f 100644
--- a/dev-packages/e2e-tests/test-applications/generic-ts3.8/package.json
+++ b/dev-packages/e2e-tests/test-applications/generic-ts3.8/package.json
@@ -10,7 +10,8 @@
     "test:assert": "pnpm -v"
   },
   "devDependencies": {
-    "typescript": "3.8.3"
+    "typescript": "3.8.3",
+    "@types/node": "^14.18.0"
   },
   "dependencies": {
     "@sentry/browser": "latest || *",