ref(test): Trigger top-level errors inside sandbox. (#11712)

Resolves: #11678
Related: #11666 

This PR updates `runScriptInSandbox` utility, which was implemented
(probably for a similar reason) but not used.
Ports relevant tests to this pattern.

Author: onurtemizkan

dev-packages/browser-integration-tests/README.md

@@ -14,16 +14,16 @@ or `init.js` is not defined in a case folder.
 
 `subject.js` contains the logic that sets up the environment to be tested. It also can be defined locally and as a group
 fallback. Unlike `template.hbs` and `init.js`, it's not required to be defined for a group, as there may be cases that
-does not require a subject, instead the logic is injected using `injectScriptAndGetEvents` from `utils/helpers.ts`.
+does not require a subject.
 
 `test.ts` is required for each test case, which contains the assertions (and if required the script injection logic).
 For every case, any set of `init.js`, `template.hbs` and `subject.js` can be defined locally, and each one of them will
 have precedence over the default definitions of the test group.
 
 To test page multi-page navigations, you can specify additional `page-*.html` (e.g. `page-0.html`, `page-1.html`) files.
 These will also be compiled and initialized with the same `init.js` and `subject.js` files that are applied to
-`template.hbs/html`. Note: `page-*.html` file lookup **doesn not** fall back to the parent directories, meaning that
-page files have to be directly in the `test.ts` directory.
+`template.hbs/html`. Note: `page-*.html` file lookup **does not** fall back to the parent directories, meaning that page
+files have to be directly in the `test.ts` directory.
 
 ```
 suites/

dev-packages/browser-integration-tests/package.json

@@ -40,7 +40,7 @@
   },
   "dependencies": {
     "@babel/preset-typescript": "^7.16.7",
-    "@playwright/test": "^1.40.1",
+    "@playwright/test": "^1.43.1",
     "@sentry-internal/rrweb": "2.11.0",
     "@sentry/browser": "8.0.0-beta.4",
     "axios": "1.6.7",

dev-packages/browser-integration-tests/suites/public-api/instrumentation/onError/non-string-arg/subject.js

@@ -2,14 +2,32 @@ import { expect } from '@playwright/test';
 import type { Event } from '@sentry/types';
 
 import { sentryTest } from '../../../../../utils/fixtures';
-import { getFirstSentryEnvelopeRequest } from '../../../../../utils/helpers';
+import { getFirstSentryEnvelopeRequest, runScriptInSandbox } from '../../../../../utils/helpers';
 
 sentryTest(
   'should catch onerror calls with non-string first argument gracefully',
-  async ({ getLocalTestPath, page }) => {
+  async ({ getLocalTestPath, page, browserName }) => {
+    if (browserName === 'webkit') {
+      // This test fails on Webkit as erros thrown from `runScriptInSandbox` are Script Errors and skipped by Sentry
+      sentryTest.skip();
+    }
+
     const url = await getLocalTestPath({ testDir: __dirname });
 
-    const eventData = await getFirstSentryEnvelopeRequest<Event>(page, url);
+    await page.goto(url);
+
+    const errorEventPromise = getFirstSentryEnvelopeRequest<Event>(page);
+
+    await runScriptInSandbox(page, {
+      content: `
+        throw {
+          type: 'Error',
+          otherKey: 'otherValue',
+        };
+      `,
+    });
+
+    const eventData = await errorEventPromise;
 
     expect(eventData.exception?.values).toHaveLength(1);
     expect(eventData.exception?.values?.[0]).toMatchObject({

dev-packages/browser-integration-tests/suites/public-api/instrumentation/onError/non-string-arg/test.ts

@@ -2,14 +2,41 @@ import { expect } from '@playwright/test';
 import type { Event } from '@sentry/types';
 
 import { sentryTest } from '../../../../../utils/fixtures';
-import { getMultipleSentryEnvelopeRequests } from '../../../../../utils/helpers';
+import { getMultipleSentryEnvelopeRequests, runScriptInSandbox } from '../../../../../utils/helpers';
 
 sentryTest(
   'should NOT catch an exception already caught [but rethrown] via Sentry.captureException',
-  async ({ getLocalTestPath, page }) => {
+  async ({ getLocalTestPath, page, browserName }) => {
+    if (browserName === 'webkit') {
+      // This test fails on Webkit as erros thrown from `runScriptInSandbox` are Script Errors and skipped by Sentry
+      sentryTest.skip();
+    }
+
     const url = await getLocalTestPath({ testDir: __dirname });
 
-    const events = await getMultipleSentryEnvelopeRequests<Event>(page, 2, { url });
+    await page.goto(url);
+
+    const errorEventsPromise = getMultipleSentryEnvelopeRequests<Event>(page, 2);
+
+    await runScriptInSandbox(page, {
+      content: `
+      try {
+        try {
+          foo();
+        } catch (e) {
+          Sentry.captureException(e);
+          throw e; // intentionally re-throw
+        }
+      } catch (e) {
+        // simulate window.onerror without generating a Script error
+        window.onerror('error', 'file.js', 1, 1, e);
+      }
+
+      Sentry.captureException(new Error('error 2'));
+    `,
+    });
+
+    const events = await errorEventsPromise;
 
     expect(events[0].exception?.values).toHaveLength(1);
     expect(events[0].exception?.values?.[0]).toMatchObject({

dev-packages/browser-integration-tests/suites/public-api/instrumentation/onError/rethrown/subject.js

@@ -2,12 +2,27 @@ import { expect } from '@playwright/test';
 import type { Event } from '@sentry/types';
 
 import { sentryTest } from '../../../../../utils/fixtures';
-import { getFirstSentryEnvelopeRequest } from '../../../../../utils/helpers';
+import { getFirstSentryEnvelopeRequest, runScriptInSandbox } from '../../../../../utils/helpers';
+
+sentryTest('should catch syntax errors', async ({ getLocalTestPath, page, browserName }) => {
+  if (browserName === 'webkit') {
+    // This test fails on Webkit as erros thrown from `runScriptInSandbox` are Script Errors and skipped by Sentry
+    sentryTest.skip();
+  }
 
-sentryTest('should catch syntax errors', async ({ getLocalTestPath, page }) => {
   const url = await getLocalTestPath({ testDir: __dirname });
 
-  const eventData = await getFirstSentryEnvelopeRequest<Event>(page, url);
+  await page.goto(url);
+
+  const errorEventPromise = getFirstSentryEnvelopeRequest<Event>(page);
+
+  await runScriptInSandbox(page, {
+    content: `
+      foo{}; // SyntaxError
+    `,
+  });
+
+  const eventData = await errorEventPromise;
 
   expect(eventData.exception?.values).toHaveLength(1);
   expect(eventData.exception?.values?.[0]).toMatchObject({

dev-packages/browser-integration-tests/suites/public-api/instrumentation/onError/rethrown/test.ts

@@ -2,12 +2,27 @@ import { expect } from '@playwright/test';
 import type { Event } from '@sentry/types';
 
 import { sentryTest } from '../../../../../utils/fixtures';
-import { getFirstSentryEnvelopeRequest } from '../../../../../utils/helpers';
+import { getFirstSentryEnvelopeRequest, runScriptInSandbox } from '../../../../../utils/helpers';
+
+sentryTest('should catch thrown errors', async ({ getLocalTestPath, page, browserName }) => {
+  if (browserName === 'webkit') {
+    // This test fails on Webkit as erros thrown from `runScriptInSandbox` are Script Errors and skipped by Sentry
+    sentryTest.skip();
+  }
 
-sentryTest('should catch thrown errors', async ({ getLocalTestPath, page }) => {
   const url = await getLocalTestPath({ testDir: __dirname });
 
-  const eventData = await getFirstSentryEnvelopeRequest<Event>(page, url);
+  await page.goto(url);
+
+  const errorEventPromise = getFirstSentryEnvelopeRequest<Event>(page);
+
+  await runScriptInSandbox(page, {
+    content: `
+      throw new Error('realError');
+      `,
+  });
+
+  const eventData = await errorEventPromise;
 
   expect(eventData.exception?.values).toHaveLength(1);
   expect(eventData.exception?.values?.[0]).toMatchObject({

dev-packages/browser-integration-tests/suites/public-api/instrumentation/onError/syntax-errors/subject.js

@@ -2,12 +2,29 @@ import { expect } from '@playwright/test';
 import type { Event } from '@sentry/types';
 
 import { sentryTest } from '../../../../../utils/fixtures';
-import { getFirstSentryEnvelopeRequest } from '../../../../../utils/helpers';
+import { getFirstSentryEnvelopeRequest, runScriptInSandbox } from '../../../../../utils/helpers';
+
+sentryTest('should catch thrown objects', async ({ getLocalTestPath, page, browserName }) => {
+  if (browserName === 'webkit') {
+    // This test fails on Webkit as erros thrown from `runScriptInSandbox` are Script Errors and skipped by Sentry
+    sentryTest.skip();
+  }
 
-sentryTest('should catch thrown objects', async ({ getLocalTestPath, page }) => {
   const url = await getLocalTestPath({ testDir: __dirname });
 
-  const eventData = await getFirstSentryEnvelopeRequest<Event>(page, url);
+  await page.goto(url);
+
+  const errorEventPromise = getFirstSentryEnvelopeRequest<Event>(page);
+
+  await runScriptInSandbox(page, {
+    content: `
+      throw {
+        error: 'stuff is broken',
+        somekey: 'ok'
+      };`,
+  });
+
+  const eventData = await errorEventPromise;
 
   expect(eventData.exception?.values).toHaveLength(1);
   expect(eventData.exception?.values?.[0]).toMatchObject({

dev-packages/browser-integration-tests/suites/public-api/instrumentation/onError/syntax-errors/test.ts

@@ -2,12 +2,27 @@ import { expect } from '@playwright/test';
 import type { Event } from '@sentry/types';
 
 import { sentryTest } from '../../../../../utils/fixtures';
-import { getFirstSentryEnvelopeRequest } from '../../../../../utils/helpers';
+import { getFirstSentryEnvelopeRequest, runScriptInSandbox } from '../../../../../utils/helpers';
+
+sentryTest('should catch thrown strings', async ({ getLocalTestPath, page, browserName }) => {
+  if (browserName === 'webkit') {
+    // This test fails on Webkit as erros thrown from `runScriptInSandbox` are Script Errors and skipped by Sentry
+    sentryTest.skip();
+  }
 
-sentryTest('should catch thrown strings', async ({ getLocalTestPath, page }) => {
   const url = await getLocalTestPath({ testDir: __dirname });
 
-  const eventData = await getFirstSentryEnvelopeRequest<Event>(page, url);
+  await page.goto(url);
+
+  const errorEventPromise = getFirstSentryEnvelopeRequest<Event>(page);
+
+  await runScriptInSandbox(page, {
+    content: `
+      throw 'stringError';
+    `,
+  });
+
+  const eventData = await errorEventPromise;
 
   expect(eventData.exception?.values).toHaveLength(1);
   expect(eventData.exception?.values?.[0]).toMatchObject({

dev-packages/browser-integration-tests/suites/public-api/instrumentation/onError/thrown-errors/subject.js

@@ -2,21 +2,48 @@ import { expect } from '@playwright/test';
 import type { Event } from '@sentry/types';
 
 import { sentryTest } from '../../../../utils/fixtures';
-import { getMultipleSentryEnvelopeRequests, shouldSkipTracingTest } from '../../../../utils/helpers';
+import {
+  getMultipleSentryEnvelopeRequests,
+  runScriptInSandbox,
+  shouldSkipTracingTest,
+} from '../../../../utils/helpers';
 
-sentryTest('should capture an error within a sync startSpan callback', async ({ getLocalTestPath, page }) => {
-  if (shouldSkipTracingTest()) {
-    sentryTest.skip();
-  }
+sentryTest(
+  'should capture an error within a sync startSpan callback',
+  async ({ getLocalTestPath, page, browserName }) => {
+    if (browserName === 'webkit') {
+      // This test fails on Webkit as erros thrown from `runScriptInSandbox` are Script Errors and skipped by Sentry
+      sentryTest.skip();
+    }
 
-  const url = await getLocalTestPath({ testDir: __dirname });
-  const gotoPromise = page.goto(url);
-  const envelopePromise = getMultipleSentryEnvelopeRequests<Event>(page, 2);
+    if (shouldSkipTracingTest()) {
+      sentryTest.skip();
+    }
 
-  const [, events] = await Promise.all([gotoPromise, envelopePromise]);
-  const txn = events.find(event => event.type === 'transaction');
-  const err = events.find(event => !event.type);
+    const url = await getLocalTestPath({ testDir: __dirname });
 
-  expect(txn).toMatchObject({ transaction: 'parent_span' });
-  expect(err?.exception?.values?.[0]?.value).toBe('Sync Error');
-});
+    await page.goto(url);
+
+    const errorEventsPromise = getMultipleSentryEnvelopeRequests<Event>(page, 2);
+
+    await runScriptInSandbox(page, {
+      content: `
+      function run() {
+        Sentry.startSpan({ name: 'parent_span' }, () => {
+          throw new Error('Sync Error');
+        });
+      }
+
+      setTimeout(run);
+      `,
+    });
+
+    const events = await errorEventsPromise;
+
+    const txn = events.find(event => event.type === 'transaction');
+    const err = events.find(event => !event.type);
+
+    expect(txn).toMatchObject({ transaction: 'parent_span' });
+    expect(err?.exception?.values?.[0]?.value).toBe('Sync Error');
+  },
+);

dev-packages/browser-integration-tests/suites/public-api/instrumentation/onError/thrown-errors/test.ts

@@ -31,7 +31,8 @@ sentryTest(
         { function: '?' },
         { function: 'qux' },
         { function: 'qux/<' },
-        { function: 'qux/</<' },
+        // The function name below was 'qux/</<' on the Firefox versions < 124
+        { function: 'qux/<' },
         { function: 'foo' },
         { function: 'bar' },
         { function: 'baz' },