diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5938f5ef2..f5e0682f8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -147,7 +147,7 @@ jobs:
     timeout-minutes: 15
 
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         ember-try-scenario:
         - ember-lts-3.8
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7745fac28..e98d2a144 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+### master
+  - Add runtime assertion to detect transpilation failures with the new async arrow fn API
+
 ### 2.3.0
   - Introduce async-arrow task() API as the new universal JS/TS API, e.g.
     `myTask = task(this, { drop: true }, async (arg: string) => {})`. This new API
diff --git a/addon/-private/task-public-api.js b/addon/-private/task-public-api.js
index 58b0d34b2..1b33ff86a 100644
--- a/addon/-private/task-public-api.js
+++ b/addon/-private/task-public-api.js
@@ -9,6 +9,7 @@ import {
   task as taskDecorator,
   taskGroup as taskGroupDecorator,
 } from './task-decorators';
+import { assert } from '@ember/debug';
 
 /**
  * TODO: update docs to reflect both old and new ES6 styles
@@ -58,6 +59,11 @@ import {
  * @returns {TaskProperty}
  */
 export function task(taskFnOrProtoOrDecoratorOptions, key, descriptor) {
+  assert(
+    `It appears you're attempting to use the new task(this, async () => { ... }) syntax, but the async arrow task function you've provided is not being properly compiled by Babel.\n\nPossible causes / remedies:\n\n1. You must pass the async function expression directly to the task() function (it is not currently supported to pass in a variable containing the async arrow fn, or any other kind of indirection)\n2. If this code is in an addon, please ensure the addon specificies ember-concurrency "2.3.0" or higher in "dependencies" (not "devDependencies")\n3. Ensure that there is only one version of ember-concurrency v2.3.0+ being used in your project (including nested dependencies) and consider using npm/yarn/pnpm resolutions to enforce a single version is used`,
+    !isUntranspiledAsyncFn(arguments[arguments.length - 1])
+  );
+
   if (
     isDecoratorOptions(taskFnOrProtoOrDecoratorOptions) ||
     (key && descriptor)
@@ -68,6 +74,10 @@ export function task(taskFnOrProtoOrDecoratorOptions, key, descriptor) {
   }
 }
 
+function isUntranspiledAsyncFn(obj) {
+  return obj && obj.constructor && obj.constructor.name === 'AsyncFunction';
+}
+
 /**
  * Build and return a "classic" TaskProperty, which is essentially a subclass of a Computed Property
  * descriptor that can be used to define Tasks on classic Ember.Objects.
diff --git a/tests/acceptance/helpers-test.js b/tests/acceptance/helpers-test.js
index 90d149b67..856a0c0eb 100644
--- a/tests/acceptance/helpers-test.js
+++ b/tests/acceptance/helpers-test.js
@@ -1,28 +1,8 @@
 import { click, visit, currentURL } from '@ember/test-helpers';
-import Ember from 'ember';
 import { setupApplicationTest } from 'ember-qunit';
 import { module, skip, test } from 'qunit';
-import {
-  macroCondition,
-  dependencySatisfies,
-  importSync,
-} from '@embroider/macros';
-
-function getDebugFunction(type) {
-  if (macroCondition(dependencySatisfies('ember-source', '^3.26.0'))) {
-    return importSync('@ember/debug').getDebugFunction(type);
-  } else {
-    return Ember[type];
-  }
-}
-
-function setDebugFunction(type, fn) {
-  if (macroCondition(dependencySatisfies('ember-source', '^3.26.0'))) {
-    return importSync('@ember/debug').setDebugFunction(type, fn);
-  } else {
-    Ember[type] = fn;
-  }
-}
+import { macroCondition, dependencySatisfies } from '@embroider/macros';
+import { getDebugFunction, setDebugFunction } from '../helpers/helpers';
 
 const originalAssert = getDebugFunction('assert');
 
diff --git a/tests/helpers/helpers.js b/tests/helpers/helpers.js
index 6fe935754..315d51ca6 100644
--- a/tests/helpers/helpers.js
+++ b/tests/helpers/helpers.js
@@ -1,6 +1,11 @@
 import { skip, test } from 'qunit';
 import { gte } from 'ember-compatibility-helpers';
 import Ember from 'ember';
+import {
+  importSync,
+  macroCondition,
+  dependencySatisfies,
+} from '@embroider/macros';
 
 export const decoratorTest = gte('3.10.0') ? test : skip;
 
@@ -8,3 +13,19 @@ export function makeAsyncError(hooks) {
   hooks.afterEach(() => (Ember.onerror = null));
   return () => new window.Promise((r) => (Ember.onerror = r));
 }
+
+export function getDebugFunction(type) {
+  if (macroCondition(dependencySatisfies('ember-source', '>3.26.0'))) {
+    return importSync('@ember/debug').getDebugFunction(type);
+  } else {
+    return Ember[type];
+  }
+}
+
+export function setDebugFunction(type, fn) {
+  if (macroCondition(dependencySatisfies('ember-source', '>3.26.0'))) {
+    return importSync('@ember/debug').setDebugFunction(type, fn);
+  } else {
+    Ember[type] = fn;
+  }
+}
diff --git a/tests/integration/async-arrow-task-test.js b/tests/integration/async-arrow-task-test.js
index 2fac52ab8..64d54c994 100644
--- a/tests/integration/async-arrow-task-test.js
+++ b/tests/integration/async-arrow-task-test.js
@@ -12,6 +12,7 @@ import {
   enqueueTask,
 } from 'ember-concurrency';
 import Component from '@glimmer/component';
+import { getDebugFunction, setDebugFunction } from '../helpers/helpers';
 
 function defer() {
   let resolve, reject;
@@ -24,6 +25,8 @@ function defer() {
   return { promise, resolve, reject };
 }
 
+const originalAssert = getDebugFunction('assert');
+
 class TestComponent extends Component {
   resolved = null;
 
@@ -89,6 +92,10 @@ module('Integration | async-arrow-task', function (hooks) {
     );
   });
 
+  hooks.afterEach(function () {
+    setDebugFunction('assert', originalAssert);
+  });
+
   test('two args - task(this, async () => {})', async function (assert) {
     assert.expect(13);
 
@@ -167,4 +174,26 @@ module('Integration | async-arrow-task', function (hooks) {
 
     await finishTest(assert);
   });
+
+  test('runtime assertion to detect improper task() use or transpilation errors', async function (assert) {
+    assert.expect(2);
+
+    let assertionDidFire = false;
+    setDebugFunction('assert', function (msg, test) {
+      if (!test) {
+        // eslint-disable-next-line qunit/no-conditional-assertions
+        assert.ok(
+          msg.includes(
+            "the async arrow task function you've provided is not being properly compiled by Babel"
+          ),
+          'expected assertion message'
+        );
+        assertionDidFire = true;
+      }
+    });
+
+    const asyncArrowFn = async () => {};
+    task(this, asyncArrowFn);
+    assert.ok(assertionDidFire);
+  });
 });