diff --git a/package-lock.json b/package-lock.json
index f0601c10..b77de0b8 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -22906,6 +22906,8 @@
     },
     "node_modules/is-windows": {
       "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
+      "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==",
       "license": "MIT",
       "engines": {
         "node": ">=0.10.0"
@@ -35285,11 +35287,13 @@
         "fs-extra": "^10.0.0",
         "fs-tree-diff": "^2.0.0",
         "handlebars": "^4.3.1",
+        "is-subdir": "^1.2.0",
         "js-string-escape": "^1.0.1",
         "lodash": "^4.17.19",
         "mini-css-extract-plugin": "^2.5.2",
         "minimatch": "^3.0.0",
         "parse5": "^6.0.1",
+        "pkg-entry-points": "^1.1.0",
         "resolve": "^1.20.0",
         "resolve-package-path": "^4.0.3",
         "semver": "^7.3.4",
@@ -35587,6 +35591,14 @@
         "@types/node": "*"
       }
     },
+    "packages/ember-auto-import/node_modules/pkg-entry-points": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/pkg-entry-points/-/pkg-entry-points-1.1.0.tgz",
+      "integrity": "sha512-9vL2T/he5Kb97GVY+V3Ih4jCC1lF3PQGIDUJIUqKM4Q6twmhrUSAa0OFj+kb8IEs4wYzEgB6kcc4oYy21kZnQw==",
+      "funding": {
+        "url": "https://github.com/privatenumber/pkg-entry-points?sponsor=1"
+      }
+    },
     "packages/ember-auto-import/node_modules/resolve-package-path": {
       "version": "4.0.3",
       "license": "MIT",
@@ -49769,12 +49781,14 @@
         "fs-extra": "^10.0.0",
         "fs-tree-diff": "^2.0.0",
         "handlebars": "^4.3.1",
+        "is-subdir": "^1.2.0",
         "js-string-escape": "^1.0.1",
         "lodash": "^4.17.19",
         "mini-css-extract-plugin": "^2.5.2",
         "minimatch": "^3.0.0",
         "npm-run-all": "^4.1.5",
         "parse5": "^6.0.1",
+        "pkg-entry-points": "^1.1.0",
         "prettier": "^2.5.1",
         "quick-temp": "^0.1.8",
         "qunit": "^2.17.2",
@@ -49961,6 +49975,11 @@
             "walk-sync": "^3.0.0"
           }
         },
+        "pkg-entry-points": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/pkg-entry-points/-/pkg-entry-points-1.1.0.tgz",
+          "integrity": "sha512-9vL2T/he5Kb97GVY+V3Ih4jCC1lF3PQGIDUJIUqKM4Q6twmhrUSAa0OFj+kb8IEs4wYzEgB6kcc4oYy21kZnQw=="
+        },
         "resolve-package-path": {
           "version": "4.0.3",
           "requires": {
@@ -58143,7 +58162,9 @@
       }
     },
     "is-windows": {
-      "version": "1.0.2"
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
+      "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA=="
     },
     "is-wsl": {
       "version": "2.2.0",
diff --git a/packages/ember-auto-import/package.json b/packages/ember-auto-import/package.json
index 0155775a..cc208346 100644
--- a/packages/ember-auto-import/package.json
+++ b/packages/ember-auto-import/package.json
@@ -56,11 +56,13 @@
     "fs-extra": "^10.0.0",
     "fs-tree-diff": "^2.0.0",
     "handlebars": "^4.3.1",
+    "is-subdir": "^1.2.0",
     "js-string-escape": "^1.0.1",
     "lodash": "^4.17.19",
     "mini-css-extract-plugin": "^2.5.2",
     "minimatch": "^3.0.0",
     "parse5": "^6.0.1",
+    "pkg-entry-points": "^1.1.0",
     "resolve": "^1.20.0",
     "resolve-package-path": "^4.0.3",
     "semver": "^7.3.4",
diff --git a/packages/ember-auto-import/ts/auto-import.ts b/packages/ember-auto-import/ts/auto-import.ts
index 15377c98..96eb7680 100644
--- a/packages/ember-auto-import/ts/auto-import.ts
+++ b/packages/ember-auto-import/ts/auto-import.ts
@@ -26,8 +26,10 @@ import type { TransformOptions } from '@babel/core';
 import { MARKER } from './analyzer-syntax';
 import path from 'path';
 import funnel from 'broccoli-funnel';
+import makeDebug from 'debug';
 
 const debugTree = buildDebugCallback('ember-auto-import');
+const debugWatch = makeDebug('ember-auto-import:watch');
 
 // This interface must be stable across all versions of ember-auto-import that
 // speak the same leader-election protocol. So don't change this unless you know
@@ -244,6 +246,7 @@ function depsFor(allAppTree: Node, packages: Set<Package>) {
     let watched = pkg.watchedDirectories;
     if (watched) {
       deps = deps.concat(watched.map((dir) => new WatchedDir(dir)));
+      debugWatch(`Adding watched directories: ${watched.join(', ')}`);
     }
   }
   return deps;
diff --git a/packages/ember-auto-import/ts/package.ts b/packages/ember-auto-import/ts/package.ts
index 64e49668..e5c55b54 100644
--- a/packages/ember-auto-import/ts/package.ts
+++ b/packages/ember-auto-import/ts/package.ts
@@ -14,6 +14,7 @@ import type { PluginItem, TransformOptions } from '@babel/core';
 import { MacrosConfig } from '@embroider/macros/src/node';
 import minimatch from 'minimatch';
 import { stripQuery } from './util';
+import { getWatchedDirectories } from './watch-utils';
 
 // from child addon instance to their parent package
 const parentCache: WeakMap<AddonInstance, Package> = new WeakMap();
@@ -514,13 +515,15 @@ export default class Package {
           for (let name of names) {
             let path = resolvePackagePath(name, cursor);
             if (!path) {
-              return undefined;
+              return [];
             }
             cursor = dirname(path);
           }
-          return cursor;
+          return getWatchedDirectories(cursor).map((relativeDir) =>
+            join(cursor, relativeDir)
+          );
         })
-        .filter(Boolean) as string[];
+        .flat();
     }
   }
 
diff --git a/packages/ember-auto-import/ts/tests/watch-utils-test.ts b/packages/ember-auto-import/ts/tests/watch-utils-test.ts
new file mode 100644
index 00000000..52d87add
--- /dev/null
+++ b/packages/ember-auto-import/ts/tests/watch-utils-test.ts
@@ -0,0 +1,269 @@
+import QUnit from 'qunit';
+import 'qunit-assertions-extra';
+import {
+  commonAncestorDirectories,
+  getImportableModules,
+  getWatchedDirectories,
+} from '../watch-utils';
+import { Project } from 'scenario-tester';
+
+const { module: Qmodule, test } = QUnit;
+
+async function generateProject(packageJson = {}, additionalFiles = {}) {
+  const project = new Project('my-package', {
+    files: {
+      'package.json': JSON.stringify(packageJson),
+      src: {
+        'index.js': 'export default 123',
+        'module.js': 'export default 123',
+        nested: {
+          'module.js': 'export default 123',
+        },
+      },
+      dist: {
+        'index.js': 'export default 123',
+        'module.js': 'export default 123',
+        nested: {
+          'module.js': 'export default 123',
+        },
+      },
+      declarations: {
+        'index.d.ts': 'export default 123',
+        'module.d.ts': 'export default 123',
+        nested: {
+          'module.d.ts': 'export default 123',
+        },
+      },
+      lib: {
+        'module.js': 'export default 123',
+      },
+      ...additionalFiles,
+    },
+  });
+
+  await project.write();
+
+  return project;
+}
+
+Qmodule('commonAncestorDirectories', function () {
+  test('returns same dirs if no nested', function (assert) {
+    const result = commonAncestorDirectories([
+      '/a/b/c/index.js',
+      '/d/index.js',
+    ]);
+
+    assert.deepEqual(result, ['/a/b/c', '/d']);
+  });
+
+  test('returns common dirs', function (assert) {
+    const result = commonAncestorDirectories([
+      '/a/b/c/index.js',
+      '/a/b/index.js',
+      '/d/index.js',
+      '/d/e/f/index.js',
+    ]);
+
+    assert.deepEqual(result, ['/a/b', '/d']);
+  });
+
+  test('ignores duplicates', function (assert) {
+    const result = commonAncestorDirectories([
+      '/a/b/c/index.js',
+      '/a/b/index.js',
+      '/a/b/c/index.js',
+      '/a/b/index.js',
+    ]);
+
+    assert.deepEqual(result, ['/a/b']);
+  });
+});
+
+Qmodule('importableModules', function (hooks) {
+  let project: Project;
+
+  hooks.afterEach(function (this: any) {
+    project?.dispose();
+  });
+
+  test('returns only modules declared in exports', async function (assert) {
+    project = await generateProject({
+      exports: './dist/index.js',
+    });
+
+    const result = getImportableModules(project.baseDir);
+
+    assert.deepEqual(result, ['./dist/index.js']);
+  });
+
+  test('ignores types condition', async function (assert) {
+    project = await generateProject({
+      exports: {
+        '.': {
+          types: './declarations/index.d.ts',
+          default: './dist/index.js',
+        },
+      },
+    });
+
+    const result = getImportableModules(project.baseDir);
+
+    assert.deepEqual(result, ['./dist/index.js']);
+  });
+
+  test('ignores node condition', async function (assert) {
+    project = await generateProject({
+      exports: {
+        '.': {
+          types: './declarations/index.d.ts',
+          default: './dist/index.js',
+        },
+        'lib/module': {
+          node: './lib/module.js',
+        },
+      },
+    });
+
+    const result = getImportableModules(project.baseDir);
+
+    assert.deepEqual(result, ['./dist/index.js']);
+  });
+
+  test('supports import condition', async function (assert) {
+    project = await generateProject({
+      exports: {
+        '.': {
+          types: './declarations/index.d.ts',
+          import: './dist/index.js',
+        },
+      },
+    });
+
+    const result = getImportableModules(project.baseDir);
+
+    assert.deepEqual(result, ['./dist/index.js']);
+  });
+
+  test('supports nested conditions', async function (assert) {
+    project = await generateProject({
+      exports: {
+        '.': {
+          import: {
+            types: './declarations/index.d.ts',
+            default: './dist/index.js',
+          },
+        },
+      },
+    });
+
+    const result = getImportableModules(project.baseDir);
+
+    assert.deepEqual(result, ['./dist/index.js']);
+  });
+
+  test('supports subpaths', async function (assert) {
+    project = await generateProject({
+      exports: {
+        '.': {
+          types: './declarations/index.d.ts',
+          default: './dist/index.js',
+        },
+        module: {
+          types: './declarations/module.d.ts',
+          default: './dist/module.js',
+        },
+        'nested/module': {
+          types: './declarations/nested/module.d.ts',
+          default: './dist/nested/module.js',
+        },
+      },
+    });
+
+    const result = getImportableModules(project.baseDir);
+
+    assert.deepEqual(result, [
+      './dist/index.js',
+      './dist/module.js',
+      './dist/nested/module.js',
+    ]);
+  });
+
+  test('supports globstar patterns', async function (assert) {
+    project = await generateProject({
+      exports: {
+        '.': {
+          types: './declarations/index.d.ts',
+          default: './dist/index.js',
+        },
+        './*': {
+          types: './declarations/*.d.ts',
+          default: './dist/*.js',
+        },
+      },
+    });
+
+    const result = getImportableModules(project.baseDir);
+
+    assert.deepEqual(result, [
+      './dist/index.js',
+      './dist/module.js',
+      './dist/nested/module.js',
+    ]);
+  });
+
+  test('returns all possible imports when having only main export', async function (assert) {
+    project = await generateProject({
+      main: './dist/index.js',
+    });
+
+    const result = getImportableModules(project.baseDir);
+
+    assert.deepEqual(result, [
+      './declarations/index.d.ts',
+      './declarations/module.d.ts',
+      './declarations/nested/module.d.ts',
+      './dist/index.js',
+      './dist/module.js',
+      './dist/nested/module.js',
+      './index.js',
+      './lib/module.js',
+      './package.json',
+      './src/index.js',
+      './src/module.js',
+      './src/nested/module.js',
+    ]);
+  });
+});
+
+Qmodule('getWatchedDirectories', function (hooks) {
+  let project: Project;
+
+  hooks.afterEach(function (this: any) {
+    project?.dispose();
+  });
+
+  test('returns only dist for typical v2 addon', async function (assert) {
+    project = await generateProject(
+      {
+        exports: {
+          '.': {
+            types: './declarations/index.d.ts',
+            default: './dist/index.js',
+          },
+          './*': {
+            types: './declarations/*.d.ts',
+            default: './dist/*.js',
+          },
+          './addon-main.js': './addon-main.cjs',
+        },
+      },
+      {
+        'addon-main.cjs': 'module.exports = {}',
+      }
+    );
+
+    const result = getWatchedDirectories(project.baseDir);
+
+    assert.deepEqual(result, ['./dist']);
+  });
+});
diff --git a/packages/ember-auto-import/ts/watch-utils.ts b/packages/ember-auto-import/ts/watch-utils.ts
new file mode 100644
index 00000000..4fea733e
--- /dev/null
+++ b/packages/ember-auto-import/ts/watch-utils.ts
@@ -0,0 +1,69 @@
+import isSubdir from 'is-subdir';
+import { dirname } from 'path';
+import { getPackageEntryPointsSync } from 'pkg-entry-points';
+
+// copied from pkg-entry-points, as we cannot use their types, see comment above
+type ConditionToPath = [conditions: string[], internalPath: string];
+type PackageEntryPoints = {
+  [subpath: string]: ConditionToPath[];
+};
+
+/**
+ * Given a list of files, it will return the smallest set of directories that contain all these files
+ */
+export function commonAncestorDirectories(dirs: string[]): string[] {
+  return dirs.reduce((results, fileOrDir) => {
+    let dir = dirname(fileOrDir);
+
+    if (results.length === 0) {
+      return [dir];
+    }
+
+    let newResults = results.filter(
+      (existingDir) => !isSubdir(dir, existingDir)
+    );
+
+    if (!newResults.some((existingDir) => isSubdir(existingDir, dir))) {
+      newResults.push(dir);
+    }
+
+    return newResults;
+  }, [] as string[]);
+}
+
+/**
+ * Given a path to a package, it will return all its internal(!) module files that are importable,
+ * taking into account explicit package.json exports, filtered down to only include importable runtime code
+ */
+export function getImportableModules(packagePath: string): string[] {
+  const entryPoints: PackageEntryPoints =
+    getPackageEntryPointsSync(packagePath);
+
+  return Object.values(entryPoints)
+    .map(
+      (alternatives) =>
+        alternatives.find(
+          ([conditions]) =>
+            (conditions.includes('import') || conditions.includes('default')) &&
+            !conditions.includes('types') &&
+            !conditions.includes('require') &&
+            !conditions.includes('node')
+        )?.[1]
+    )
+    .filter((item): item is string => !!item)
+    .filter((item, index, array) => array.indexOf(item) === index);
+}
+
+/**
+ * Given a package path, it will return the list smallest set of directories that contain importable code.
+ * This can be used to constrain the set of directories used for file watching, to not include the whole package directory.
+ */
+export function getWatchedDirectories(packagePath: string): string[] {
+  const modules = getImportableModules(packagePath).filter(
+    (module) =>
+      // this is a workaround for excluding the addon-main.cjs module commonly used in v2 addons, which is _not_ importable in runtime code,
+      // but the generic logic based on (conditional) exports does not exclude that out of the box.
+      !module.match(/\/addon-main.c?js$/)
+  );
+  return commonAncestorDirectories(modules);
+}