Import rework

.gitmodules

@@ -1,6 +1,8 @@
 [submodule "community-wgsl"]
 	path = community-wgsl
 	url = https://github.com/wgsl-tooling-wg/community-wgsl
+	update = merge
 [submodule "wesl-testsuite"]
 	path = wesl-testsuite
 	url = https://github.com/wgsl-tooling-wg/wesl-testsuite.git
+	update = merge

linker/packages/linker/src/AbstractElems.ts

@@ -1,4 +1,4 @@
-import { ImportTree } from "./ImportTree.ts";
+import { ImportStatement } from "./ImportTree.ts";
 import { DeclIdent, RefIdent, SrcModule } from "./Scope.ts";
 
 /**
@@ -161,7 +161,7 @@ export interface GlobalVarElem extends ElemWithContentsBase {
 /** an import statement */
 export interface ImportElem extends ElemWithContentsBase {
   kind: "import";
-  imports: ImportTree;
+  imports: ImportStatement;
 }
 
 /** an entire file */

linker/packages/linker/src/BindIdents.ts

@@ -197,9 +197,8 @@ function matchingImport(
 ): string[] | undefined {
   const identParts = ident.originalName.split("::");
   for (const flat of flatImports) {
-    const impTail = overlapTail(flat.importPath, identParts);
-    if (impTail) {
-      return [...flat.modulePath, ...impTail];
+    if (flat.importPath.at(-1) === identParts.at(0)) {
+      return [...flat.modulePath, ...identParts.slice(1)];
     }
   }
 }

linker/packages/linker/src/FlattenTreeImport.ts

@@ -1,9 +1,8 @@
-import { tracing } from "mini-parse";
 import {
-  ImportTree,
-  PathSegment,
-  SegmentList,
-  SimpleSegment,
+  ImportCollection,
+  ImportItem,
+  ImportSegment,
+  ImportStatement,
 } from "./ImportTree.js";
 
 export interface FlatImport {
@@ -16,44 +15,41 @@ export interface FlatImport {
  *
  * @return map from import path (with 'as' renaming) to module Path
  */
-export function flattenTreeImport(imp: ImportTree): FlatImport[] {
-  return recursiveResolve([], [], imp.segments);
+export function flattenTreeImport(imp: ImportStatement): FlatImport[] {
+  return recursiveResolve([], [], imp.segments, imp.finalSegment);
 
   /** recurse through segments of path, producing  */
   function recursiveResolve(
     resolvedImportPath: string[],
     resolvedExportPath: string[],
-    remainingPath: PathSegment[],
+    remainingPath: ImportSegment[],
+    finalSegment: ImportCollection | ImportItem,
   ): FlatImport[] {
-    const [segment, ...rest] = remainingPath;
-    if (segment === undefined) {
-      throw new Error(`undefined segment ${imp.segments}`);
-    }
-    if (segment instanceof SimpleSegment) {
-      const importPath = [...resolvedImportPath, segment.as || segment.name];
+    if (remainingPath.length > 0) {
+      const [segment, ...rest] = remainingPath;
+      const importPath = [...resolvedImportPath, segment.name];
       const modulePath = [...resolvedExportPath, segment.name];
-      if (rest.length) {
-        // we're in the middle of the path so keep recursing
-        return recursiveResolve(importPath, modulePath, rest);
-      } else {
-        return [{ importPath, modulePath }];
-      }
-    }
-    if (segment instanceof SegmentList) {
+      return recursiveResolve(importPath, modulePath, rest, finalSegment);
+    } else if (finalSegment instanceof ImportCollection) {
       // resolve path with each element in the list
-      return segment.list.flatMap(elem => {
-        const rPath = [elem, ...rest];
-        return recursiveResolve(resolvedImportPath, resolvedExportPath, rPath);
+      return finalSegment.subTrees.flatMap(elem => {
+        return recursiveResolve(
+          resolvedImportPath,
+          resolvedExportPath,
+          elem.segments,
+          elem.finalSegment,
+        );
       });
-    } else if (segment instanceof ImportTree) {
-      return recursiveResolve(
-        resolvedImportPath,
-        resolvedExportPath,
-        segment.segments,
-      );
+    } else if (finalSegment instanceof ImportItem) {
+      const importPath = [
+        ...resolvedImportPath,
+        finalSegment.as || finalSegment.name,
+      ];
+      const modulePath = [...resolvedExportPath, finalSegment.name];
+      return [{ importPath, modulePath }];
+    } else {
+      console.error(finalSegment);
+      throw new Error("unknown segment type", { cause: finalSegment });
     }
-
-    if (tracing) console.log("unknown segment type", segment); // should be impossible
-    return [];
   }
 }

linker/packages/linker/src/ImportGrammar.ts

@@ -1,156 +1,120 @@
 import {
-  disablePreParse,
+  delimited,
   kind,
-  makeEolf,
-  matchOneOf,
   NoTags,
   opt,
   or,
   Parser,
+  preceded,
   repeat,
   repeatPlus,
   seq,
   setTraceName,
   TagRecord,
   tagScope,
-  tokenMatcher,
+  terminated,
   tokens,
-  tokenSkipSet,
   tracing,
   withSepPlus,
 } from "mini-parse";
+import { mainTokens } from "./WESLTokens.js";
 import {
-  importElem,
-  importList,
-  importSegment,
-  importTree,
-} from "./WESLCollect.js";
-import { digits, eol, ident } from "./WESLTokens.js";
-
-// TODO now that ';' is required, special ws and eol handling is probably not needed.
-const skipWsSet = new Set(["ws"]);
-function skipWs<V, T extends TagRecord>(p: Parser<V, T>): Parser<V, T> {
-  return tokenSkipSet(skipWsSet, p);
+  ImportCollection,
+  ImportItem,
+  ImportSegment,
+  ImportStatement,
+} from "./ImportTree.js";
+import { ImportElem } from "./AbstractElems.js";
+import { importElem } from "./WESLCollect.js";
+
+const wordToken = kind(mainTokens.ident);
+
+function segment(text: string) {
+  return new ImportSegment(text);
 }
-function noSkipWs<V, T extends TagRecord>(p: Parser<V, T>): Parser<V, T> {
-  return tokenSkipSet(null, p);
+function segments(
+  ...values: (ImportSegment | ImportSegment[])[]
+): ImportSegment[] {
+  return values.flat();
 }
 
-const importSymbolSet = "/ { } , ( ) .. . * ; @ #"; // Had to add @ and # here to get the parsing tests to work. Weird.
-const importSymbol = matchOneOf(importSymbolSet);
-
-// TODO reconsider whether we need a separate token set for import statements vs wgsl/wesl
-export const importTokens = tokenMatcher({
-  ws: /\s+/,
-  importSymbol,
-  ident, // TODO allow '-' in pkg names?
-  digits,
-});
-
-export const eolTokens = tokenMatcher({
-  ws: /[ \t]+/, // don't include \n, for eolf
-  eol,
-});
-
-const eolf = disablePreParse(makeEolf(eolTokens, importTokens.ws));
-const wordToken = kind(importTokens.ident);
-
-// forward references for mutual recursion
-let packagePath: Parser<any, NoTags> = null as any;
-
-// prettier-ignore
-const simpleSegment = tagScope(
-  wordToken                             .ptag("segment").collect(importSegment),
-);
-
-// prettier-ignore
 /** last simple segment is allowed to have an 'as' rename */
-const lastSimpleSegment = tagScope(
-  seq(
-    wordToken                           .ptag("segment"),
-    skipWs(opt(seq("as", wordToken      .ptag("as")))),
-  )                                     .collect(importSegment),
+const item_import = seq(wordToken, opt(preceded("as", wordToken))).mapValue(
+  v => new ImportItem(v[0], v[1]),
 );
 
-/** an item an a collection list {a, b} */
-// prettier-ignore
-const collectionItem = or(
-  tagScope(or(() => packagePath)        .collect(importTree)),
-  lastSimpleSegment,
-);
-
-// prettier-ignore
-const importCollection = tagScope(
-  seq(
-    "{",
-    skipWs(
-      seq(
-        withSepPlus(",", () => collectionItem     .ctag("list")),
-        "}",
-      ),
+// forward references for mutual recursion
+let import_collection: Parser<ImportCollection, NoTags> = null as any;
+
+const import_path = seq(
+  repeatPlus(terminated(wordToken.mapValue(segment), "::")),
+  or(() => import_collection, item_import),
+).mapValue(v => new ImportStatement(v[0], v[1]));
+
+import_collection = delimited(
+  "{",
+  withSepPlus(",", () =>
+    or(
+      import_path,
+      item_import.mapValue(v => new ImportStatement([], v)),
     ),
-  ).collect(importList),
+  ).mapValue(v => new ImportCollection(v)),
+  "}",
 );
 
-/** a relative path element like "./" or "../" */
-// prettier-ignore
-const relativeSegment = tagScope(
-  seq(
-    or(".", "..")                   .ptag("segment"), 
-    "/"
-  )                                 .collect(importSegment),
-)                                   .ctag("p");
-
-const lastSegment = or(lastSimpleSegment, importCollection);
+const import_relative = seq(
+  or("package", "super").mapValue(segment),
+  "::",
+  repeat(terminated(or("super").mapValue(segment), "::")),
+).mapValue(v => segments(v[0], v[2]));
 
-// prettier-ignore
-const packageTail = seq(
-  repeat(
-    seq(
-      simpleSegment                 .ctag("p"), 
-      "/"
-    )
-  ),
-  lastSegment                       .ctag("p"),
-);
-
-/** a module path starting with ../ or ./ */
-const relativePath = seq(repeatPlus(relativeSegment), packageTail);
-
-// prettier-ignore
-const packagePrefix = tagScope(
-  seq(
-    wordToken                     .ptag("segment"), 
-    "/"
-  )                               .collect(importSegment),
-)                                 .ctag("p");
-
-/** a module path, starting with a simple element */
-packagePath = seq(packagePrefix, packageTail);
-
-const fullPath = noSkipWs(
-  seq(kind(importTokens.ws), or(relativePath, packagePath)),
+const import_package = terminated(wordToken.mapValue(segment), "::").mapValue(
+  segments,
 );
 
 /** parse a WESL style wgsl import statement. */
-// prettier-ignore
-export const weslImport = tagScope(
+export const weslImport: Parser<ImportElem, NoTags> = tagScope(
   tokens(
-    importTokens,
-    seq("import", fullPath, opt(";"), eolf)     .collect(importElem),
+    mainTokens,
+    delimited(
+      "import",
+      seq(
+        or(import_relative, import_package),
+        or(import_collection, import_path, item_import),
+      ).mapValue(v => {
+        if (v[1] instanceof ImportStatement) {
+          return new ImportStatement(
+            segments(v[0], v[1].segments),
+            v[1].finalSegment,
+          );
+        } else {
+          return new ImportStatement(v[0], v[1]);
+        }
+      }),
+      ";",
+    )
+      .span()
+      .mapValue(
+        (v): ImportElem => ({
+          kind: "import",
+          contents: [],
+          imports: v.value,
+          start: v.span[0],
+          end: v.span[1],
+        }),
+      )
+      .ptag("owo")
+      .collect(importElem),
   ),
 );
 
 if (tracing) {
   const names: Record<string, Parser<unknown, TagRecord>> = {
-    simpleSegment,
-    lastSimpleSegment,
-    importCollection,
-    relativeSegment,
-    relativePath,
-    packagePrefix,
-    packagePath,
-    fullPath,
+    item_import,
+    import_path,
+    import_collection,
+    import_relative,
+    import_package,
     weslImport,
   };