Add --pyodide-source option and remove manual downloading of the prebuilt packages relying on Pyodide's caching mechanism

packages/desktop/README.md

@@ -95,6 +95,19 @@ See the [./samples](./samples) directory for sample projects.
 To make your app secure, be sure to use the latest version of Electron.
 This is [announced](https://www.electronjs.org/docs/latest/tutorial/security#16-use-a-current-version-of-electron) as one of the security best practices in the Electron document too.
 
+## Use a custom Pyodide source
+
+The `dump` command downloads some Pyodide resources such as the prebuilt package wheel files from [the JsDelivr CDN](https://pyodide.org/en/stable/usage/downloading-and-deploying.html#cdn) by default.
+If you want to use a different Pyodide source, for example when accessing JsDelivr (`cdn.jsdelivr.net`) is restricted in your environment,
+you can specify a URL or a path to the Pyodide source by setting the `--pyodide-source` option of the `dump` command.
+
+For example, if you downloaded a Pyodide package from the [Pyodide releases](https://pyodide.org/en/stable/usage/downloading-and-deploying.html#github-releases) and saved it in `/path/to/pyodide/`, you can specify the URL to the Pyodide package like below.
+
+```sh
+npm run dump -- --pyodide-source /path/to/pyodide/
+yarn dump --pyodide-source /path/to/pyodide/
+```
+
 ## Configure the app
 
 ### Hide the toolbar, hamburger menu, and the footer

packages/desktop/bin-src/dump_artifacts/index.ts

@@ -5,9 +5,12 @@ import { hideBin } from "yargs/helpers";
 import path from "node:path";
 import fsPromises from "node:fs/promises";
 import fsExtra from "fs-extra";
-import { loadPyodide, type PyodideInterface } from "pyodide";
-import { makePyodideUrl } from "./url";
-import { PrebuiltPackagesData } from "./pyodide_packages";
+import {
+  loadPyodide,
+  version as pyodideVersion,
+  type PyodideInterface,
+} from "pyodide";
+import { PrebuiltPackagesDataReader } from "./pyodide_packages";
 import { dumpManifest } from "./manifest";
 import { readConfig } from "./config";
 import { validateRequirements, parseRequirementsTxt } from "@stlite/common";
@@ -76,22 +79,30 @@ async function copyBuildDirectory(options: CopyBuildDirectoryOptions) {
   await fsExtra.copy(sourceDir, options.copyTo, { errorOnExist: true });
 }
 
-interface InspectUsedPrebuiltPackagesOptions {
+interface LoadUsedPrebuiltPackagesOptions {
+  pyodideSource: string;
+  pyodideRuntimeDir: string;
   requirements: string[];
 }
 /**
- * Get the list of the prebuilt packages used by the given requirements.
- * These package files (`pyodide/*.whl`) will be vendored in the app executable
- * and loaded at runtime to avoid problems such as https://github.com/whitphx/stlite/issues/558
+ * Load the Pyodide runtime and install the given requirements to load the prebuilt packages used by the requirements.
+ * Those prebuilt package wheels will be downloaded/copied to a local directory, `pyodideRuntimeDir`.
+ * Pyodide's caching mechanism available in the Node environment is used here as the wheel file downloader.
+ * `pyodideRuntimeDir` should be "build/pyodide" so that the downloaded/copied files will be vendored in the app executable.
+ * This vendoring and runtime-loading mechanism is necessary to avoid problems such as https://github.com/whitphx/stlite/issues/558
  */
-async function inspectUsedPrebuiltPackages(
-  options: InspectUsedPrebuiltPackagesOptions
+async function saveUsedPrebuiltPackages(
+  options: LoadUsedPrebuiltPackagesOptions
 ): Promise<string[]> {
   if (options.requirements.length === 0) {
     return [];
   }
 
-  const pyodide = await loadPyodide();
+  const pyodide = await loadPyodide({
+    packageCacheDir: options.pyodideRuntimeDir,
+  });
+  // @ts-ignore
+  pyodide._api.setCdnUrl(options.pyodideSource);
 
   await installPackages(pyodide, {
     requirements: options.requirements,
@@ -148,35 +159,45 @@ async function installPackages(
 interface CreateSitePackagesSnapshotOptions {
   requirements: string[];
   usedPrebuiltPackages: string[];
+  pyodideRuntimeDir: string;
+  pyodideSource: string;
   saveTo: string;
 }
 async function createSitePackagesSnapshot(
   options: CreateSitePackagesSnapshotOptions
 ) {
   logger.info("Create the site-packages snapshot file...");
 
-  const pyodide = await loadPyodide();
+  const pyodide = await loadPyodide({
+    packageCacheDir: options.pyodideRuntimeDir,
+  });
+  // @ts-ignore
+  pyodide._api.setCdnUrl(options.pyodideSource);
 
   await ensureLoadPackage(pyodide, "micropip");
   const micropip = pyodide.pyimport("micropip");
 
-  const prebuiltPackagesData = await PrebuiltPackagesData.getInstance();
+  const prebuiltPackagesDataReader = new PrebuiltPackagesDataReader(
+    options.pyodideSource
+  );
 
   const mockedPackages: string[] = [];
   if (options.usedPrebuiltPackages.length > 0) {
     logger.info(
       "Mocking prebuilt packages so that they will not be included in the site-packages snapshot because these will be installed from the vendored wheel files at runtime..."
     );
-    options.usedPrebuiltPackages.forEach((pkg) => {
-      const packageInfo = prebuiltPackagesData.getPackageInfoByName(pkg);
+    for (const pkg of options.usedPrebuiltPackages) {
+      const packageInfo = await prebuiltPackagesDataReader.getPackageInfoByName(
+        pkg
+      );
       if (packageInfo == null) {
         throw new Error(`Package ${pkg} is not found in the lock file.`);
       }
 
       logger.debug(`Mock ${packageInfo.name} ${packageInfo.version}`);
       micropip.add_mock_package(packageInfo.name, packageInfo.version);
       mockedPackages.push(packageInfo.name);
-    });
+    }
   }
 
   logger.info(`Install the requirements %j`, options.requirements);
@@ -215,7 +236,7 @@ async function createSitePackagesSnapshot(
 interface CopyAppDirectoryOptions {
   cwd: string;
   filePathPatterns: string[];
-  buildAppDirectory: string;
+  destAppDir: string;
 }
 
 async function copyAppDirectory(options: CopyAppDirectoryOptions) {
@@ -238,7 +259,7 @@ async function copyAppDirectory(options: CopyAppDirectoryOptions) {
       await Promise.all(
         fileRelPaths.map(async (relPath) => {
           const srcPath = path.resolve(options.cwd, relPath);
-          const destPath = path.resolve(options.buildAppDirectory, relPath);
+          const destPath = path.resolve(options.destAppDir, relPath);
           logger.debug(`Copy ${srcPath} to ${destPath}`);
           await fsExtra.copy(srcPath, destPath, {
             errorOnExist: true,
@@ -249,12 +270,12 @@ async function copyAppDirectory(options: CopyAppDirectoryOptions) {
   );
 }
 
-async function assertAppDirectoryContainsEntrypoint(
-  appDirectory: string,
+async function assertAppDirContainsEntrypoint(
+  appDir: string,
   entrypoint: string
 ) {
   try {
-    await fsPromises.access(path.resolve(appDirectory, entrypoint));
+    await fsPromises.access(path.resolve(appDir, entrypoint));
   } catch {
     throw new Error(
       `The entrypoint file "${entrypoint}" is not included in the bundled files.`
@@ -281,42 +302,6 @@ async function writePrebuiltPackagesTxt(
   });
 }
 
-interface DownloadPrebuiltPackageWheelsOptions {
-  packages: string[];
-  destDir: string;
-}
-async function downloadPrebuiltPackageWheels(
-  options: DownloadPrebuiltPackageWheelsOptions
-) {
-  const prebuiltPackagesData = await PrebuiltPackagesData.getInstance();
-  const usedPrebuiltPackages = options.packages.map((pkgName) =>
-    prebuiltPackagesData.getPackageInfoByName(pkgName)
-  );
-  const usedPrebuiltPackageUrls = usedPrebuiltPackages.map((pkg) =>
-    makePyodideUrl(pkg.file_name)
-  );
-
-  logger.info("Downloading the used prebuilt packages...");
-  await Promise.all(
-    usedPrebuiltPackageUrls.map(async (pkgUrl) => {
-      const dstPath = path.resolve(
-        options.destDir,
-        "./pyodide",
-        path.basename(pkgUrl)
-      );
-      logger.debug(`Download ${pkgUrl} to ${dstPath}`);
-      const res = await fetch(pkgUrl);
-      if (!res.ok) {
-        throw new Error(
-          `Failed to download ${pkgUrl}: ${res.status} ${res.statusText}`
-        );
-      }
-      const buf = await res.arrayBuffer();
-      await fsPromises.writeFile(dstPath, Buffer.from(buf));
-    })
-  );
-}
-
 yargs(hideBin(process.argv))
   .command(
     "* [appHomeDirSource] [packages..]",
@@ -355,6 +340,18 @@ yargs(hideBin(process.argv))
     alias: "k",
     describe: "Keep the existing build directory contents except appHomeDir.",
   })
+  .options("pyodideSource", {
+    type: "string",
+    describe:
+      "The base URL or path of the Pyodide files to download or copy, such as the prebuild package wheels and pyodide-lock.json",
+    default: `https://cdn.jsdelivr.net/pyodide/v${pyodideVersion}/full/`,
+    coerce: (urlOrPath) => {
+      if (!urlOrPath.endsWith("/")) {
+        urlOrPath += "/";
+      }
+      return urlOrPath;
+    },
+  })
   .options("logLevel", {
     type: "string",
     default: "info",
@@ -406,27 +403,31 @@ yargs(hideBin(process.argv))
     ]);
     logger.info("Validated dependency list: %j", dependencies);
 
-    const usedPrebuiltPackages = await inspectUsedPrebuiltPackages({
+    await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild });
+
+    const usedPrebuiltPackages = await saveUsedPrebuiltPackages({
+      pyodideSource: args.pyodideSource,
+      pyodideRuntimeDir: path.resolve(destDir, "./pyodide"),
       requirements: dependencies,
     });
     logger.info(
       "The prebuilt packages loaded for the given requirements: %j",
       usedPrebuiltPackages
     );
 
-    await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild });
-
-    const buildAppDirectory = path.resolve(destDir, "./app_files"); // This path will be loaded in the `readStreamlitAppDirectory` handler in electron/main.ts.
+    const destAppDir = path.resolve(destDir, "./app_files"); // This path will be loaded in the `readStreamlitAppDirectory` handler in electron/main.ts.
     await copyAppDirectory({
       cwd: projectDir,
       filePathPatterns: config.files,
-      buildAppDirectory,
+      destAppDir,
     });
-    assertAppDirectoryContainsEntrypoint(buildAppDirectory, config.entrypoint);
+    assertAppDirContainsEntrypoint(destAppDir, config.entrypoint);
 
     await createSitePackagesSnapshot({
       requirements: dependencies,
       usedPrebuiltPackages,
+      pyodideSource: args.pyodideSource,
+      pyodideRuntimeDir: path.resolve(destDir, "./pyodide"),
       saveTo: path.resolve(destDir, "./site-packages-snapshot.tar.gz"), // This path will be loaded in the `readSitePackagesSnapshot` handler in electron/main.ts.
     });
     // These prebuilt packages will be vendored in the build artifact by `downloadPrebuiltPackageWheels()`
@@ -439,10 +440,6 @@ yargs(hideBin(process.argv))
       path.resolve(destDir, "./prebuilt-packages.txt"), // This path will be loaded in the `readRequirements` handler in electron/main.ts.
       usedPrebuiltPackages
     );
-    await downloadPrebuiltPackageWheels({
-      packages: usedPrebuiltPackages,
-      destDir,
-    });
     await dumpManifest({
       packageJsonStliteDesktopField: packageJson.stlite?.desktop,
       manifestFilePath: path.resolve(destDir, "./stlite-manifest.json"),

packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts

@@ -1,4 +1,5 @@
-import { makePyodideUrl } from "./url";
+import path from "node:path";
+import fsPromises from "node:fs/promises";
 import { logger } from "./logger";
 
 interface PackageInfo {
@@ -7,39 +8,57 @@ interface PackageInfo {
   file_name: string;
   depends: string[];
 }
-export class PrebuiltPackagesData {
-  private static _instance: PrebuiltPackagesData;
+export class PrebuiltPackagesDataReader {
+  private sourceUrl: string;
+  private isRemote: boolean;
   private _data: Record<string, PackageInfo> | null = null;
 
-  private constructor() {}
-
-  private static async loadPrebuiltPackageData(): Promise<
-    Record<string, PackageInfo>
-  > {
-    const url = makePyodideUrl("pyodide-lock.json");
+  constructor(sourceUrl: string) {
+    // These path logics are based on https://github.com/pyodide/pyodide/blob/0.25.1/src/js/compat.ts#L122
+    if (sourceUrl.startsWith("file://")) {
+      // handle file:// with filesystem operations rather than with fetch.
+      sourceUrl = sourceUrl.slice("file://".length);
+    }
+    this.sourceUrl = sourceUrl;
+    this.isRemote = sourceUrl.includes("://");
+  }
 
-    logger.info(`Load the Pyodide pyodide-lock.json from ${url}`);
-    const res = await fetch(url, undefined);
-    const resJson = await res.json();
+  private async readJson(filepath: string): Promise<any> {
+    const url = path.join(this.sourceUrl, filepath);
 
-    return resJson.packages;
+    if (this.isRemote) {
+      logger.debug(`Fetching ${url}`);
+      const res = await fetch(url);
+      if (!res.ok) {
+        throw new Error(
+          `Failed to download ${url}: ${res.status} ${res.statusText}`
+        );
+      }
+      return await res.json();
+    } else {
+      logger.debug(`Reading ${url}`);
+      const buf = await fsPromises.readFile(url);
+      return JSON.parse(buf.toString());
+    }
   }
 
-  static async getInstance(): Promise<PrebuiltPackagesData> {
-    if (this._instance == null) {
-      this._instance = new PrebuiltPackagesData();
-      this._instance._data = await this.loadPrebuiltPackageData();
+  private async loadPrebuiltPackageData(): Promise<
+    Record<string, PackageInfo>
+  > {
+    if (this._data != null) {
+      return this._data;
     }
-    return this._instance;
+
+    logger.info(`Load pyodide-lock.json`);
+    const lockJson = await this.readJson("pyodide-lock.json");
+
+    this._data = lockJson.packages;
+    return lockJson.packages;
   }
 
-  public getPackageInfoByName(pkgName: string): PackageInfo {
-    if (this._data == null) {
-      throw new Error("The package data is not loaded yet.");
-    }
-    const pkgInfo = Object.values(this._data).find(
-      (pkg) => pkg.name === pkgName
-    );
+  public async getPackageInfoByName(pkgName: string): Promise<PackageInfo> {
+    const data = await this.loadPrebuiltPackageData();
+    const pkgInfo = Object.values(data).find((pkg) => pkg.name === pkgName);
     if (pkgInfo == null) {
       throw new Error(`Package ${pkgName} is not found in the lock file.`);
     }