Merge pull request #31 from cognitedata/anders/release-15-02-22

[Production] Bump to 0.47.0 (Streamlit 1.31.0)

.github/actions/init-all/action.yml

@@ -12,7 +12,7 @@ runs:
   steps:
     ## Set up Python and Poetry environment
     - name: Set up Python ${{ inputs.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
 

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "pip"
+    directory: "/packages/kernel/py/stlite-server"
+    schedule:
+      interval: "daily"

.github/workflows/deploy-staging.yml

@@ -141,5 +141,4 @@ jobs:
         channelId: live
         projectId: fusion-217032465111
         target: cognite-stlite-standalone-staging
-
-
+

CHANGELOG.md

@@ -7,6 +7,70 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.47.0] - 2024-02-11
+
+### `@stlite/kernel`
+
+#### Changed
+
+- Stop publishing `@stlite/kernel` to NPM, [#743](https://github.com/whitphx/stlite/pull/743).
+
+## [0.46.1] - 2024-02-11
+
+### `@stlite/kernel`
+
+#### Fixed
+
+- Restore the `pyodide-http` patch which was removed in 0.45.1, [#742](https://github.com/whitphx/stlite/pull/742) and [#738](https://github.com/whitphx/stlite/pull/738), which was released once as 0.45.3 but not effective.
+
+## [0.46.0] - 2024-02-11
+
+### `@stlite/desktop`
+
+#### Changed
+
+- Update Streamlit to 1.31.0, [#741](https://github.com/whitphx/stlite/pull/741)
+
+## [0.45.5] - 2024-02-11
+
+### `@stlite/desktop`
+
+#### Fixed
+
+- Ship the stlite wheels along with the `desktop` package instead of loading them fron the hosted `kernel` package, [#722](https://github.com/whitphx/stlite/pull/722).
+
+## [0.45.4] - 2024-02-11
+
+### `@stlite/common`
+
+#### Fixed
+
+- `requirements.txt` parser to ignore comments, [#739](https://github.com/whitphx/stlite/pull/739).
+
+## [0.45.3] - 2024-02-11
+
+### `@stlite/kernel`
+
+#### Fixed
+
+- Restore the `pyodide-http` patch which was removed in 0.45.1, [#738](https://github.com/whitphx/stlite/pull/738).
+
+## [0.45.2] - 2024-02-08
+
+### `@stlite/*`
+
+#### Fixed
+
+- Internal package updates.
+
+## [0.45.1] - 2024-02-07
+
+### `@stlite/kernel`
+
+#### Fixed
+
+- Remove the HTTP patch because [`urllib3` started to support Pyodide since 2.2.0](https://urllib3.readthedocs.io/en/stable/reference/contrib/emscripten.html), [#689](https://github.com/whitphx/stlite/pull/689).
+
 ## [0.45.0] - 2024-01-20
 
 ### `@stlite/kernel`

Makefile

@@ -8,7 +8,7 @@ desktop := packages/desktop/build/*
 kernel := packages/kernel/dist/*
 stlite-server-wheel := packages/kernel/py/stlite-server/dist/stlite_server-0.1.0-py3-none-any.whl
 streamlit_proto := streamlit/frontend/lib/src/proto.d.ts
-streamlit_wheel := packages/kernel/py/streamlit/lib/dist/streamlit-1.30.0-cp311-none-any.whl
+streamlit_wheel := packages/kernel/py/streamlit/lib/dist/streamlit-1.31.0-cp311-none-any.whl
 streamlit_frontend_lib_prod := streamlit/frontend/lib/dist/*
 
 .PHONY: all
@@ -127,7 +127,7 @@ $(streamlit_wheel): venv $(streamlit_proto) streamlit/lib/streamlit/**/*.py stre
 		exit 1; \
 	fi && \
 	cd streamlit && SNOWPARK_CONDA_BUILD=true $(MAKE) distribution && cd .. && \
-	pyodide py-compile --keep streamlit/lib/dist/streamlit-1.30.0-py2.py3-none-any.whl && \
+	pyodide py-compile --keep streamlit/lib/dist/streamlit-1.31.0-py2.py3-none-any.whl && \
 	mkdir -p $(dir $(streamlit_wheel)) && \
 	cp streamlit/lib/dist/$(notdir $(streamlit_wheel)) $(streamlit_wheel)
 

README.md

@@ -3,7 +3,6 @@
 [![Test, Build, and Publish](https://github.com/whitphx/stlite/actions/workflows/main.yml/badge.svg?branch=main)](https://github.com/whitphx/stlite/actions/workflows/main.yml)
 [![Build and Deploy GitHub Pages](https://github.com/whitphx/stlite/actions/workflows/gh-pages.yml/badge.svg)](https://github.com/whitphx/stlite/actions/workflows/gh-pages.yml)
 
-[![npm (@stlite/kernel)](https://img.shields.io/npm/v/@stlite/kernel?label=%40stlite%2Fkernel)](https://www.npmjs.com/package/@stlite/kernel)
 [![npm (scoped)](https://img.shields.io/npm/v/@stlite/mountable?label=%40stlite%2Fmountable)](https://www.npmjs.com/package/@stlite/mountable)
 [![npm (@stlite/desktop)](https://img.shields.io/npm/v/@stlite/desktop?label=%40stlite%2Fdesktop)](https://www.npmjs.com/package/@stlite/desktop)
 
@@ -298,7 +297,7 @@ As _stlite_ runs on the web browser environment ([Pyodide](https://pyodide.org/)
 - `time.sleep()` is no-op. Use `asyncio.sleep()` instead. This is a restriction from Pyodide runtime. See https://github.com/pyodide/pyodide/issues/2354. The following section about top-level await may also help to know how to use async functions on stlite.
 - There are some small differences in how (less common) data types of DataFrame columns are handled in `st.dataframe()`, `st.data_editor()`, `st.table()`, and Altair-based charts. The reason is that stlite uses the Parquet format instead of the Arrow IPC format to serialize dataframes (Ref: [#601](https://github.com/whitphx/stlite/pull/601)).
 - For URL access, `urllib` and `requests` don't work on Pyodide/stlite, so we have to use alternative methods provided by Pyodide, such as [`pyodide.http.pyfetch()`](https://pyodide.org/en/stable/usage/api/python-api/http.html#pyodide.http.pyfetch) or [`pyodide.http.open_url()`](https://pyodide.org/en/stable/usage/api/python-api/http.html#pyodide.http.open_url). See https://pyodide.org/en/stable/usage/faq.html#how-can-i-load-external-files-in-pyodide for the details. For `pyodide.http.pyfetch()`, see also the following section about top-level await.
-- The C extension packages that are not built for Pyodide cannot be installed. See https://pyodide.org/en/stable/usage/faq.html#micropip-can-t-find-a-pure-python-wheel for the details.
+- Packages including binary extensions (e.g. C/Rust/Fortran/etc) that are not built for the Pyodide environment cannot be installed. See https://pyodide.org/en/stable/usage/faq.html#why-can-t-micropip-find-a-pure-python-wheel-for-a-package for the details.
 
 Other problems are tracked at GitHub Issues: https://github.com/whitphx/stlite/issues
 If you find a new problem, please report it.

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/@lerna-lite/cli/schemas/lerna-schema.json",
-  "version": "0.45.0",
+  "version": "0.47.0",
   "packages": ["packages/*"],
   "command": {
     "version": {

package.json

@@ -12,16 +12,16 @@
     "prepare": "husky install"
   },
   "devDependencies": {
-    "@lerna-lite/cli": "^1.11.0",
+    "@lerna-lite/cli": "^3.3.0",
     "chokidar-cli": "^3.0.0",
-    "concurrently": "^7.4.0",
+    "concurrently": "^8.2.2",
     "cross-env": "^7.0.3",
     "eslint-config-prettier": "^8.6.0",
     "husky": "^8.0.1",
     "lint-staged": "^13.0.3",
     "npm-run-all": "^4.1.5",
     "prettier": "2.7.1",
-    "rimraf": "^3.0.2",
+    "rimraf": "^5.0.5",
     "wait-on": "^6.0.1"
   },
   "resolutions": {

packages/common-react/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stlite/common-react",
-  "version": "0.45.0",
+  "version": "0.47.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "Apache-2.0",
@@ -14,7 +14,7 @@
     "check:prettier": "prettier --check ."
   },
   "dependencies": {
-    "@streamlit/lib": "1.30.0"
+    "@streamlit/lib": "1.31.0"
   },
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^5.49.0",

packages/common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stlite/common",
-  "version": "0.45.0",
+  "version": "0.47.0",
   "main": "dist/cjs/index.js",
   "module": "dist/esm/index.js",
   "types": "dist/esm/index.d.ts",
@@ -28,6 +28,6 @@
     "@typescript-eslint/parser": "^5.49.0",
     "eslint": "^8.33.0",
     "typescript": "^4.9.4",
-    "vitest": "^0.21.1"
+    "vitest": "^1.2.2"
   }
 }

packages/common/src/requirements-txt.test.ts

@@ -7,11 +7,14 @@ describe("parseRequirementsTxt", () => {
 
 packageB
 # comment
-packageC`;
+packageC
+packageD # comment
+`;
     expect(parseRequirementsTxt(content)).toEqual([
       "packageA",
       "packageB",
       "packageC",
+      "packageD",
     ]);
   });
 });

packages/common/src/requirements-txt.ts

@@ -1,7 +1,10 @@
+const pipCommentRegexp = /\s#.*$/;
+
 export function parseRequirementsTxt(content: string): string[] {
   return content
     .split("\n")
-    .filter((r) => !r.startsWith("#"))
+    .filter((r) => !r.startsWith("#")) // https://pip.pypa.io/en/latest/reference/requirements-file-format/#comments
+    .map((r) => r.replace(pipCommentRegexp, "")) // https://pip.pypa.io/en/latest/reference/requirements-file-format/#comments
     .map((r) => r.trim())
     .filter((r) => r !== "");
 }

packages/desktop/.gitignore

@@ -25,3 +25,4 @@ yarn-error.log*
 !.env
 
 /bin/*.js
+/wheels

packages/desktop/README.md

@@ -24,10 +24,10 @@ Convert your [Streamlit](https://streamlit.io/) application into a desktop app w
        }
      },
      "devDependencies": {
-       "@stlite/desktop": "0.39.0",
+       "@stlite/desktop": "^0.46.1",
        "cross-env": "^7.0.3",
-       "electron": "^26.2.4",
-       "electron-builder": "^24.6.4"
+       "electron": "^28.2.1",
+       "electron-builder": "^24.9.1"
      }
    }
    ```

packages/desktop/bin/dump_artifacts.ts

@@ -65,7 +65,6 @@ async function copyBuildDirectory(options: CopyBuildDirectoryOptions) {
 
 interface InspectUsedBuiltinPackagesOptions {
   requirements: string[];
-  useLocalKernelWheels: boolean;
 }
 /**
  * Get the list of the built-in packages used by the given requirements.
@@ -82,7 +81,6 @@ async function inspectUsedBuiltinPackages(
   const pyodide = await loadPyodide();
 
   await installPackages(pyodide, {
-    useLocalKernelWheels: options.useLocalKernelWheels,
     requirements: options.requirements,
   });
 
@@ -123,7 +121,6 @@ async function prepareLocalWheel(
 
 interface InstallStreamlitWheelsOptions {
   requirements: string[];
-  useLocalKernelWheels: boolean;
 }
 async function installPackages(
   pyodide: PyodideInterface,
@@ -133,51 +130,24 @@ async function installPackages(
   const micropip = pyodide.pyimport("micropip");
 
   const requirements: string[] = [...options.requirements];
-  if (options.useLocalKernelWheels) {
-    const stliteKernelDir = path.dirname(require.resolve("@stlite/kernel")); // -> /path/to/kernel/dist
-    const stliteKernelPyDir = path.resolve(stliteKernelDir, "../py"); // -> /path/to/kernel/py
-    // TODO: Set the wheel file names dynamically
-    const stliteServerWheel = await prepareLocalWheel(
-      pyodide,
-      path.join(
-        stliteKernelPyDir,
-        "stlite-server/dist/stlite_server-0.1.0-py3-none-any.whl"
-      )
-    );
-    requirements.push(stliteServerWheel);
-    const streamlitWheel = await prepareLocalWheel(
-      pyodide,
-      path.join(
-        stliteKernelPyDir,
-        "streamlit/lib/dist/streamlit-1.30.0-cp311-none-any.whl"
-      )
-    );
-    requirements.push(streamlitWheel);
-  } else {
-    const packageJson = require(path.resolve(__dirname, "../package.json"));
-    const version = packageJson.version;
-
-    const jsDelivrFilesUrl = `https://data.jsdelivr.com/v1/package/npm/@stlite/kernel@${version}/flat`;
-    const jsDelivrFilesRes = await fetch(jsDelivrFilesUrl);
-    const jsDelivrFilesJson = await jsDelivrFilesRes.json();
-    const wheelFiles = jsDelivrFilesJson.files.filter((fileData) =>
-      fileData.name.endsWith(".whl")
-    );
-    const wheelUrls = wheelFiles.map(
-      (wheelFile) =>
-        `https://cdn.jsdelivr.net/npm/@stlite/kernel@${version}${wheelFile.name}`
-    );
 
-    console.log("Kernel wheels:", wheelUrls);
-    requirements.push(...wheelUrls);
-  }
+  const wheelsDir = path.join(__dirname, "../wheels");
+  const stliteServerWheel = await prepareLocalWheel(
+    pyodide,
+    path.join(wheelsDir, "stlite_server-0.1.0-py3-none-any.whl")
+  );
+  requirements.push(stliteServerWheel);
+  const streamlitWheel = await prepareLocalWheel(
+    pyodide,
+    path.join(wheelsDir, "streamlit-1.31.0-cp311-none-any.whl")
+  );
+  requirements.push(streamlitWheel);
 
   console.log("Install the packages:", requirements);
   await micropip.install.callKwargs(requirements, { keep_going: true });
 }
 
 interface CreateSitePackagesSnapshotOptions {
-  useLocalKernelWheels: boolean;
   requirements: string[];
   usedBuiltinPackages: string[];
   saveTo: string;
@@ -216,7 +186,6 @@ async function createSitePackagesSnapshot(
   );
 
   await installPackages(pyodide, {
-    useLocalKernelWheels: options.useLocalKernelWheels,
     requirements: options.requirements,
   });
 
@@ -382,12 +351,6 @@ yargs(hideBin(process.argv))
     alias: "r",
     default: [],
   })
-  .options("localKernelWheels", {
-    describe: "Use the locally installed kernel wheels",
-    type: "boolean",
-    alias: "l",
-    default: false,
-  })
   .options("keepOldBuild", {
     type: "boolean",
     default: false,
@@ -405,7 +368,7 @@ yargs(hideBin(process.argv))
       throw new Error(`${args.appHomeDirSource} does not exist.`);
     }
 
-    let requirements = args.packages;
+    let requirements = args.packages ?? [];
     for (const requirementTxtFilePath of args.requirement) {
       requirements = requirements.concat(
         await readRequirements(requirementTxtFilePath)
@@ -415,14 +378,12 @@ yargs(hideBin(process.argv))
 
     const usedBuiltinPackages = await inspectUsedBuiltinPackages({
       requirements: requirements,
-      useLocalKernelWheels: args.localKernelWheels,
     });
     console.log("The built-in packages loaded for the given requirements:");
     console.log(usedBuiltinPackages);
 
     await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild });
     await createSitePackagesSnapshot({
-      useLocalKernelWheels: args.localKernelWheels,
       requirements: requirements,
       usedBuiltinPackages,
       saveTo: path.resolve(destDir, "./site-packages-snapshot.tar.gz"), // This path will be loaded in the `readSitePackagesSnapshot` handler in electron/main.ts.