ci: test on emscripten target

This adds CI to build libpython3.11 for wasm32-emscripten and
running tests against it. We need to patch instant to work
around the emscripten_get_now:
sebcrozet/instant#47

We also have to patch emscripten to work aroung the "undefined
symbol gxx_personality_v0" error:
emscripten-core/emscripten#17128

I set up a nox file to download and install emscripten,
download and build cpython, set appropriate environment variables
then run cargo test. The workflow just installs python, rust,
node, and nox and runs the nox session.

I xfailed all the test failures. There are problems with datetime.
iter_dict_nosegv and test_filenotfounderror should probably be
fixable. The tests that involve threads or asyncio probably can't
be fixed.

.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - emscripten
   pull_request:
   workflow_dispatch:
 
@@ -339,3 +340,23 @@ jobs:
         with:
           file: coverage.lcov
           name: ${{ matrix.os }}
+
+  emscripten:
+    name: emscripten
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.11.0-beta.1
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: wasm32-unknown-emscripten
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 14
+      - run: pip install nox
+      - name: Test
+        run: nox -s test_emscripten

Cargo.toml

@@ -168,3 +168,8 @@ members = [
 no-default-features = true
 features = ["macros", "num-bigint", "num-complex", "hashbrown", "serde", "multiple-pymethods", "indexmap", "eyre"]
 rustdoc-args = ["--cfg", "docsrs"]
+
+[patch.crates-io]
+# Instant misspells emscripten_get_now by including a leading underscore.
+# https://github.com/sebcrozet/instant/pull/47
+instant = { git = 'https://github.com/hoodmane/instant/', branch= 'emscripten-no-leading-underscore' }

emscripten/Makefile

@@ -0,0 +1,81 @@
+CURDIR=$(abspath .)
+
+BUILDROOT ?= $(CURDIR)/builddir
+export EMSDKDIR = $(BUILDROOT)/emsdk
+
+PLATFORM=wasm32_emscripten
+SYSCONFIGDATA_NAME=_sysconfigdata__$(PLATFORM)
+
+# BASH_ENV tells bash to run pyodide_env.sh on startup, which sets various
+# environment variables. The next line instructs make to use bash to run each
+# command.
+export BASH_ENV := $(CURDIR)/env.sh
+SHELL := /bin/bash
+
+EMSCRIPTEN_VERSION=3.1.13
+
+PYMAJORMINORMICRO ?= 3.11.0
+PYPRERELEASE ?= b1
+
+version_tuple := $(subst ., ,$(PYMAJORMINORMICRO:v%=%))
+export PYMAJOR=$(word 1,$(version_tuple))
+export PYMINOR=$(word 2,$(version_tuple))
+export PYMICRO=$(word 3,$(version_tuple))
+PYVERSION=$(PYMAJORMINORMICRO)$(PYPRERELEASE)
+PYMAJORMINOR=$(PYMAJOR).$(PYMINOR)
+
+
+PYTHONURL=https://www.python.org/ftp/python/$(PYMAJORMINORMICRO)/Python-$(PYVERSION).tgz
+PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz
+PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION)
+
+export PYTHONLIBDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/lib
+
+all: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a
+
+$(BUILDROOT)/.exists: 
+	mkdir -p $(BUILDROOT)
+	touch $@
+
+
+$(EMSDKDIR): $(CURDIR)/emscripten_patches/* $(BUILDROOT)/.exists
+	git clone https://github.com/emscripten-core/emsdk.git --depth 1 --branch $(EMSCRIPTEN_VERSION) $(EMSDKDIR)
+	$(EMSDKDIR)/emsdk install $(EMSCRIPTEN_VERSION)
+	cd $(EMSDKDIR)/upstream/emscripten && cat $(CURDIR)/emscripten_patches/* | patch -p1
+	$(EMSDKDIR)/emsdk activate $(EMSCRIPTEN_VERSION)
+
+
+$(PYTHONTARBALL):
+	[ -d $(BUILDROOT)/downloads ] || mkdir -p $(BUILDROOT)/downloads
+	wget -q -O $@ $(PYTHONURL)
+
+$(PYTHONBUILD)/.patched: $(PYTHONTARBALL)
+	[ -d $(PYTHONBUILD) ] || ( \
+		mkdir -p $(dir $(PYTHONBUILD));\
+		tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \
+	)
+	touch $@
+
+$(PYTHONBUILD)/Makefile: $(PYTHONBUILD)/.patched $(BUILDROOT)/emsdk
+	cd $(PYTHONBUILD) && \
+	CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
+  	emconfigure ./configure -C \
+		--host=wasm32-unknown-emscripten \
+		--build=$(shell $(PYTHONBUILD)/config.guess) \
+		--with-emscripten-target=browser \
+		--enable-wasm-dynamic-linking \
+		--with-build-python=python3.11
+
+$(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a : $(PYTHONBUILD)/Makefile
+	cd $(PYTHONBUILD) && \
+		emmake make -j3 libpython$(PYMAJORMINOR).a
+
+	_PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_PROJECT_BASE=$(PYTHONBUILD) python3.11 -m sysconfig --generate-posix-vars
+	cp `cat pybuilddir.txt`/$(SYSCONFIGDATA_NAME).py $(PYTHONBUILD)/Lib
+
+	mkdir -p $(PYTHONLIBDIR)
+	find $(PYTHONBUILD) -name '*.a' -exec cp {} $(PYTHONLIBDIR) \;
+	cp -r $(PYTHONBUILD)/Lib $(PYTHONLIBDIR)/python$(PYMAJORMINOR)
+
+clean:
+	rm -rf $(BUILDROOT)

emscripten/emscripten_patches/0001-Add-_gxx_personality_v0-stub-to-library.js.patch

@@ -0,0 +1,27 @@
+From 4b56f37c3dc9185a235a8314086c4d7a6239b2f8 Mon Sep 17 00:00:00 2001
+From: Hood Chatham <roberthoodchatham@gmail.com>
+Date: Sat, 4 Jun 2022 19:19:47 -0700
+Subject: [PATCH] Add _gxx_personality_v0 stub to library.js
+
+Mitigation for:
+https://github.com/emscripten-core/emscripten/issues/17128
+---
+ src/library.js | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/src/library.js b/src/library.js
+index e7bb4c38e..7d01744df 100644
+--- a/src/library.js
++++ b/src/library.js
+@@ -403,6 +403,8 @@ mergeInto(LibraryManager.library, {
+     abort('Assertion failed: ' + UTF8ToString(condition) + ', at: ' + [filename ? UTF8ToString(filename) : 'unknown filename', line, func ? UTF8ToString(func) : 'unknown function']);
+   },
+
++  __gxx_personality_v0: function() {},
++
+   // ==========================================================================
+   // time.h
+   // ==========================================================================
+-- 
+2.25.1
+

emscripten/env.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+
+# emsdk_env.sh is fairly noisy, and suppress error message if the file doesn't
+# exist yet (i.e. before building emsdk)
+# shellcheck source=/dev/null
+source "$EMSDKDIR/emsdk_env.sh" 2> /dev/null || true
+EMCC_PATH=$(which emcc.py || echo ".")
+EM_DIR=$(dirname "$EMCC_PATH")
+export EM_DIR

emscripten/runner.py

@@ -0,0 +1,8 @@
+#!/usr/local/bin/python
+import pathlib
+import sys
+import subprocess
+
+p = pathlib.Path(sys.argv[1])
+
+sys.exit(subprocess.call(["node", p.name], cwd=p.parent))

noxfile.py

@@ -1,5 +1,7 @@
 import time
 from glob import glob
+from pathlib import Path
+import re
 
 import nox
 
@@ -128,3 +130,49 @@ def contributors(session: nox.Session) -> None:
 
     for author in authors:
         print(f"@{author}")
+
+
+@nox.session(venv_backend="none")
+def test_emscripten(session: nox.Session):
+    builddir = Path(session.create_tmp()).absolute().parent
+    emscripten_dir = Path(__file__).parent / "emscripten"
+
+    pyversion = "3.11.0b1"
+    pymajor, pyminor, pymicro = pyversion.split(".")
+    pymicro, pydev = re.match("([0-9]*)([^0-9].*)?", pymicro).groups()
+    if pydev is None:
+        pydev = ""
+
+    pymajorminor = f"{pymajor}.{pyminor}"
+    pymajorminormicro = f"{pymajorminor}.{pymicro}"
+    session.run(
+        "make",
+        "-C",
+        str(emscripten_dir),
+        f"BUILDROOT={builddir}",
+        f"PYMAJORMINORMICRO={pymajorminormicro}",
+        f"PYPRERELEASE={pydev}",
+        external=True,
+    )
+    libdir = builddir / f"install/Python-{pyversion}/lib"
+    pythonlibdir = libdir / f"python{pymajorminor}"
+
+    target = "wasm32-unknown-emscripten"
+
+    session.env["CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUNNER"] = "python " + str(
+        emscripten_dir / "runner.py"
+    )
+    session.env["RUSTFLAGS"] = " ".join(
+        [
+            f"-L native={libdir}",
+            "-C link-arg=--preload-file",
+            f"-C link-arg={pythonlibdir}@/lib/python{pymajorminor}",
+            f"-C link-arg=-lpython{pymajorminor}",
+            "-C link-arg=-lexpat",
+            "-C link-arg=-lmpdec",
+        ]
+    )
+    session.env["CARGO_BUILD_TARGET"] = target
+    session.env["PYO3_CROSS_LIB_DIR"] = pythonlibdir
+    session.run("rustup", "target", "add", target, "--toolchain", "stable")
+    session.run("bash", "-c", f"source {builddir/'emsdk/emsdk_env.sh'} && cargo test")

src/ffi/tests.rs

@@ -6,6 +6,7 @@ use crate::types::PyString;
 #[cfg(target_endian = "little")]
 use libc::wchar_t;
 
+#[cfg_attr(target_arch = "wasm32", ignore)]
 #[test]
 fn test_datetime_fromtimestamp() {
     Python::with_gil(|py| {
@@ -23,6 +24,7 @@ fn test_datetime_fromtimestamp() {
     })
 }
 
+#[cfg_attr(target_arch = "wasm32", ignore)]
 #[test]
 fn test_date_fromtimestamp() {
     Python::with_gil(|py| {
@@ -40,6 +42,7 @@ fn test_date_fromtimestamp() {
     })
 }
 
+#[cfg_attr(target_arch = "wasm32", ignore)]
 #[test]
 fn test_utc_timezone() {
     Python::with_gil(|py| {
@@ -183,6 +186,7 @@ fn ucs4() {
 }
 
 #[test]
+#[cfg_attr(target_arch = "wasm32", ignore)]
 #[cfg(not(PyPy))]
 fn test_get_tzinfo() {
     crate::Python::with_gil(|py| {

src/gil.rs

@@ -729,6 +729,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(target_arch = "wasm32", ignore)]
     fn test_clone_without_gil() {
         use crate::{Py, PyAny};
         use std::{sync::Arc, thread};
@@ -799,6 +800,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(target_arch = "wasm32", ignore)]
     fn test_clone_in_other_thread() {
         use crate::Py;
         use std::{sync::Arc, thread};

src/marker.rs

@@ -942,6 +942,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(target_arch = "wasm32", ignore)]
     fn test_allow_threads_releases_and_acquires_gil() {
         Python::with_gil(|py| {
             let b = std::sync::Arc::new(std::sync::Barrier::new(2));

src/types/datetime.rs

@@ -546,6 +546,7 @@ fn opt_to_pyobj(py: Python<'_>, opt: Option<&PyObject>) -> *mut ffi::PyObject {
 #[cfg(test)]
 mod tests {
     #[test]
+    #[cfg_attr(target_arch = "wasm32", ignore)]
     fn test_new_with_fold() {
         crate::Python::with_gil(|py| {
             use crate::types::{PyDateTime, PyTimeAccess};
@@ -560,6 +561,7 @@ mod tests {
 
     #[cfg(not(PyPy))]
     #[test]
+    #[cfg_attr(target_arch = "wasm32", ignore)]
     fn test_get_tzinfo() {
         crate::Python::with_gil(|py| {
             use crate::conversion::ToPyObject;

tests/test_class_basics.rs

@@ -229,6 +229,7 @@ impl UnsendableChild {
     }
 }
 
+#[cfg_attr(target_arch = "wasm32", ignore)]
 fn test_unsendable<T: PyClass + 'static>() -> PyResult<()> {
     let obj = std::thread::spawn(|| -> PyResult<_> {
         Python::with_gil(|py| {
@@ -259,6 +260,7 @@ fn test_unsendable<T: PyClass + 'static>() -> PyResult<()> {
 
 /// If a class is marked as `unsendable`, it panics when accessed by another thread.
 #[test]
+#[cfg_attr(target_arch = "wasm32", ignore)]
 #[should_panic(
     expected = "test_class_basics::UnsendableBase is unsendable, but sent to another thread!"
 )]
@@ -267,6 +269,7 @@ fn panic_unsendable_base() {
 }
 
 #[test]
+#[cfg_attr(target_arch = "wasm32", ignore)]
 #[should_panic(
     expected = "test_class_basics::UnsendableBase is unsendable, but sent to another thread!"
 )]

tests/test_compile_error.rs

@@ -1,13 +1,15 @@
 #![cfg(feature = "macros")]
 
 #[rustversion::stable]
+#[cfg_attr(target_arch = "wasm32", ignore)]
 #[test]
 fn test_compile_errors() {
     // stable - require all tests to pass
     _test_compile_errors()
 }
 
 #[cfg(not(feature = "nightly"))]
+#[cfg_attr(target_arch = "wasm32", ignore)]
 #[rustversion::nightly]
 #[test]
 fn test_compile_errors() {
@@ -17,6 +19,7 @@ fn test_compile_errors() {
 }
 
 #[cfg(feature = "nightly")]
+#[cfg_attr(target_arch = "wasm32", ignore)]
 #[rustversion::nightly]
 #[test]
 fn test_compile_errors() {

tests/test_dict_iter.rs

@@ -2,6 +2,7 @@ use pyo3::prelude::*;
 use pyo3::types::IntoPyDict;
 
 #[test]
+#[cfg_attr(target_arch = "wasm32", ignore)]
 fn iter_dict_nosegv() {
     let gil = Python::acquire_gil();
     let py = gil.python();

tests/test_exceptions.rs

@@ -17,6 +17,7 @@ fn fail_to_open_file() -> PyResult<()> {
 }
 
 #[test]
+#[cfg_attr(target_arch = "wasm32", ignore)]
 #[cfg(not(target_os = "windows"))]
 fn test_filenotfounderror() {
     let gil = Python::acquire_gil();

tests/test_proto_methods.rs

@@ -698,6 +698,7 @@ impl OnceFuture {
 }
 
 #[test]
+#[cfg_attr(target_arch = "wasm32", ignore)]
 fn test_await() {
     let gil = Python::acquire_gil();
     let py = gil.python();
@@ -747,6 +748,7 @@ impl AsyncIterator {
 }
 
 #[test]
+#[cfg_attr(target_arch = "wasm32", ignore)]
 fn test_anext_aiter() {
     let gil = Python::acquire_gil();
     let py = gil.python();