Fix building sdist and wheels and conda packages

.github/workflows/main.yml

@@ -507,8 +507,8 @@ jobs:
           BOOST_ROOT=$(echo $BOOST_ROOT | sed 's/\\/\//g')
           mkdir -p $BOOST_ROOT
           curl --progress-bar --location --output $BOOST_ROOT/download.7z $BOOST_URL
-          7z -o$BOOST_ROOT x $BOOST_ROOT/download.7z -y -bd boost_1_75_0/boost
-          mv $BOOST_ROOT/boost_1_75_0/boost $BOOST_ROOT/boost
+          7z -o$BOOST_ROOT x $BOOST_ROOT/download.7z -y -bd boost_1_78_0/boost
+          mv $BOOST_ROOT/boost_1_78_0/boost $BOOST_ROOT/boost
           rm $BOOST_ROOT/download.7z
         shell: bash
       - name: Build Cantera
@@ -647,8 +647,8 @@ jobs:
         BOOST_ROOT=$(echo $BOOST_ROOT | sed 's/\\/\//g')
         mkdir -p $BOOST_ROOT
         curl --progress-bar --location --output $BOOST_ROOT/download.7z $BOOST_URL
-        7z -o$BOOST_ROOT x $BOOST_ROOT/download.7z -y -bd boost_1_75_0/boost
-        mv $BOOST_ROOT/boost_1_75_0/boost $BOOST_ROOT/boost
+        7z -o$BOOST_ROOT x $BOOST_ROOT/download.7z -y -bd boost_1_78_0/boost
+        mv $BOOST_ROOT/boost_1_78_0/boost $BOOST_ROOT/boost
         rm $BOOST_ROOT/download.7z
       shell: bash
     - name: Build Cantera

SConstruct

@@ -851,6 +851,22 @@ if 'sphinx' in COMMAND_LINE_TARGETS:
 if "sdist" in COMMAND_LINE_TARGETS:
     env["python_sdist"] = True
 
+    if env["python_package"] == "default":
+        logger.info("'sdist' target was specified. Setting 'python_package' to none.")
+        env["python_package"] = "none"
+    elif env["python_package"] in ("full", "y"):
+        logger.error("'sdist' target was specified. Cannot also build Python package.")
+        sys.exit(1)
+    for ext_dependency in ("sundials", "fmt", "yamlcpp", "eigen"):
+        if env[f"system_{ext_dependency}"] == "y":
+            logger.error(f"'sdist' target was specified. Cannot use 'system_{ext_dependency}'.")
+            sys.exit(1)
+        else:
+            env[f"system_{ext_dependency}"] = "n"
+
+    logger.info("'sdist' target was specified. Setting 'use_pch' to False.")
+    env["use_pch"] = False
+
 for arg in ARGUMENTS:
     if arg not in config:
         logger.error(f"Encountered unexpected command line option: '{arg}'")
@@ -1979,7 +1995,7 @@ else:
     env["cantera_shared_libs"] = ["cantera"] + env["external_libs"]
 
 # Add targets from the SConscript files in the various subdirectories
-Export('env', 'build', 'libraryTargets', 'install', 'buildSample')
+Export('env', 'build', 'libraryTargets', 'install', 'buildSample', "configh")
 
 # ext needs to come before src so that libraryTargets is fully populated
 VariantDir('build/ext', 'ext', duplicate=0)

interfaces/cython/cantera/base.pyx

@@ -417,7 +417,7 @@ cdef class _SolutionBase:
             Y.skip_user_defined = skip_user_defined
         if filename is None:
             return Y.to_string()
-        Y.to_file(filename)
+        Y.to_file(str(filename))
 
     def __getitem__(self, selection):
         copy = self.__class__(origin=self)

interfaces/cython/cantera/test/test_composite.py

@@ -122,10 +122,10 @@ class TestPickle(utilities.CanteraTest):
     def test_pickle_gas(self):
         gas = ct.Solution("h2o2.yaml", transport_model=None)
         gas.TPX = 500, 500000, "H2:.75,O2:.25"
-        with open("gas.pkl", "wb") as pkl:
+        with open(self.test_work_path / "gas.pkl", "wb") as pkl:
             pickle.dump(gas, pkl)
 
-        with open("gas.pkl", "rb") as pkl:
+        with open(self.test_work_path / "gas.pkl", "rb") as pkl:
             gas2 = pickle.load(pkl)
         self.assertNear(gas.T, gas2.T)
         self.assertNear(gas.P, gas2.P)
@@ -137,10 +137,10 @@ def test_pickle_gas_with_transport(self):
         gas = ct.Solution("h2o2.yaml")
         gas.TPX = 500, 500000, "H2:.75,O2:.25"
         gas.transport_model = "Multi"
-        with open("gas.pkl", "wb") as pkl:
+        with open(self.test_work_path / "gas.pkl", "wb") as pkl:
             pickle.dump(gas, pkl)
 
-        with open("gas.pkl", "rb") as pkl:
+        with open(self.test_work_path / "gas.pkl", "rb") as pkl:
             gas2 = pickle.load(pkl)
         self.assertNear(gas.T, gas2.T)
         self.assertNear(gas.P, gas2.P)
@@ -154,7 +154,7 @@ def test_pickle_interface(self):
         interface = ct.Interface("diamond.yaml", "diamond_100", (gas, solid))
 
         with self.assertRaises(NotImplementedError):
-            with open("interface.pkl", "wb") as pkl:
+            with open(self.test_work_path / "interface.pkl", "wb") as pkl:
                 pickle.dump(interface, pkl)
 
 
@@ -660,8 +660,8 @@ def test_yaml_simple(self):
         gas.TPX = 500, ct.one_atm, 'H2: 1.0, O2: 1.0'
         gas.equilibrate('HP')
         gas.TP = 1500, ct.one_atm
-        gas.write_yaml('h2o2-generated.yaml')
-        generated = utilities.load_yaml("h2o2-generated.yaml")
+        gas.write_yaml(self.test_work_path / "h2o2-generated.yaml")
+        generated = utilities.load_yaml(self.test_work_path / "h2o2-generated.yaml")
         for key in ('generator', 'date', 'phases', 'species', 'reactions'):
             self.assertIn(key, generated)
         self.assertEqual(generated['phases'][0]['transport'], 'mixture-averaged')
@@ -670,7 +670,7 @@ def test_yaml_simple(self):
         for blessed, generated in zip(gas.reactions(), generated["reactions"]):
             assert blessed.equation == generated["equation"]
 
-        gas2 = ct.Solution("h2o2-generated.yaml")
+        gas2 = ct.Solution(self.test_work_path / "h2o2-generated.yaml")
         self.assertArrayNear(gas.concentrations, gas2.concentrations)
         self.assertArrayNear(gas.partial_molar_enthalpies,
                              gas2.partial_molar_enthalpies)
@@ -684,8 +684,8 @@ def test_yaml_outunits1(self):
         gas.equilibrate('HP')
         gas.TP = 1500, ct.one_atm
         units = {'length': 'cm', 'quantity': 'mol', 'energy': 'cal'}
-        gas.write_yaml('h2o2-generated.yaml', units=units)
-        generated = utilities.load_yaml("h2o2-generated.yaml")
+        gas.write_yaml(self.test_work_path / "h2o2-generated.yaml", units=units)
+        generated = utilities.load_yaml(self.test_work_path / "h2o2-generated.yaml")
         original = utilities.load_yaml(self.cantera_data_path / "h2o2.yaml")
         self.assertEqual(generated['units'], units)
 
@@ -694,7 +694,7 @@ def test_yaml_outunits1(self):
                 self.assertNear(r1['rate-constant']['A'], r2['rate-constant']['A'])
                 self.assertNear(r1['rate-constant']['Ea'], r2['rate-constant']['Ea'])
 
-        gas2 = ct.Solution("h2o2-generated.yaml")
+        gas2 = ct.Solution(self.test_work_path / "h2o2-generated.yaml")
         self.assertArrayNear(gas.concentrations, gas2.concentrations)
         self.assertArrayNear(gas.partial_molar_enthalpies,
                              gas2.partial_molar_enthalpies)
@@ -709,16 +709,16 @@ def test_yaml_outunits2(self):
         gas.TP = 1500, ct.one_atm
         units = {'length': 'cm', 'quantity': 'mol', 'energy': 'cal'}
         system = ct.UnitSystem(units)
-        gas.write_yaml('h2o2-generated.yaml', units=system)
-        generated = utilities.load_yaml("h2o2-generated.yaml")
+        gas.write_yaml(self.test_work_path / "h2o2-generated.yaml", units=system)
+        generated = utilities.load_yaml(self.test_work_path / "h2o2-generated.yaml")
         original = utilities.load_yaml(self.cantera_data_path / "h2o2.yaml")
 
         for r1, r2 in zip(original['reactions'], generated['reactions']):
             if 'rate-constant' in r1:
                 self.assertNear(r1['rate-constant']['A'], r2['rate-constant']['A'])
                 self.assertNear(r1['rate-constant']['Ea'], r2['rate-constant']['Ea'])
 
-        gas2 = ct.Solution("h2o2-generated.yaml")
+        gas2 = ct.Solution(self.test_work_path / "h2o2-generated.yaml")
         self.assertArrayNear(gas.concentrations, gas2.concentrations)
         self.assertArrayNear(gas.partial_molar_enthalpies,
                              gas2.partial_molar_enthalpies)
@@ -727,14 +727,14 @@ def test_yaml_outunits2(self):
         self.assertArrayNear(gas.mix_diff_coeffs, gas2.mix_diff_coeffs)
 
     def check_ptcombust(self, gas, surf):
-        generated = utilities.load_yaml("ptcombust-generated.yaml")
+        generated = utilities.load_yaml(self.test_work_path / "ptcombust-generated.yaml")
         for key in ("phases", "species", "gas-reactions", "Pt_surf-reactions"):
             self.assertIn(key, generated)
         self.assertEqual(len(generated["gas-reactions"]), gas.n_reactions)
         self.assertEqual(len(generated["Pt_surf-reactions"]), surf.n_reactions)
         self.assertEqual(len(generated["species"]), surf.n_total_species)
 
-        surf2 = ct.Solution("ptcombust-generated.yaml", "Pt_surf")
+        surf2 = ct.Solution(self.test_work_path / "ptcombust-generated.yaml", "Pt_surf")
         self.assertArrayNear(surf.concentrations, surf2.concentrations)
         self.assertArrayNear(surf.partial_molar_enthalpies,
                              surf2.partial_molar_enthalpies)
@@ -746,23 +746,23 @@ def test_yaml_surface_explicit(self):
         surf = ct.Interface("ptcombust.yaml", "Pt_surf", [gas])
         gas.TPY = 900, ct.one_atm, np.ones(gas.n_species)
         surf.coverages = np.ones(surf.n_species)
-        surf.write_yaml("ptcombust-generated.yaml")
+        surf.write_yaml(self.test_work_path / "ptcombust-generated.yaml")
         self.check_ptcombust(gas, surf)
 
     def test_yaml_surface_adjacent(self):
         surf = ct.Interface("ptcombust.yaml", "Pt_surf")
         gas = surf.adjacent["gas"]
         gas.TPY = 900, ct.one_atm, np.ones(gas.n_species)
         surf.coverages = np.ones(surf.n_species)
-        surf.write_yaml("ptcombust-generated.yaml")
+        surf.write_yaml(self.test_work_path / "ptcombust-generated.yaml")
         self.check_ptcombust(gas, surf)
 
     def test_yaml_eos(self):
         ice = ct.Solution('water.yaml', 'ice')
         ice.TP = 270, 2 * ct.one_atm
-        ice.write_yaml('ice-generated.yaml', units={'length': 'mm', 'mass': 'g'})
+        ice.write_yaml(self.test_work_path / "ice-generated.yaml", units={'length': 'mm', 'mass': 'g'})
 
-        ice2 = ct.Solution('ice-generated.yaml')
+        ice2 = ct.Solution(self.test_work_path / "ice-generated.yaml")
         self.assertNear(ice.density, ice2.density)
         self.assertNear(ice.entropy_mole, ice2.entropy_mole)
 
@@ -779,7 +779,7 @@ def test_yaml_inconsistent_species(self):
                                  h2.thermo.reference_pressure, nasa_coeffs)
         gas2.modify_species(gas2.species_index('H2'), h2)
         with self.assertRaisesRegex(ct.CanteraError, "different definitions"):
-            gas.write_yaml('h2o2-error.yaml', phases=gas2)
+            gas.write_yaml(self.test_work_path / "h2o2-error.yaml", phases=gas2)
 
     def test_yaml_user_data(self):
         gas = ct.Solution("h2o2.yaml")
@@ -791,8 +791,8 @@ def test_yaml_user_data(self):
         S.thermo.update_user_data({"something": (False, True)})
         gas.reaction(5).update_user_data({"baked-beans": True})
 
-        gas.write_yaml("h2o2-generated-user-data.yaml")
-        gas2 = ct.Solution("h2o2-generated-user-data.yaml")
+        gas.write_yaml(self.test_work_path / "h2o2-generated-user-data.yaml")
+        gas2 = ct.Solution(self.test_work_path / "h2o2-generated-user-data.yaml")
         data2 = gas2.species(2).input_data
 
         self.assertEqual(gas2.input_data["spam"], extra["spam"])

interfaces/python_sdist/MANIFEST.in

@@ -1,10 +1,18 @@
+# The files listed here will be included in the sdist. They will also be included in
+# the wheel file if they are also listed in setup.cfg:options.package_data and not
+# listed in setup.cfg:options.exclude_package_data.
+
 include cantera/_cantera.cpp
 include cantera/_cantera.h
 recursive-include cantera *.pyx
 include sundials_config.h.in
+include config.h.in
 graft include
+
+# The C/C++ files in these folders are included automatically because they're in
+# the source of the extension.
 recursive-include ext *.h
 recursive-include src *.h
-recursive-include ext/libexecstream/posix *.cpp
-recursive-include ext/libexecstream/win *.cpp
+
+recursive-include ext/libexecstream *.cpp
 exclude include/cantera/ext/sundials/sundials_config.h

interfaces/python_sdist/SConscript

@@ -9,7 +9,8 @@ from build.env import IsolatedEnvBuilder
 
 from buildutils import logger
 
-Import("env")
+Import("env", "configh")
+configh = configh.copy()
 
 localenv = env.Clone()
 
@@ -89,18 +90,79 @@ def replace_git_hash(target, source, env):
 sdist(localenv.RecursiveInstall(
     "src",
     "#src",
-    exclude=["fortran", "matlab", r"global\.cpp", "SCons.*"],
+    exclude=["fortran", "matlab", "clib", r"global\.cpp", "SCons.*"],
 ))
 
 sdist(localenv.Command("src/base/global.cpp", "#src/base/global.cpp",
                        replace_git_hash))
 
-include_target = sdist(localenv.Command("include", "#include",
-                                        Copy("$TARGET", "$SOURCE")))
-localenv.Depends(include_target, env["config_h_target"])
-localenv.Depends(include_target, env["ext_include_copies_target"])
+# This is the only bit of the clib that we need for the Python interface
+clib_defs_target = sdist(localenv.Command(
+    "include/cantera/clib/clib_defs.h",
+    "#include/cantera/clib/clib_defs.h",
+    Copy("$TARGET", "$SOURCE")
+))
+
+# The clib is handled by clib_defs_target. The include/cantera/ext folder is handled by
+# ext_include_target. config.h needs to be filled on the user's machine by setup.py and
+# the template file is handled by config_h_in_target.
+include_target = sdist(localenv.RecursiveInstall(
+    "include",
+    "#include",
+    exclude=[
+        "clib$",
+        "ext$",
+        r"config\.h\.in",
+        r"config\.h",
+    ],
+))
+localenv.Depends(clib_defs_target, include_target)
+
+ext_include_target = sdist(localenv.Command(
+    "include/cantera/ext",
+    "#include/cantera/ext",
+    Copy("$TARGET", "$SOURCE"),
+))
+localenv.Depends(ext_include_target, env["ext_include_copies_target"])
+localenv.Depends(ext_include_target, include_target)
+
+ext_target = sdist(localenv.Command("ext", "#ext", copy_ext_src))
+
+# Disable Lapack for the sdist
+configh["CT_USE_LAPACK"] = None
+configh["CT_SUNDIALS_USE_LAPACK"] = None
+
+# These keys are determined by the build machine
+for key in ("SOLARIS", "DARWIN", "CT_USE_DEMANGLE"):
+    del configh[key]
 
-sdist(localenv.Command("ext", "#ext", copy_ext_src))
+
+class DefineDict:
+    def __init__(self, data):
+        self.data = data
+
+    def __getitem__(self, key):
+        if key not in self.data:
+            return f"{{{key!s}!s}}"
+        elif self.data[key] is None:
+            return f"/* #undef {key!s} */"
+        else:
+            return f"#define {key!s} {self.data[key]!s}"
+
+
+def config_builder(target, source, env):
+    configh = env["configh"]
+    config_h = Path(str(target[0]))
+    config_h_in = Path(str(source[0])).read_text()
+    config_h.write_text(config_h_in.format_map(configh))
+
+
+localenv["configh"] = DefineDict(configh)
+config_h_in_target = sdist(localenv.Command(
+    "config.h.in",
+    "#include/cantera/base/config.h.in",
+    config_builder,
+))
 
 # Use RecursiveInstall to make sure that files are not overwritten during the copy.
 # A normal Copy Action would fail because of the existing directories.

interfaces/python_sdist/setup.cfg.in

@@ -50,12 +50,19 @@ packages =
     cantera.test.data
     cantera.examples
 
+# These options include data in the sdist and wheel if the files are also listed in
+# MANIFEST.in. Note that only files that are inside the "cantera" packages will be
+# included in the wheel.
 [options.package_data]
 cantera.data = *.*, */*.*
 cantera.test.data = *.*, */*.*
 cantera.examples = */*.*
 cantera = *.pxd
 
+# These options exclude data from the wheel file but not from the sdist
+[options.exclude_package_data]
+cantera = *.cpp, *.h, *.pyx
+
 [options.extras_require]
 hdf5 = h5py
 pandas = pandas