From 47e7b32f2e2043528b6f967fb0feb9aa1bfb260b Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Fri, 17 Dec 2021 17:20:15 +0100 Subject: [PATCH 1/3] Respect tmpdir --- frameworks/shared/caller.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frameworks/shared/caller.py b/frameworks/shared/caller.py index ee60b42d4..a860529d6 100644 --- a/frameworks/shared/caller.py +++ b/frameworks/shared/caller.py @@ -82,7 +82,8 @@ def run_in_venv(caller_file, script_file: str, *args, input_data: Union[dict, ns], dataset: Dataset, config: TaskConfig, options: Union[None, dict, ns] = None, process_results=None, - python_exec=None): + python_exec=None, + retain_tmp_env: bool = True): here = dir_of(caller_file) if python_exec is None: # use local virtual env by default python_exec = venv_python_exec(here) @@ -93,6 +94,12 @@ def run_in_venv(caller_file, script_file: str, *args, ser_config = options['serialization'] env = options['env'] or ns() + # To honour any arguments about tmpdir, we copy over + # the TMP, TEMP and TMPDIR environment variables + if retain_tmp_env: + for env_var in ["TMP", "TEMP", "TMPDIR"]: + env[env_var] = os.environ.get(env_var) + with TemporaryDirectory() as tmpdir: ds = _make_input_dataset(input_data, dataset, tmpdir, serialization=ser_config) From fa7f900f6d97bd857d97f4cfa2c8f47dc4767fdd Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Sat, 18 Dec 2021 13:10:11 +0100 Subject: [PATCH 2/3] Fixed submodule --- frameworks/autosklearn/setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/frameworks/autosklearn/setup.sh b/frameworks/autosklearn/setup.sh index f0099c3e3..63edcf04f 100755 --- a/frameworks/autosklearn/setup.sh +++ b/frameworks/autosklearn/setup.sh @@ -36,6 +36,7 @@ else git clone --recurse-submodules --shallow-submodules ${DEPTH} ${REPO} ${TARGET_DIR} cd ${TARGET_DIR} git checkout "${COMMIT}" + git submodule update --init --recursive cd ${HERE} PIP install -U -e ${TARGET_DIR} fi From c129d7a614bf4f23e932e8267fdc7e3c34c66eb0 Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Thu, 6 Jan 2022 22:46:37 +0100 Subject: [PATCH 3/3] feat: retain environment vars for framework venv --- frameworks/shared/caller.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/frameworks/shared/caller.py b/frameworks/shared/caller.py index a860529d6..2c82148e5 100644 --- a/frameworks/shared/caller.py +++ b/frameworks/shared/caller.py @@ -3,7 +3,7 @@ import os import re from tempfile import TemporaryDirectory, mktemp -from typing import Optional, Union +from typing import List, Optional, Union import numpy as np @@ -83,7 +83,7 @@ def run_in_venv(caller_file, script_file: str, *args, options: Union[None, dict, ns] = None, process_results=None, python_exec=None, - retain_tmp_env: bool = True): + retained_env_vars: Optional[List[str]] = ['TMP', 'TEMP', 'TMPDIR']): here = dir_of(caller_file) if python_exec is None: # use local virtual env by default python_exec = venv_python_exec(here) @@ -94,11 +94,12 @@ def run_in_venv(caller_file, script_file: str, *args, ser_config = options['serialization'] env = options['env'] or ns() - # To honour any arguments about tmpdir, we copy over - # the TMP, TEMP and TMPDIR environment variables - if retain_tmp_env: - for env_var in ["TMP", "TEMP", "TMPDIR"]: - env[env_var] = os.environ.get(env_var) + # Add any env variables specified if they are defined in the environment + if retained_env_vars: + for env_var in retained_env_vars: + env_val = os.environ.get(env_var) + if env_val is not None: + env[env_var] = env_val with TemporaryDirectory() as tmpdir: