forked from aws/fmeval
-
Notifications
You must be signed in to change notification settings - Fork 0
/
devtool
executable file
·219 lines (192 loc) · 6.42 KB
/
devtool
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
#!/usr/bin/env bash
set -euo pipefail
set -x
lint() {
echo ""
echo "Lint checks"
echo -e "===========\n"
if [ -d .git ]
then
echo "1. pre-commit hooks"
echo "==================="
pre-commit run -v -a
echo ""
else
echo "1. pre-commit hooks (Skipped)"
echo "==================="
echo ""
fi
echo "2. Flake8"
echo "========="
flake8 . --config=setup.cfg
echo ""
echo "3. Mypy"
echo "======="
echo y | mypy --install-types --junit-xml tmp/typecheck.xml --html-report tmp --config-file setup.cfg src/ || \
mypy --junit-xml tmp/typecheck.xml --html-report tmp --config-file setup.cfg --show-traceback src/
echo ""
echo "4. Poetry lock check"
echo "===================="
echo "Note: This only syncs poetry.lock with pyproject.toml. If you wish to update all downstream dependencies, run 'poetry update'"
install_poetry_version 1.2.0
poetry lock --check || (poetry lock --no-update && poetry lock --check)
echo ""
echo "Lint: SUCCESS"
}
running_in_codebuild() {
if [[ -n ${CODEBUILD_BUILD_ID:-} ]]; then
return 0
else
return 1
fi
}
env_setup() {
echo "OS Type: ${OSTYPE}"
echo -n "Python version: "
python --version
echo ""
if [[ $OSTYPE == 'darwin'* ]]; then
echo "Detected darwin OS Type, setting OBJC_DISABLE_INITIALIZE_FORK_SAFETY in env"
export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
fi
}
is_cn_region() {
if [ "$(echo "${AWS_DEFAULT_REGION:-us-west-2}" |cut -c -3)" = "cn-" ]
then
return 0
else
return 1
fi
}
# Pass in poetry version as argument
install_poetry_version() {
# Poetry will be installed to this path in codebuild
# Explicitly export the path so all shells have the updated path
if running_in_codebuild && [[ -z $(command -v poetry) ]]
then
echo "Exporting path to poetry for codebuild"
export PATH="/root/.local/bin:$PATH"
fi
# Installs poetry safely and system-wide using the recommended installer
# Also update poetry to the latest --preview to support features such as dependency groups and plugins which
# are not yet officially released.x
if [[ -z $(command -v poetry) ]]
then
# If running in codebuild, we can just install. No need for the fancy parsing
if running_in_codebuild
then
if is_cn_region
then
# Use pipx to install poetry as the installer times out in CN region
python3 -m pip install pipx
pipx install poetry==$1
else
curl -sSL https://install.python-poetry.org | python3 - --version $1
fi
else
# Note: If the script fails at this time, it may have timed out. Separate the curl and python3 commands and
# run them directly instead of storing into a variable to check logs.
POETRY_SCRIPT=`curl -sSL https://install.python-poetry.org`
echo "Installed poetry installer, running the script"
POETRY_INSTALL_OUTPUT=`python3 -c "$POETRY_SCRIPT" --version $1`
LIGHT_PURPLE='\033[1;35m'
ORANGE='\033[0;33m'
LIGHT_RED='\033[1;31m'
NC='\033[0m'
# Automatically append to PATH using output of poetry installation
# Note: If this stops working, poetry probably changed the output of their installation command
echo -e "${LIGHT_PURPLE}$POETRY_INSTALL_OUTPUT${NC}"
if [[ $POETRY_INSTALL_OUTPUT == *"Add \`export PATH="*"to your shell configuration file."* ]]
then
EXPORT_POETRY=$(echo tr -d '\n' $POETRY_INSTALL_OUTPUT | sed -z 's/.*Add `export PATH=\"\(.*\):$PATH"` to your shell configuration file.*/\1/')
if [[ $(echo $PATH) != *"${EXPORT_POETRY}"* ]]
then
echo -e "${LIGHT_RED}PATH is missing poetry root at ${ORANGE}${EXPORT_POETRY}${NC}"
echo -e "${LIGHT_RED}Please add the above export PATH to your shell configuration file.${NC}"
exit 1
fi
elif [[ $POETRY_INSTALL_OUTPUT == *"The latest version"*"is already installed"* ]]
then
echo -e "${LIGHT_RED}Poetry is already installed, but not on the PATH. Did you forget to add it?${NC}"
exit 1
fi
fi
else
echo "poetry is already installed: "
poetry --version
fi
poetry config virtualenvs.prefer-active-python true
# Don't create virtual environments by default. This is needed because our deep canaries don't use virtual envs
poetry config virtualenvs.create false
# too much parallelism when installing packages can run into throttling and timeouts, especially in CN regions
poetry config installer.max-workers 2
}
unit_test_with_coverage() {
echo "Test with coverage"
echo "=================="
coverage run -m pytest -x --pspec test/unit
coverage report -m --fail-under=100
}
run_integ_tests() {
pytest test/integration/
}
create_commit_hash_file(){
if [[ -z "${CODEBUILD_RESOLVED_SOURCE_VERSION:-}" ]]; then
echo $(git rev-parse HEAD) > src/fmeval/COMMIT_HASH
else
echo "${CODEBUILD_RESOLVED_SOURCE_VERSION}" > src/fmeval/COMMIT_HASH
fi
}
build_package() {
create_commit_hash_file
poetry build
rm src/fmeval/COMMIT_HASH
}
install_poetry() {
install_poetry_version 1.2.0
}
install_deps() {
install_poetry
poetry install
}
install_deps_dev() {
install_poetry
poetry install --only dev
}
install_package() {
wheel_name=$(ls -t dist | head -n 1)
pip3 install --upgrade dist/$wheel_name --upgrade --upgrade-strategy only-if-needed --force-reinstall
}
all() {
env_setup
install_deps
lint
unit_test_with_coverage
build_package
echo "All build and tests passed. 😊"
}
##############################################################
# MAIN
#
# Run function passed as argument
set +x
if [ $# -gt 0 ]
then
SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
echo "CD -> $SCRIPTPATH"
cd $SCRIPTPATH
echo ==================
echo $@
echo ==================
$@
else
cat<<EOF
**Developer tool**
==================
$0: Execute a function by passing it as an argument to the script:
Possible commands:
==================
EOF
declare -F | cut -d' ' -f3
echo
fi