HELM install may be broken for next version release.

[AndrewJGaut]

Steps to reproduce:
python3.8 -m venv test-venv
source test-venv/bin/activate
pip install git+https://github.com/stanford-crfm/helm.git@main
helm-run --suite v1 --max-eval-instances 10 --run-specs "mmlu:subject=philosophy,model=huggingface/gpt2"

Error:

Traceback (most recent call last):
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/bin/helm-run", line 6, in <module>
    from helm.benchmark.run import main
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/helm/benchmark/__init__.py", line 74, in <module>
    from .metrics import summarization_metrics  # noqa
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/helm/benchmark/metrics/summarization_metrics.py", line 7, in <module>
    import spacy
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/spacy/__init__.py", line 14, in <module>
    from . import pipeline  # noqa: F401
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/spacy/pipeline/__init__.py", line 1, in <module>
    from .attributeruler import AttributeRuler
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/spacy/pipeline/attributeruler.py", line 6, in <module>
    from .pipe import Pipe
  File "spacy/pipeline/pipe.pyx", line 8, in init spacy.pipeline.pipe
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/spacy/training/__init__.py", line 11, in <module>
    from .callbacks import create_copy_from_base_model  # noqa: F401
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/spacy/training/callbacks.py", line 3, in <module>
    from ..language import Language
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/spacy/language.py", line 25, in <module>
    from .training.initialize import init_vocab, init_tok2vec
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/spacy/training/initialize.py", line 14, in <module>
    from .pretrain import get_tok2vec_ref
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/spacy/training/pretrain.py", line 16, in <module>
    from ..schemas import ConfigSchemaPretrain
  File "/Users/agaut/Documents/stanford_grad_school/second-year/codalabs/helm-codalab/venv4/lib/python3.8/site-packages/spacy/schemas.py", line 216, in <module>
    class TokenPattern(BaseModel):
  File "pydantic/main.py", line 299, in pydantic.main.ModelMetaclass.__new__
  File "pydantic/fields.py", line 411, in pydantic.fields.ModelField.infer
  File "pydantic/fields.py", line 342, in pydantic.fields.ModelField.__init__
  File "pydantic/fields.py", line 451, in pydantic.fields.ModelField.prepare
  File "pydantic/fields.py", line 545, in pydantic.fields.ModelField._type_analysis
  File "pydantic/fields.py", line 550, in pydantic.fields.ModelField._type_analysis
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/typing.py", line 771, in __subclasscheck__
    return issubclass(cls, self.__origin__)
TypeError: issubclass() arg 1 must be a class

The issue is resolved if you pip install -r https://raw.githubusercontent.com/stanford-crfm/helm/main/requirements-freeze.txt, so it appears to be an issue with the versions pip is using for downloaded dependencies for specified packages in requirements.txt

[AndrewJGaut]

Note: here is the output when I run pip freeze from local environment after running pip install git+https://github.com/stanford-crfm/helm.git@main

2captcha-python==1.2.0
absl-py==1.4.0
aiodns==3.0.0
aiohttp==3.8.4
aiohttp-retry==2.8.3
aiosignal==1.3.1
aleph-alpha-client==2.14.0
anthropic==0.2.9
anyio==3.6.2
async-generator==1.10
async-timeout==4.0.2
attrs==23.1.0
beautifulsoup4==4.12.2
bert-score==0.3.13
blanc==0.3.0
blis==0.7.9
boto3==1.26.142
botocore==1.29.142
bottle==0.12.25
cachetools==5.3.0
catalogue==2.0.8
cattrs==22.2.0
certifi==2023.5.7
cffi==1.15.1
charset-normalizer==3.1.0
click==8.1.3
colorama==0.4.6
colorcet==3.0.1
contourpy==1.0.7
crfm-helm==0.2.2
cycler==0.11.0
cymem==2.0.7
Cython==0.29.35
dacite==1.6.0
datasets==2.5.2
dill==0.3.5.1
emoji==2.4.0
exceptiongroup==1.1.1
filelock==3.12.0
fonttools==4.39.4
frozenlist==1.3.3
fsspec==2023.5.0
gdown==4.4.0
gin-config==0.5.0
google-api-core==2.11.0
google-api-python-client==2.64.0
google-auth==2.19.0
google-auth-httplib2==0.1.0
googleapis-common-protos==1.59.0
gunicorn==20.1.0
h11==0.14.0
httpcore==0.17.2
httplib2==0.22.0
httpx==0.24.1
huggingface-hub==0.14.1
icetk==0.0.7
idna==3.4
importlib-metadata==6.6.0
importlib-resources==5.10.4
Jinja2==3.1.2
jmespath==1.0.1
joblib==1.2.0
jsonlines==3.1.0
kiwisolver==1.4.4
langcodes==3.3.0
llvmlite==0.39.1
lxml==4.9.2
Mako==1.2.4
MarkupSafe==2.1.2
matplotlib==3.6.3
moverscore==1.0.3
mpmath==1.3.0
multidict==6.0.4
multiprocess==0.70.14
murmurhash==1.0.9
networkx==3.1
nltk==3.8.1
numba==0.56.4
numpy==1.23.5
openai==0.27.7
outcome==1.2.0
packaging==23.1
pandas==2.0.1
param==1.13.0
parameterized==0.9.0
pathy==0.10.1
Pillow==9.5.0
portalocker==2.7.0
preshed==3.0.8
protobuf==3.20.3
psutil==5.9.5
pyarrow==12.0.0
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycares==4.3.0
pycparser==2.21
pyct==0.5.0
pydantic==1.8.2
pyemd==0.5.1
pyext==0.7
pyhocon==0.3.60
pymongo==4.2.0
pyparsing==3.0.9
python-dateutil==2.8.2
pytorch-pretrained-bert==0.6.2
pytrec-eval==0.5
pytz==2023.3
PyYAML==6.0
regex==2023.5.5
requests==2.31.0
responses==0.18.0
retrying==1.3.4
revChatGPT==0.1.1
rouge-score==0.1.2
rsa==4.9
s3transfer==0.6.1
sacrebleu==2.2.1
sacremoses==0.0.53
scikit-learn==1.1.3
scipy==1.9.3
seaborn==0.11.2
selenium==4.9.1
sentencepiece==0.1.99
six==1.16.0
smart-open==6.3.0
sniffio==1.3.0
sortedcontainers==2.4.0
soupsieve==2.4.1
spacy==3.2.6
spacy-legacy==3.0.12
spacy-loggers==1.0.4
sqlitedict==1.7.0
srsly==2.4.6
stanza==1.5.0
summ-eval==0.892
surge-api==1.1.0
sympy==1.11.1
tabulate==0.9.0
thinc==8.0.17
threadpoolctl==3.1.0
tiktoken==0.3.3
tls-client==0.2.1
tokenizers==0.13.3
torch==1.12.1
torchvision==0.13.1
tqdm==4.64.1
transformers==4.28.1
trio==0.22.0
trio-websocket==0.10.2
typer==0.4.2
typing==3.7.4.3
typing-extensions==4.6.2
tzdata==2023.3
uncertainty-calibration==0.1.4
undetected-chromedriver==3.4.7
uritemplate==4.1.1
urllib3==2.0.2
wasabi==0.10.1
websocket-client==1.3.3
websockets==11.0.3
wsproto==1.2.0
xlrd==2.0.1
xxhash==3.2.0
yarl==1.9.2
zipp==3.15.0
zstandard==0.18.0