Merge pull request #108 from iamdefinitelyahuman/refactor-solc-bin

Install from solc-bin and separate building from source logic

docs/version-management.rst

@@ -83,39 +83,39 @@ Setting the Active Version
         Version('0.5.17')
 
 
+Importing Already-Installed Versions
+====================================
+
+.. py:function:: solcx.import_installed_solc(solcx_binary_path=None)
+
+    Search for and copy installed ``solc`` versions into the local installation folder.
+
+    This function is especially useful on OSX, to access Solidity versions that you have installed from homebrew and where a precompiled binary is not available.
+
+    .. code-block:: python
+
+        >>> solcx.import_installed_solc()
+        [Version('0.7.0'), Version('0.6.12')]
+
+
 Installing Solidity
 ===================
 
+py-solc-x downloads and installs precompiled binaries from `solc-bin.ethereum.org <solc-bin.ethereum.org>`_. Different binaries are available depending on your operating system.
+
 Getting Installable Versions
 ----------------------------
 
-.. py:function:: solcx.get_available_solc_versions(headers=None, compilable=False)
+.. py:function:: solcx.get_installable_solc_versions()
 
     Return a list of all ``solc`` versions that can be installed by py-solc-x.
 
-        ``headers`` Dict
-            Headers to include in the request to Github.
-        ``compilable`` bool
-            If ``True``, return a list of versions that can be compiled from source.
 
     .. code-block:: python
 
-        >>> solcx.get_available_solc_versions()
+        >>> solcx.get_installable_solc_versions()
         [Version('0.7.0'), Version('0.6.12'), Version('0.6.11'), Version('0.6.10'), Version('0.6.9'), Version('0.6.8'), Version('0.6.7'), Version('0.6.6'), Version('0.6.5'), Version('0.6.4'), Version('0.6.3'), Version('0.6.2'), Version('0.6.1'), Version('0.6.0'), Version('0.5.17'), Version('0.5.16'), Version('0.5.15'), Version('0.5.14'), Version('0.5.13'), Version('0.5.12'), Version('0.5.11'), Version('0.5.10'), Version('0.5.9'), Version('0.5.8'), Version('0.5.7'), Version('0.5.6'), Version('0.5.5'), Version('0.5.4'), Version('0.5.3'), Version('0.5.2'), Version('0.5.1'), Version('0.5.0'), Version('0.4.26'), Version('0.4.25'), Version('0.4.24'), Version('0.4.23'), Version('0.4.22'), Version('0.4.21'), Version('0.4.20'), Version('0.4.19'), Version('0.4.18'), Version('0.4.17'), Version('0.4.16'), Version('0.4.15'), Version('0.4.14'), Version('0.4.13'), Version('0.4.12'), Version('0.4.11')]
 
-Importing Already-Installed Versions
-------------------------------------
-
-.. py:function:: solcx.import_installed_solc(solcx_binary_path=None)
-
-    Search for and copy installed ``solc`` versions into the local installation folder.
-
-    .. code-block:: python
-
-        >>> solcx.import_installed_solc()
-        [Version('0.7.0'), Version('0.6.12')]
-
-
 Installing Precompiled Binaries
 -------------------------------
 
@@ -131,19 +131,41 @@ Installing Precompiled Binaries
         ``solcx_binary_path`` Path | str
             User-defined path, used to override the default installation directory.
 
-Buildling from Source
----------------------
+Building from Source
+====================
+
+When a precompiled version of Solidity isn't available for your operating system, you may still install it by building from the source code. Source code is downloaded from `Github <https://github.com/ethereum/solidity/releases>`_.
+
+.. note::
+
+    If you wish to compile from source you must first install the required `solc dependencies <https://solidity.readthedocs.io/en/latest/installing-solidity.html#building-from-source>`_.
+
+
+Getting Compilable Versions
+---------------------------
+
+.. py:function:: solcx.get_compilable_solc_versions(headers=None)
+
+    Return a list of all ``solc`` versions that can be installed by py-solc-x.
+
+        ``headers`` Dict
+            Headers to include in the request to Github.
+
+    .. code-block:: python
+
+        >>> solcx.get_compilable_solc_versions()
+        [Version('0.7.0'), Version('0.6.12'), Version('0.6.11'), Version('0.6.10'), Version('0.6.9'), Version('0.6.8'), Version('0.6.7'), Version('0.6.6'), Version('0.6.5'), Version('0.6.4'), Version('0.6.3'), Version('0.6.2'), Version('0.6.1'), Version('0.6.0'), Version('0.5.17'), Version('0.5.16'), Version('0.5.15'), Version('0.5.14'), Version('0.5.13'), Version('0.5.12'), Version('0.5.11'), Version('0.5.10'), Version('0.5.9'), Version('0.5.8'), Version('0.5.7'), Version('0.5.6'), Version('0.5.5'), Version('0.5.4'), Version('0.5.3'), Version('0.5.2'), Version('0.5.1'), Version('0.5.0'), Version('0.4.26'), Version('0.4.25'), Version('0.4.24'), Version('0.4.23'), Version('0.4.22'), Version('0.4.21'), Version('0.4.20'), Version('0.4.19'), Version('0.4.18'), Version('0.4.17'), Version('0.4.16'), Version('0.4.15'), Version('0.4.14'), Version('0.4.13'), Version('0.4.12'), Version('0.4.11')]
+
+
+Compiling Solidity from Source
+------------------------------
 
 .. py:function:: solcx.compile_solc(version, show_progress=False, solcx_binary_path=None)
 
     Install a version of ``solc`` by downloading and compiling source code.
 
     This function is only available when using Linux or OSX.
 
-    .. note::
-
-        If you wish to compile from source you must first install the required `solc dependencies <https://solidity.readthedocs.io/en/latest/installing-solidity.html#building-from-source>`_.
-
     **Arguments:**
 
         ``version`` str | Version

solcx/__init__.py

@@ -1,6 +1,7 @@
 from solcx.install import (
     compile_solc,
-    get_available_solc_versions,
+    get_compilable_solc_versions,
+    get_installable_solc_versions,
     get_installed_solc_versions,
     get_solcx_install_folder,
     import_installed_solc,

solcx/install.py

@@ -38,16 +38,11 @@
     tqdm = None
 
 
-DOWNLOAD_BASE = "https://github.com/ethereum/solidity/releases/download/v{}/{}"
-ALL_RELEASES = "https://api.github.com/repos/ethereum/solidity/releases?per_page=100"
-
-MINIMAL_SOLC_VERSION = "v0.4.11"
-VERSION_REGEX = {
-    "darwin": "solc-macos",
-    "source": "solidity_[0-9].[0-9].[0-9]{1,}.tar.gz",
-    "linux": "solc-static-linux",
-    "win32": "solidity-windows.zip",
-}
+BINARY_DOWNLOAD_BASE = "https://solc-bin.ethereum.org/{}-amd64/{}"
+SOURCE_DOWNLOAD_BASE = "https://github.com/ethereum/solidity/releases/download/v{}/{}"
+GITHUB_RELEASES = "https://api.github.com/repos/ethereum/solidity/releases?per_page=100"
+
+MINIMAL_SOLC_VERSION = Version("0.4.11")
 LOGGER = logging.getLogger("solcx")
 
 SOLCX_BINARY_PATH_VARIABLE = "SOLCX_BINARY_PATH"
@@ -58,8 +53,10 @@
 def _get_os_name() -> str:
     if sys.platform.startswith("linux"):
         return "linux"
-    if sys.platform in ("darwin", "win32"):
-        return sys.platform
+    if sys.platform == "darwin":
+        return "macosx"
+    if sys.platform == "win32":
+        return "windows"
     raise OSError(f"Unsupported OS: '{sys.platform}' - py-solc-x supports Linux, OSX and Windows")
 
 
@@ -75,7 +72,7 @@ def _convert_and_validate_version(version: Union[str, Version]) -> Version:
 
 def _unlink_solc(solc_path: Path) -> None:
     solc_path.unlink()
-    if _get_os_name() == "win32":
+    if _get_os_name() == "windows":
         shutil.rmtree(solc_path.parent)
 
 
@@ -107,7 +104,7 @@ def get_solcx_install_folder(solcx_binary_path: Union[Path, str] = None) -> Path
 
 def _get_which_solc() -> Path:
     # get the path for the currently installed `solc` version, if any
-    if _get_os_name() == "win32":
+    if _get_os_name() == "windows":
         response = subprocess.check_output(["where.exe", "solc"], encoding="utf8").strip()
     else:
         response = subprocess.check_output(["which", "solc"], encoding="utf8").strip()
@@ -135,7 +132,7 @@ def import_installed_solc(solcx_binary_path: Union[Path, str] = None) -> List[Ve
         path_list = []
 
     # on OSX, also copy all versions of solc from cellar
-    if _get_os_name() == "darwin":
+    if _get_os_name() == "macosx":
         path_list.extend(Path("/usr/local/Cellar").glob("solidity*/**/solc"))
 
     imported_versions = []
@@ -147,7 +144,7 @@ def import_installed_solc(solcx_binary_path: Union[Path, str] = None) -> List[Ve
             continue
 
         copy_path = get_solcx_install_folder(solcx_binary_path).joinpath(f"solc-v{version}")
-        if _get_os_name() == "win32":
+        if _get_os_name() == "windows":
             copy_path.mkdir()
             copy_path = copy_path.joinpath("solc.exe")
 
@@ -191,7 +188,7 @@ def get_executable(
 
     version = _convert_and_validate_version(version)
     solc_bin = get_solcx_install_folder(solcx_binary_path).joinpath(f"solc-v{version}")
-    if sys.platform == "win32":
+    if _get_os_name() == "windows":
         solc_bin = solc_bin.joinpath("solc.exe")
     if not solc_bin.exists():
         raise SolcNotInstalled(
@@ -304,7 +301,7 @@ def install_solc_pragma(
     Version
         Installed `solc` version.
     """
-    version = _select_pragma_version(pragma_string, get_available_solc_versions())
+    version = _select_pragma_version(pragma_string, get_installable_solc_versions())
     if not version:
         raise UnsupportedVersionError("Compatible solc version does not exist")
     if install:
@@ -313,33 +310,50 @@ def install_solc_pragma(
     return version
 
 
-def get_available_solc_versions(
-    headers: Optional[Dict] = None, compilable: bool = False
-) -> List[Version]:
+def get_installable_solc_versions() -> List[Version]:
     """
     Return a list of all `solc` versions that can be installed by py-solc-x.
 
+    Returns
+    -------
+    List
+        List of Versions objects of installable `solc` versions.
+    """
+    data = requests.get(BINARY_DOWNLOAD_BASE.format(_get_os_name(), "list.json"))
+    if data.status_code != 200:
+        raise ConnectionError(
+            f"Status {data.status_code} when getting solc versions from solc-bin.ethereum.org"
+        )
+    version_list = sorted((Version(i) for i in data.json()["releases"]), reverse=True)
+    version_list = [i for i in version_list if i >= MINIMAL_SOLC_VERSION]
+    return version_list
+
+
+def get_compilable_solc_versions(headers: Optional[Dict] = None) -> List[Version]:
+    """
+    Return a list of all `solc` versions that can be compiled from source by py-solc-x.
+
     Arguments
     ---------
     headers : Dict, optional
         Headers to include in the request to Github.
-    compilable : bool, optional
-        If True, return a list of versions that can be compiled from source.
 
     Returns
     -------
     List
         List of Versions objects of installable `solc` versions.
     """
+    if _get_os_name() == "windows":
+        raise OSError("Compiling from source is not supported on Windows systems")
+
     version_list = []
-    regex_key = "source" if compilable else _get_os_name()
-    pattern = VERSION_REGEX[regex_key]
+    pattern = "solidity_[0-9].[0-9].[0-9]{1,}.tar.gz"
 
     if headers is None and os.getenv("GITHUB_TOKEN") is not None:
         auth = b64encode(os.environ["GITHUB_TOKEN"].encode()).decode()
         headers = {"Authorization": f"Basic {auth}"}
 
-    data = requests.get(ALL_RELEASES, headers=headers)
+    data = requests.get(GITHUB_RELEASES, headers=headers)
     if data.status_code != 200:
         msg = (
             f"Status {data.status_code} when getting solc versions from Github:"
@@ -354,11 +368,11 @@ def get_available_solc_versions(
         raise ConnectionError(msg)
 
     for release in data.json():
+        version = Version.coerce(release["tag_name"].lstrip("v"))
         asset = next((i for i in release["assets"] if re.match(pattern, i["name"])), False)
         if asset:
-            version = Version.coerce(release["tag_name"].lstrip("v"))
             version_list.append(version)
-        if release["tag_name"] == MINIMAL_SOLC_VERSION:
+        if version == MINIMAL_SOLC_VERSION:
             break
     return sorted(version_list, reverse=True)
 
@@ -406,7 +420,7 @@ def install_solc(
     """
 
     if version == "latest":
-        version = get_available_solc_versions()[0]
+        version = get_installable_solc_versions()[0]
     else:
         version = _convert_and_validate_version(version)
 
@@ -419,12 +433,22 @@ def install_solc(
             LOGGER.info(f"solc {version} already installed at: {path}")
             return version
 
-        elif os_name == "linux":
-            _install_solc_unix(version, "solc-static-linux", show_progress, solcx_binary_path)
-        elif os_name == "darwin":
-            _install_solc_unix(version, "solc-macos", show_progress, solcx_binary_path)
-        elif os_name == "win32":
-            _install_solc_windows(version, show_progress, solcx_binary_path)
+        data = requests.get(BINARY_DOWNLOAD_BASE.format(_get_os_name(), "list.json"))
+        if data.status_code != 200:
+            raise ConnectionError(
+                f"Status {data.status_code} when getting solc versions from solc-bin.ethereum.org"
+            )
+        try:
+            filename = data.json()["releases"][str(version)]
+        except KeyError:
+            raise SolcInstallationError(f"Solc binary for v{version} is not available for this OS")
+
+        if os_name == "linux":
+            _install_solc_unix(version, filename, show_progress, solcx_binary_path)
+        elif os_name == "macosx":
+            _install_solc_unix(version, filename, show_progress, solcx_binary_path)
+        elif os_name == "windows":
+            _install_solc_windows(version, filename, show_progress, solcx_binary_path)
 
         try:
             _validate_installation(version, solcx_binary_path)
@@ -459,11 +483,11 @@ def compile_solc(
     Version
         installed solc version
     """
-    if _get_os_name() == "win32":
+    if _get_os_name() == "windows":
         raise OSError("Compiling from source is not supported on Windows systems")
 
     if version == "latest":
-        version = get_available_solc_versions(compilable=True)[0]
+        version = get_compilable_solc_versions()[0]
     else:
         version = _convert_and_validate_version(version)
 
@@ -476,7 +500,7 @@ def compile_solc(
             return version
 
         temp_path = _get_temp_folder()
-        download = DOWNLOAD_BASE.format(version, f"solidity_{version}.tar.gz")
+        download = SOURCE_DOWNLOAD_BASE.format(version, f"solidity_{version}.tar.gz")
         install_path = get_solcx_install_folder(solcx_binary_path).joinpath(f"solc-v{version}")
 
         content = _download_solc(download, show_progress)
@@ -506,7 +530,7 @@ def compile_solc(
                 " while attempting to build solc from the source.\n"
                 "This is likely due to a missing or incorrect version of a build dependency."
             )
-            if _get_os_name() == "darwin":
+            if _get_os_name() == "macosx":
                 err_msg = (
                     f"{err_msg}\n\nFor suggested installation options: "
                     "https://github.com/iamdefinitelyahuman/py-solc-x/wiki/Installing-Solidity-on-OSX"  # noqa: E501
@@ -538,6 +562,7 @@ def _get_temp_folder() -> Path:
 
 
 def _download_solc(url: str, show_progress: bool) -> bytes:
+    LOGGER.info(f"Downloading from {url}")
     response = requests.get(url, stream=show_progress)
     if response.status_code == 404:
         raise DownloadError(
@@ -566,10 +591,9 @@ def _download_solc(url: str, show_progress: bool) -> bytes:
 def _install_solc_unix(
     version: Version, filename: str, show_progress: bool, solcx_binary_path: Union[Path, str, None]
 ) -> None:
-    download = DOWNLOAD_BASE.format(version, filename)
+    download = BINARY_DOWNLOAD_BASE.format(_get_os_name(), filename)
     install_path = get_solcx_install_folder(solcx_binary_path).joinpath(f"solc-v{version}")
 
-    LOGGER.info(f"Downloading solc {version} from {download}")
     content = _download_solc(download, show_progress)
     with open(install_path, "wb") as fp:
         fp.write(content)
@@ -578,9 +602,9 @@ def _install_solc_unix(
 
 
 def _install_solc_windows(
-    version: Version, show_progress: bool, solcx_binary_path: Union[Path, str, None]
+    version: Version, filename: str, show_progress: bool, solcx_binary_path: Union[Path, str, None]
 ) -> None:
-    download = DOWNLOAD_BASE.format(version, "solidity-windows.zip")
+    download = BINARY_DOWNLOAD_BASE.format(_get_os_name(), filename)
     install_path = get_solcx_install_folder(solcx_binary_path).joinpath(f"solc-v{version}")
 
     temp_path = _get_temp_folder()

solcx/wrapper.py

@@ -1,3 +1,4 @@
+import re
 import subprocess
 from pathlib import Path
 from typing import Any, Dict, List, Tuple, Union
@@ -10,9 +11,10 @@
 
 def _get_solc_version(solc_binary: Union[Path, str]) -> Version:
     # private wrapper function to get `solc` version
-    stdout_data = subprocess.check_output([solc_binary, "--version"], encoding="utf8").strip()
-    stdout_data = stdout_data[stdout_data.index("Version: ") + 9 : stdout_data.index("+")]
-    return Version.coerce(stdout_data)
+    stdout_data = subprocess.check_output([solc_binary, "--version"], encoding="utf8")
+    version_str = re.findall(r"(?<=Version: ).*?(?=\+)", stdout_data)[0]
+    version_str = re.sub(r"\.0(?=[1-9])", ".", version_str)
+    return Version.coerce(version_str)
 
 
 def _to_string(key: str, value: Any) -> str: