Create Windows Installers (#312)

Create Windows Installers

appveyor.yml

@@ -6,86 +6,62 @@ environment:
   TWINE_REPOSITORY_URL:
     'https://upload.pypi.org/legacy/'
   matrix:
-    - PYTHON: "C:\\Python35"
-      PYTHON_VERSION: "3.5.x"
-      PYTHON_ARCH: "32"
-
     - PYTHON: "C:\\Python35-x64"
       PYTHON_VERSION: "3.5.x"
       PYTHON_ARCH: "64"
 
-    - PYTHON: "C:\\Python36"
+    - PYTHON: "C:\\Python36-x64"
       PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "64"
+
+    - PYTHON: "C:\\Python35"
+      PYTHON_VERSION: "3.5.x"
       PYTHON_ARCH: "32"
 
-    - PYTHON: "C:\\Python36-x64"
+    - PYTHON: "C:\\Python36"
       PYTHON_VERSION: "3.6.x"
-      PYTHON_ARCH: "64"
+      PYTHON_ARCH: "32"
+
+
+max_jobs: 4
 
 branches:
   only:
     - master
     - /^v\d+\.\d+(\.\d+)*(-\S*)?$/
 
-
 install:
-  # If there is a newer build queued for the same PR, cancel this one.
-  # The AppVeyor 'rollout builds' option is supposed to serve the same
-  # purpose but it is problematic because it tends to cancel builds pushed
-  # directly to master instead of just PR builds (or the converse).
-  # credits: JuliaLang developers.
   - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
         https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
         Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
           throw "There are newer queued builds for this pull request, failing early." }
-  - ECHO "Filesystem root:"
-  - ps: "ls \"C:/\""
-
-  - ECHO "Installed SDKs:"
-  - ps: "ls \"C:/Program Files/Microsoft SDKs/Windows\""
-
-  # Install Python (from the official .msi of http://python.org) and pip when
-  # not already installed.
+  # Install Python (from the official .msi of http://python.org) and pip when not already installed.
   - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
+  - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
+  - python --version
+  - python -c "import struct; print(struct.calcsize('P') * 8)"
+  - pip install --disable-pip-version-check --user --upgrade pip
 
-  # Prepend newly installed Python to the PATH of this build (this cannot be
-  # done from inside the powershell script as it would require to restart
-  # the parent CMD process).
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-
-  # Check that we have the expected version and architecture for Python
-  - "python --version"
-  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
-
-  # Upgrade to the latest version of pip to avoid it displaying warnings
-  # about it being out of date.
-  - "pip install --disable-pip-version-check --user --upgrade pip"
-
-  # Install the build dependencies of the project. If some dependencies contain
-  # compiled extensions and are not provided as pre-built wheel packages,
-  # pip will build them from source using the MSVC compiler matching the
-  # target Python version and architecture
   - pip install -r requirements.txt
-  - pip install pytest wheel twine
+  - pip install pytest
 
 build_script:
   # Build the compiled extension
   - python setup.py build_ext --inplace
 
 test_script:
-  # Run the project tests
   - python -m pytest -s
 
 after_test:
-  # If tests are successful, create binary packages for the project.
-  - python -c "import tempfile, os; open(os.path.join(tempfile.gettempdir(), 'urh_releasing'), 'w').close()"
-  - python setup.py bdist_wheel
-#  - python setup.py bdist_wininst
-  - ps: "ls dist"
+  - IF "%APPVEYOR_REPO_TAG%" == "true" (pip install wheel twine cx_freeze six appdirs packaging)
+  - IF "%APPVEYOR_REPO_TAG%" == "true" (python -c "import tempfile, os; open(os.path.join(tempfile.gettempdir(), 'urh_releasing'), 'w').close()")
+  - IF "%APPVEYOR_REPO_TAG%" == "true" (python setup.py bdist_wheel)
+  - IF "%APPVEYOR_REPO_TAG%" == "true" (IF "%PYTHON_VERSION%" == "3.5.x" (python build_cx.py))
+  - IF "%APPVEYOR_REPO_TAG%" == "true" (dir dist)
 
 artifacts:
 #  # Archive the generated packages in the ci.appveyor.com build report.
   - path: dist\*
 
 on_success:
-  -  cmd: IF "%APPVEYOR_REPO_TAG%" == "true" (twine upload --skip-existing dist/*)
+  -  IF "%APPVEYOR_REPO_TAG%" == "true" (twine upload --skip-existing dist/*)

build_cx.py

@@ -0,0 +1,78 @@
+import os
+import sys
+
+import cx_Freeze
+
+import src.urh.version as version
+
+def build_exe(build_cmd='build'):
+    # have to make sure args looks right
+    sys.argv = sys.argv[:1] + [build_cmd]
+
+    app_path = os.path.join(os.path.dirname(__file__), "src", "urh", "main.py")
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src"))
+
+    if sys.platform == 'win32':
+        include_files = [os.path.join("data", 'icons', 'appicon.ico')]
+        arch = "x64" if sys.maxsize > 2 ** 32 else "x86"
+        lib_path = os.path.join("src", "urh", "dev", "native", "lib", "win", arch)
+        for f in os.listdir(lib_path):
+            include_files.append(os.path.join(lib_path, f))
+
+        executables = [cx_Freeze.Executable(
+            app_path,
+            targetName="urh.exe",
+            icon=os.path.join("data", 'icons', 'appicon.ico'),
+            shortcutName="Universal Radio Hacker",
+            shortcutDir="DesktopFolder",
+            base="Win32GUI")]
+    else:
+        include_files = [os.path.join("data", 'icons', 'appicon.png')]
+        executables = [cx_Freeze.Executable(
+            app_path,
+            targetName="urh",
+            icon=os.path.join("data", 'icons', 'appicon.png'))]
+
+    for f in os.listdir(os.path.join("src", "urh", "dev", "gr", "scripts")):
+        if f.endswith(".py"):
+            include_files.append(os.path.join("src", "urh", "dev", "gr", "scripts", f))
+
+    plugins = []
+    plugin_path = os.path.join("src", "urh", "plugins")
+    for plugin in os.listdir(plugin_path):
+        if os.path.isdir(os.path.join(plugin_path, plugin)):
+            for f in os.listdir(os.path.join(plugin_path, plugin)):
+                if f.endswith(".py") and f != "__init__.py":
+                    plugins.append("urh.plugins.{0}.{1}".format(plugin, f.replace(".py", "")))
+
+    options = {
+        'build_exe': {
+            "include_files": include_files,
+            "include_msvcr": True,
+            "excludes": ["tkinter"],
+            "includes": ['numpy.core._methods', 'numpy.lib.format', 'six', 'appdirs',
+                         'packaging', 'packaging.version', 'packaging.specifiers', 'packaging.requirements',
+                         'setuptools.msvc'] + plugins
+        },
+        'bdist_msi': {
+            "upgrade_code": "{96abcdef-1337-4711-cafe-beef4a1ce42}"
+        }
+    }
+
+    cx_Freeze.setup(
+        name="Universal Radio Hacker",
+        description="Universal Radio Hacker: investigate wireless protocols like a boss",
+        author="Johannes Pohl",
+        author_email="Johannes.Pohl90@gmail.com",
+        url="https://github.com/jopohl/urh",
+        license="GNU General Public License (GPL) 3",
+        version=version.VERSION,
+        executables=executables,
+        options=options
+    )
+
+if __name__ == '__main__':
+    if sys.platform == "win32":
+        build_exe("bdist_msi")
+    else:
+        build_exe()

src/urh/dev/BackendHandler.py

@@ -96,7 +96,7 @@ def __init__(self):
         if not hasattr(sys, 'frozen'):
             self.path = os.path.dirname(os.path.realpath(__file__))
         else:
-            self.path = os.path.join(os.path.dirname(sys.executable), "dev")
+            self.path = os.path.dirname(sys.executable)
 
         self.device_backends = {}
         """:type: dict[str, BackendContainer] """
@@ -175,7 +175,10 @@ def set_gnuradio_installed_status(self):
                 return
 
     def __device_has_gr_scripts(self, devname: str):
-        script_path = os.path.join(self.path, "gr", "scripts")
+        if not hasattr(sys, "frozen"):
+            script_path = os.path.join(self.path, "gr", "scripts")
+        else:
+            script_path = self.path
         devname = devname.lower().split(" ")[0]
         has_send_file = False
         has_recv_file = False

src/urh/dev/gr/AbstractBaseThread.py

@@ -196,11 +196,10 @@ def initialize_process(self):
         self.started.emit()
 
         if not hasattr(sys, 'frozen'):
-            rp = os.path.dirname(os.path.realpath(__file__))
+            rp = os.path.realpath(os.path.join(os.path.dirname(__file__), "scripts"))
         else:
-            rp = os.path.join(os.path.dirname(sys.executable), "dev", "gr")
+            rp = os.path.realpath(os.path.dirname(sys.executable))
 
-        rp = os.path.realpath(os.path.join(rp, "scripts"))
         suffix = "_recv.py" if self._receiving else "_send.py"
         filename = self.device.lower().split(" ")[0] + suffix
 

src/urh/main.py

@@ -58,6 +58,9 @@ def main():
         import urh.cythonext.path_creator
         import urh.cythonext.util
     except ImportError:
+        if hasattr(sys, "frozen"):
+            print("C++ Extensions not found. Exiting...")
+            sys.exit(1)
         print("Could not find C++ extensions, trying to build them.")
         old_dir = os.curdir
         os.chdir(os.path.join(src_dir, "urh", "cythonext"))
@@ -120,6 +123,8 @@ def main():
         # Ensure we get the app icon in windows taskbar
         import ctypes
         ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID("jopohl.urh")
+        import multiprocessing as mp
+        mp.freeze_support()
 
     main_window.showMaximized()
     # main_window.setFixedSize(1920, 1080 - 30)  # Youtube

src/urh/plugins/PluginManager.py

@@ -1,23 +1,14 @@
 import importlib
 import os
 
-import sys
-
 from urh import constants
 from urh.plugins.Plugin import Plugin, ProtocolPlugin
 from urh.util.Logger import logger
 
 
 class PluginManager(object):
     def __init__(self):
-        if hasattr(sys, 'frozen'):
-            exe_path = sys.executable
-            if os.path.islink(exe_path):
-                exe_path = os.readlink(exe_path)
-            self.plugin_path = os.path.realpath(os.path.join(exe_path, "..", "plugins"))
-            sys.path.append(os.path.realpath(os.path.join(exe_path, "..")))
-        else:
-            self.plugin_path = os.path.dirname(os.path.realpath(__file__))
+        self.plugin_path = os.path.dirname(os.path.realpath(__file__))
         self.installed_plugins = self.load_installed_plugins()
 
     @property
@@ -28,7 +19,7 @@ def load_installed_plugins(self):
         """ :rtype: list of Plugin """
         result = []
         plugin_dirs = [d for d in os.listdir(self.plugin_path) if os.path.isdir(os.path.join(self.plugin_path, d))]
-        settings  = constants.SETTINGS
+        settings = constants.SETTINGS
 
         for d in plugin_dirs:
             if d == "__pycache__":
@@ -49,10 +40,7 @@ def load_installed_plugins(self):
     @staticmethod
     def load_plugin(plugin_name):
         classname = plugin_name + "Plugin"
-        if hasattr(sys, 'frozen'):
-            module_path = "plugins." + plugin_name + "." + classname
-        else:
-            module_path = "urh.plugins." + plugin_name + "." + classname
+        module_path = "urh.plugins." + plugin_name + "." + classname
 
         module = importlib.import_module(module_path)
         return getattr(module, classname)

src/urh/util/SettingsProxy.py

@@ -1,4 +1,5 @@
 import psutil
+import sys
 
 from urh import constants
 from urh.util.Formatter import Formatter
@@ -20,5 +21,11 @@ def get_receive_buffer_size(resume_on_full_receive_buffer: bool, spectrum_mode:
             # Take 60% of avail memory
             threshold = constants.SETTINGS.value('ram_threshold', 0.6, float)
             num_samples = threshold * (psutil.virtual_memory().available / 8)
+
+        # Do not let it allocate too much on 32 bit
+        if 8*num_samples > sys.maxsize // 2:
+            num_samples = sys.maxsize // (8 * 2)
+            logger.info("Correcting buffer size to {}".format(num_samples))
+
         logger.info("Initializing receive buffer with size {0}B".format(Formatter.big_value_with_suffix(num_samples*8)))
         return int(num_samples)

src/urh/util/util.py

@@ -7,14 +7,17 @@
 
 def set_windows_lib_path():
     if sys.platform == "win32":
-        util_dir = os.path.dirname(os.path.realpath(__file__)) if not os.path.islink(__file__) \
-            else os.path.dirname(os.path.realpath(os.readlink(__file__)))
-        urh_dir = os.path.realpath(os.path.join(util_dir, ".."))
-        assert os.path.isdir(urh_dir)
-
-        arch = "x64" if sys.maxsize > 2 ** 32 else "x86"
-        dll_dir = os.path.realpath(os.path.join(urh_dir, "dev", "native", "lib", "win", arch))
-        print("Using DLLs from:", dll_dir)
+        if not hasattr(sys, "frozen"):
+            util_dir = os.path.dirname(os.path.realpath(__file__)) if not os.path.islink(__file__) \
+                else os.path.dirname(os.path.realpath(os.readlink(__file__)))
+            urh_dir = os.path.realpath(os.path.join(util_dir, ".."))
+            assert os.path.isdir(urh_dir)
+
+            arch = "x64" if sys.maxsize > 2 ** 32 else "x86"
+            dll_dir = os.path.realpath(os.path.join(urh_dir, "dev", "native", "lib", "win", arch))
+            print("Using DLLs from:", dll_dir)
+        else:
+            dll_dir = os.path.dirname(sys.executable)
         os.environ['PATH'] = os.environ['PATH'] + ";" + dll_dir