Set up CI with Azure Pipelines

README.md

@@ -1,7 +1,6 @@
 ![Screenshot](images/AccelDRM_lock.png)
 
-![Build Status](https://codebuild.eu-west-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiVXRTOGxGaW5hZ1BzVGtoa0pBODhPanFlL3RpbFJPeGo5dHNxN3Uwb1ZuUmZOc1VkejBOeGk2WGVodzZPK2NvdS9WWVZUSlJhZXFMbEgrejN1VDN6TE13PSIsIml2UGFyYW1ldGVyU3BlYyI6IkhtMFRSeW1aRzVWVUpZcVQiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=master)
-[![codecov](https://codecov.io/gh/Accelize/drmlib/branch/master/graph/badge.svg)](https://codecov.io/gh/Accelize/drmlib)
+[![Build Status](https://dev.azure.com/Accelize/DRM/_apis/build/status/DRM%20Library?branchName=master)](https://dev.azure.com/Accelize/DRM/_build/latest?definitionId=2&branchName=master)[![codecov](https://codecov.io/gh/Accelize/drmlib/branch/master/graph/badge.svg)](https://codecov.io/gh/Accelize/drmlib)
 
 # :closed_lock_with_key: Accelize DRM library : AccelDRM
 

deployment/Dockerfile.j2

@@ -0,0 +1,80 @@
+# Define Packages build environments for all supported OS
+# Used in the "Release" stage of "azure-pipeline.yml"
+
+FROM {{ osName }}:{{ osVersion }}
+
+ENV PACKAGES_DIR /opt/packages
+ENV TOX_PARALLEL_NO_SPINNER 1
+
+# "Building" and "Testing" common layer
+RUN \
+{%- if osName == "centos" and osVersion == "7" %}
+yum install -y epel-release yum-plugin-fastestmirror && \
+yum install -y \
+{%- elif osName in ("fedora", "centos") %}
+echo "fastestmirror=True" >> /etc/dnf/dnf.conf && \
+echo "max_parallel_downloads=10" >> /etc/dnf/dnf.conf && \
+echo "install_weak_deps=False" >> /etc/dnf/dnf.conf && \
+{%- if osName == "centos" %}
+dnf install -y epel-release 'dnf-command(config-manager)' && \
+dnf config-manager --set-enabled PowerTools && \
+{%- endif %}
+dnf install -y \
+{%- elif osName in ("debian", "ubuntu") %}
+echo "APT::Install-Recommends "0";" >> /etc/apt/apt.conf.d/no-recommend && \
+echo "APT::Install-Suggests "0";" >> /etc/apt/apt.conf.d/no-recommend && \
+echo "APT::Acquire::Queue-Mode "access";" >> /etc/apt/apt.conf.d/parallel && \
+echo "APT::Acquire::Retries 10;" >> /etc/apt/apt.conf.d/parallel && \
+apt-get update && \
+apt-get install -y \
+{%- endif %}
+    python3-pip && \
+echo "[global]" > /etc/pip.conf && \
+echo "disable-pip-version-check = true " >> /etc/pip.conf && \
+python3 -m pip install -U{% if osName == "centos" and osVersion == "7" %} --prefix='/usr'{% endif %} \
+    pip \
+    setuptools && \
+python3 -m pip install -U{% if osName == "centos" and osVersion == "7" %} --prefix='/usr'{% endif %} \
+    cython \
+    flake8 \
+    pytest \
+    requests \
+    tox \
+    wheel
+
+{%- if buildLayer %}
+# "Building" only layer
+RUN \
+{%- if osName in ("centos", "fedora") and osVersion != "7" %}
+dnf install -y \
+{%- elif osName == "centos" and osVersion == "7" %}
+yum install -y \
+{%- elif osName in ("debian", "ubuntu") %}
+apt-get install -y \
+{%- endif %}
+    doxygen \
+    gnupg \
+    make \
+{%- if osName in ("debian", "ubuntu") %}
+    dpkg-dev \
+    dpkg-sig \
+    file \
+    g++ \
+    libcurl4-openssl-dev \
+    libjsoncpp-dev \
+    pkg-config \
+    python3-dev && \
+{%- elif osName in ("centos", "fedora") %}
+    gcc \
+    gcc-c++ \
+    jsoncpp-devel \
+    libcurl-devel \
+    python3-devel \
+    rpm-build \
+    rpm-sign && \
+{%- endif %}
+python3 -m pip install -U{% if osName == "centos" and osVersion == "7" %} --prefix='/usr'{% endif %} \
+    breathe \
+    cmake \
+    sphinx_rtd_theme \
+{%- endif %}

deployment/README.md

@@ -105,24 +105,13 @@ Supported OS version end of life:
 
 #### Actions to perform
 
-For Fedora & Ubuntu rolling:
-- Increment `FROM <OS>:` in:
-  * `deployment/context/build-<OS>_previous_latest.Dockerfile`
-  * `deployment/context/build-<OS>_latest.Dockerfile`
-  * `deployment/context/test-aws_f1-<OS>_previous_latest.Dockerfile`
-  * `deployment/context/test-aws_f1-<OS>_latest.Dockerfile`
-
-For others :
-- Create Dockerfiles for new versions and remove end of life versions
-  Dockerfiles:
-  * `deployment/context/build-<OS>_<version>.Dockerfile`
-  * `deployment/context/test-aws_f1-<OS>_<version>.Dockerfile`
-- Update Dockerfile names in docker compose file: 
-  `deployment/docker-compose.yml`
-- Update [Docker-Hub builds](https://cloud.docker.com/u/accelize/repository/docker/accelize/accelizedrm-ci/builds)
-  with Dockerfile names
-- Update `-x OS` entries in the AWS CodeBuild buildspec:
-  `deployment/buildspec.yml`
+For all:
+- In the file `deployment/azure-pipelines.yml`, the increment the versions in
+  the `matrix` section of the job `buildPackages` of the stage `release`.
+- Eventually update `deployment/Dockerfile.j2` if installation instruction 
+  changed.
+
+For CentOS, Debian and Ubuntu LTS only:
 - Update supported OS list in documentation (And eventually installation
   instructions): `doc/drm_library_installation.rst`
 

deployment/acid/__init__.py

@@ -0,0 +1,171 @@
+# coding=utf-8
+"""
+Accelize utilities for CI/CD pipeline
+"""
+from os.path import basename as _basename, dirname as _dirname, join as _join
+
+
+def _call(command, **kwargs):
+    """
+    Call a command with automatic error handling.
+
+    Args:
+        command (iterable of str or str):
+        kwargs: subprocess.run keyword arguments.
+
+    Returns:
+        subprocess.CompletedProcess
+    """
+    from subprocess import run, PIPE
+    process = run(command, universal_newlines=True, stderr=PIPE, **kwargs)
+
+    if process.returncode:
+        if process.stderr:
+            msg = f'\nStderr messages:\033[30m\n{process.stderr}'
+        else:
+            msg = '\033[30m'
+        raise RuntimeError(
+            f"\033[31mError code: {process.returncode}{msg}")
+
+    return process
+
+
+def ensure_pip_packages(*packages):
+    """
+    Ensure pip packages are installed.
+
+    Args:
+        *packages (str): Package to install.
+    """
+    _call(['pip', 'install', '--disable-pip-version-check', '-q'] +
+          list(packages))
+
+
+def export(name, value, is_output=True):
+    """
+    Set a Azure pipeline variable.
+
+    Args:
+        name (str): Variable name.
+        value (str): Variable value.
+        is_output (bool): Make variable available to future jobs.
+    """
+    print(f'##vso[task.setvariable variable={name}'
+          f'{";isOutput=true" if is_output else ""}]{value}')
+
+
+def render_template(src, dst, show=True, **kwargs):
+    """
+    Render a file from a template using Jinja2.
+
+    Args:
+        src (str): Source template.
+        dst (str): Destination file.
+        show (bool): If true, print result.
+        kwargs: Template arguments.
+    """
+    ensure_pip_packages('jinja2')
+
+    from jinja2 import Environment, FileSystemLoader
+    env = Environment(loader=FileSystemLoader(_dirname(src)))
+    template = env.get_template(_basename(src))
+    rendered = template.render(**kwargs)
+    if show:
+        print('\033[34m== START RENDERED ==\033[30m\n'
+              f'{rendered}'
+              '\n\033[34m== END RENDERED ==\033[30m')
+    with open(dst, 'wt') as file:
+        file.write(rendered)
+
+
+def ensure_ansible_roles(path):
+    """
+    Ensure playbook dependencies roles are presents.
+
+    Args:
+        path (str): "playbook.yml" path.
+    """
+    ensure_pip_packages('pyyaml')
+    try:
+        from yaml import CSafeLoader as Loader
+    except ImportError:
+        from yaml import SafeLoader as Loader
+    from yaml import load
+
+    # Get dependencies from playbook
+    try:
+        with open(path, 'rt') as file:
+            roles = load(file, Loader=Loader)[0]['roles']
+    except KeyError:
+        # No roles
+        return
+    dependencies = set()
+    for role in roles:
+        try:
+            # formatted as - name: role_name
+            name = role['name']
+        except KeyError:
+            # Formatted as - role_name
+            name = role
+        if "." in name:
+            dependencies.add(name)
+
+    # Install dependencies
+    if dependencies:
+        _call(['ansible-galaxy', 'install'] + list(dependencies))
+
+
+def define_unique_name(project, repo, branch, build_id):
+    """
+    Name an unique name based on build information.
+
+    Args:
+        project (str): Project name.
+        repo (str): Repository name.
+        branch (str): Branch name.
+        build_id (str): Build ID.
+
+    Returns:
+        str: Unique name.
+    """
+    from secrets import token_hex
+
+    name = ''.join(c for c in ''.join((
+        project.capitalize(),
+        repo.split('/')[-1].capitalize(),
+        branch.capitalize(),
+        build_id.capitalize(),
+        token_hex(8)
+    )) if c.isalnum())
+
+    print('\n'.join((
+        f'Project: {project}',
+        f'Repository: {repo}',
+        f'Branch: {branch}',
+        f'Build ID: {build_id}',
+        f'Resources Name: {name}')))
+
+    return name
+
+
+def dump_tfvars(directory, **variables):
+    """
+    Dump variables in "terraform.tfvars.json".
+    Update existing files.
+
+    Args:
+        directory (str): Output directory.
+        **variables: Variables
+    """
+    from json import dump, load
+    path = _join(directory, 'terraform.tfvars.json')
+
+    try:
+        with open(path, 'rt') as json_file:
+            tfvars = load(json_file)
+    except FileNotFoundError:
+        tfvars = dict()
+
+    tfvars.update(variables)
+    with open(path, 'wt') as json_file:
+        dump(tfvars, json_file)

deployment/acid/agents/.artifactignore

@@ -0,0 +1,5 @@
+.terraform/
+*.tf
+*.py
+*.pem
+.artifactignore

deployment/acid/agents/awsEc2/.artifactignore

@@ -0,0 +1 @@
+../.artifactignore