From 08ea87f1637a02dee097d8715544e7357f032ef2 Mon Sep 17 00:00:00 2001 From: Jeff Thiessen Date: Fri, 10 Jun 2022 08:22:43 -0500 Subject: [PATCH 1/7] [WIP] Switch from travisci to github actions for tests. (#24) * Create ci-tests.yml * just unit tests for now * install pytest * make pytest command use python -m pytest * more python libraries, add testing maxtrix * switch py.test to python -m pytest * specify python 3 * specify pip3 * add setuptools * Use conda incubator for management * init bash for activate * default activate __irida_importer_py3.6.7 * now this is advance building * Add postgresql * add check for current conda env * Try using postgres service instead of action * do it via ubuntu service * unit tests * install pytest * irida running locally, galaxy tbd * get basic int test flow working w/o galaxy, trying for github actions * code restructure, galaxy not starting * galaxy, conda, and selenium running locally * galaxy running and stopping * switch to gunicorn for galaxy host, improve irida/galaxy up check tests, tests passing locally * add galaxy configuration post setup * reorder some things to ensure irida/galaxy are killed after tests pass/fail * tool import test working locally * fake files working, galaxy->irida talking, oauth broken * authentication working on final importer test * reset postgres setup, add CONDA_PREFIX environment variable * switch to using ChromeDriverManager for selenium * force chromedriver to run headlesss and without gpu * cleanup now that test runners are all working (YAY) * fix selenium tests for IRIDA release 22.05 * swap dataset data output order --- .ci/install_deps.sh | 58 --- .github/workflows/ci-tests.yml | 105 +++++ .gitignore | 3 + .travis.yml | 35 -- CHANGELOG.md | 3 + Makefile | 34 ++ README.md | 17 +- irida_import/irida_import.xml.sample | 2 +- .../{install.sh => install_galaxy.sh} | 30 +- .../integration/bash_scripts/install_irida.sh | 19 + .../tests/integration/create_client.sql | 12 + .../tests/integration/galaxy_data_setup.py | 202 ++++++++ .../tests/integration/irida_data_setup.py | 162 +++++++ .../integration/start_integration_tests.py | 35 ++ .../integration/test_irida_import_int.py | 437 ------------------ .../tests/integration/test_irida_importer.py | 217 +++++++++ .../tests/integration/tests_integration.py | 142 ++++++ irida_import/tests/integration/util.py | 40 -- irida_import/tests/unit/test_irida_import.py | 10 +- run-tests.sh | 93 ---- setup.py | 14 + 21 files changed, 967 insertions(+), 703 deletions(-) delete mode 100755 .ci/install_deps.sh create mode 100644 .github/workflows/ci-tests.yml delete mode 100644 .travis.yml create mode 100644 Makefile rename irida_import/tests/integration/bash_scripts/{install.sh => install_galaxy.sh} (77%) create mode 100755 irida_import/tests/integration/bash_scripts/install_irida.sh create mode 100644 irida_import/tests/integration/create_client.sql create mode 100644 irida_import/tests/integration/galaxy_data_setup.py create mode 100644 irida_import/tests/integration/irida_data_setup.py create mode 100644 irida_import/tests/integration/start_integration_tests.py delete mode 100644 irida_import/tests/integration/test_irida_import_int.py create mode 100644 irida_import/tests/integration/test_irida_importer.py create mode 100644 irida_import/tests/integration/tests_integration.py delete mode 100644 irida_import/tests/integration/util.py delete mode 100755 run-tests.sh diff --git a/.ci/install_deps.sh b/.ci/install_deps.sh deleted file mode 100755 index 69275535..00000000 --- a/.ci/install_deps.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -e - -CHROMEDRIVER_VERSION=$1 -PYTHON_VERSION=$2 - -#some setup of bioconda 'borrowed' from https://github.com/bioconda/bioconda-recipes/tree/master/.circleci -CONDA_ENV="__irida_importer_py${PYTHON_VERSION}" - -# check if conda is available -conda --version 2>/dev/null 1>/dev/null -if [ $? -ne 0 ]; -then - WORKSPACE=`pwd` - BASH_ENV=`mktemp` - - # Set path - echo "export PATH=\"$WORKSPACE/miniconda/bin:$PATH\"" >> $BASH_ENV - source $BASH_ENV - - # check if we already installed conda to $WORKSPACE/miniconda - conda --version 2>/dev/null 1>/dev/null - if [ $? -ne 0 ]; - then - echo "Installing conda" - - curl -L -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - bash miniconda.sh -b -p $WORKSPACE/miniconda - fi -fi - -# check for conda environment -conda list -n $CONDA_ENV 2>/dev/null 1>/dev/null -if [ $? -ne 0 ]; -then - echo "Installing conda environment $CONDA_ENV" - - - conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults --name $CONDA_ENV bioblend=0.13.0 oauthlib=3.0.1 requests=2.22.0 requests-oauthlib=1.2.0 simplejson=3.8.1 python=$PYTHON_VERSION pip - - source activate $CONDA_ENV - pip install pytest pytest-cov pytest-mock mock subprocess32 selenium -else - source activate $CONDA_ENV -fi - -# Install chromedriver if not correct version -TEST_CHROMEDRIVER_VERSION=`chromedriver --version | sed -e 's/^ChromeDriver //' -e 's/ (.*//' 2>/dev/null` -if [ "$TEST_CHROMEDRIVER_VERSION" != "$CHROMEDRIVER_VERSION" ]; -then - echo "Downloading Chromedriver Version: $CHROMEDRIVER_VERSION" - wget --no-verbose -O /tmp/chromedriver_linux64.zip https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip - pythonbin=`which python` - bindir=`dirname $pythonbin` - unzip /tmp/chromedriver_linux64.zip -d $bindir - chmod 755 $bindir/chromedriver -else - echo "Chromedriver version [" $CHROMEDRIVER_VERSION "] already exists, not installing" -fi diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml new file mode 100644 index 00000000..9924e9e5 --- /dev/null +++ b/.github/workflows/ci-tests.yml @@ -0,0 +1,105 @@ +name: Integration Tests + +on: + pull_request: # Run on all pull requests + push: + branches: # Run on any push to development or master + - development + - master + schedule: # Run weekly on development and master + - cron: 0 2 * * 1 + branches: development + - cron: 0 2 * * 1 + branches: master + + +jobs: + build-unittests: + runs-on: ubuntu-20.04 #See pre-installed software at https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md + strategy: + fail-fast: False #Setting so that if one of the test suites fail, the other will continue + matrix: + python-version: [ 3.6 ] + + steps: + - uses: actions/checkout@v2 #Checkout the project from git + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Run Unit Tests + run: make unittests + + build-irida-integration: + runs-on: ubuntu-20.04 #See pre-installed software at https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md + env: + MYSQL_PORT: 3306 + MYSQL_USER: test + MYSQL_PASSWORD: test + MYSQL_DATABASE: irida_importer_test + MYSQL_HOST: 127.0.0.1 + MYSQL_ROOT_PASSWORD: password + NODE_OPTIONS: "--max-old-space-size=4096" + CONDA_PREFIX: /usr/share/miniconda + + strategy: + fail-fast: False #Setting so that if one of the test suites fail, the other will continue + matrix: + branch: ['master','development'] # IRIDA Branches to test against + + steps: + - uses: actions/checkout@v2 #Checkout the project from git + - name: Setup MySQL + uses: mirromutth/mysql-action@v1.1 + with: + host port: ${{ env.MYSQL_PORT }} + character set server: 'utf8' + collation server: 'utf8_general_ci' + mysql version: '5.7' + mysql database: ${{ env.MYSQL_DATABASE }} + mysql user: ${{ env.MYSQL_USER }} + mysql password: ${{ env.MYSQL_PASSWORD }} + mysql root password: ${{ env.MYSQL_ROOT_PASSWORD }} #The root superuser password + - name: Verify MySQL connection + timeout-minutes: 10 + run: | + while ! mysqladmin ping -h"${{ env.MYSQL_HOST }}" -P"${{ env.MYSQL_PORT }}" --silent; do + sleep 1 + done + - name: Set up JDK 11 # Installs java 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: MySQL Setup (SUDO) # Sets ONLY_FULL_GROUP_BY flag and gives runner privileges over database + run: | + sudo mysql -e "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" -h ${{ env.MYSQL_HOST }} -P ${{ env.MYSQL_PORT }} -p${{ env.MYSQL_ROOT_PASSWORD }}; + sudo mysql -e "CREATE USER '${{ env.MYSQL_USER }}'@'%' IDENTIFIED BY '${{ env.MYSQL_PASSWORD }}'; GRANT ALL ON ${{ env.MYSQL_DATABASE }}.* to '${{ env.MYSQL_USER }}'@'%';" -h ${{ env.MYSQL_HOST }} -P ${{ env.MYSQL_PORT }} -p${{ env.MYSQL_ROOT_PASSWORD }}; + - name: Set up PostgreSQL + run: | + sudo apt-get update + sudo apt-get install postgresql libpq-dev + sudo service postgresql start + sudo -u postgres createuser --superuser "$USER" + createdb runner + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: 3.6.7 + activate-environment: __irida_importer_py3.6.7 + - name: Conda info + shell: bash -l {0} + run: conda info + - name: Install conda packages + shell: bash -l {0} + run: | + conda config --add channels defaults + conda config --add channels conda-forge + conda config --add channels bioconda + conda config --add channels iuc + conda install bioblend=0.13.0 oauthlib=3.0.1 requests=2.22.0 requests-oauthlib=1.2.0 simplejson=3.8.1 + - name: Setup galaxy conda env + shell: bash -l {0} + run: conda create --name _galaxy_ python=3.7.6 + - name: Run Integration Tests ${{ matrix.branch }} + timeout-minutes: 60 + run: make integrationtests branch=${{ matrix.branch }} db_host=${{ env.MYSQL_HOST }} db_port=${{ env.MYSQL_PORT }} diff --git a/.gitignore b/.gitignore index 245c401d..55a230d5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ env /.venv miniconda* .vscode +.virtualenv +irida_import.egg-info +irida_import/tests/integration/tmp diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b7e1770a..00000000 --- a/.travis.yml +++ /dev/null @@ -1,35 +0,0 @@ -sudo: required -language: java - -dist: xenial - -services: - - mysql - - postgresql - - xvfb - -jdk: - - openjdk11 - -env: - global: - - "MYSQL_DATABASE=irida_test" - - "MYSQL_USER=test" - - "MYSQL_PASSWORD=test" - - "MAVEN_OPTS=-Djava.awt.headless=true" - - "CHROMEDRIVER_VERSION=79.0.3945.36" - - "PYTHON_VERSION=3.6.7" - - "XVFB_STARTED=1" - -cache: - directories: - - $HOME/.m2 - -install: - - mysql -e "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" - - mysql -e "CREATE USER '$MYSQL_USER'@'localhost' IDENTIFIED BY '$MYSQL_PASSWORD'; GRANT ALL ON $MYSQL_DATABASE.* to '$MYSQL_USER'@'localhost';" - - psql -c 'create database galaxy_test;' - - mvn -version - - source .ci/install_deps.sh $CHROMEDRIVER_VERSION - -script: ./run-tests.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 13093ecb..e7ebd641 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All notable changes to irida-galaxy-importer will be documeted in this file. +## 2.0.1 +* Switched from travisCI tests to Github Actions + ## 2.0.0 * Added support for Python 3, while maintaining Python 2 support diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..f23bf182 --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ +SHELL=/bin/bash +IRIDA_VERSION?=master + +requirements: clean env + source .virtualenv/bin/activate + python3 -m pip install -e . + +clean: + rm -rf irida_import/.pytest_cache + rm -rf .virtualenv + rm -rf irida_import.egg-info/ + find -name "*pyc" -delete + rm -rf /tmp/repos + rm -rf irida_import/tests/integration/tmp + rm -rf venv + +env: + python3 -m venv .virtualenv + source .virtualenv/bin/activate + python3 --version + python3 -m pip install --upgrade wheel pip + +unittests: clean env + source .virtualenv/bin/activate + pip3 install -e . + pip3 install pytest + pytest irida_import/tests/unit/*.py + +integrationtests: clean env + source .virtualenv/bin/activate + python3 -m pip install -e .[TEST] + python3 -m pip install pytest + mkdir /tmp/repos + python3 irida_import/tests/integration/start_integration_tests.py $(branch) $(db_host) $(db_port) diff --git a/README.md b/README.md index 8779c895..952cb83c 100644 --- a/README.md +++ b/README.md @@ -268,25 +268,18 @@ mysql -u root -e "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROU ``` ## 3.2. Running tests - -To run all the test, you can run: - -```bash -./run-tests.sh -``` - -If you just want to run just the unit tests (much quicker) you can do: +Unit tests can be run with: ```bash -./run-tests.sh unit +make unittests ``` -If you just want to run just the integration tests you can do: +Integration tests can be run with ```bash -./run-tests.sh integration +make integrationtests branch=master ``` - +This tests against the `master` branch of IRIDA [galaxy]: https://galaxyproject.org/ [irida]: https://www.irida.ca/ diff --git a/irida_import/irida_import.xml.sample b/irida_import/irida_import.xml.sample index 82694f5e..6c82ac14 100644 --- a/irida_import/irida_import.xml.sample +++ b/irida_import/irida_import.xml.sample @@ -15,8 +15,8 @@ - diff --git a/irida_import/tests/integration/bash_scripts/install.sh b/irida_import/tests/integration/bash_scripts/install_galaxy.sh similarity index 77% rename from irida_import/tests/integration/bash_scripts/install.sh rename to irida_import/tests/integration/bash_scripts/install_galaxy.sh index 6deb0862..931d492a 100755 --- a/irida_import/tests/integration/bash_scripts/install.sh +++ b/irida_import/tests/integration/bash_scripts/install_galaxy.sh @@ -4,26 +4,7 @@ args=("$@") tool_loc=${args[0]} galaxy_port=${args[1]} -mkdir repos -pushd repos - -echo "Downloading IRIDA..." -git clone https://github.com/phac-nml/irida.git -pushd irida -git checkout master > irida-checkout.log 2>&1 -git fetch -git reset --hard -git clean -fd -git pull -echo "Preparing IRIDA for first excecution..." -rm -rf /tmp/shed_tools/ -pkill -u $USER -f "python ./scripts/paster.py" || true - -pushd lib -./install-libs.sh > /dev/null -popd -popd -echo "IRIDA has been installed" +pushd /tmp/repos echo "Downloading Galaxy..." git clone https://github.com/galaxyproject/galaxy/ > galaxy-clone.log 2>&1 @@ -36,6 +17,10 @@ git pull echo "Preparing Galaxy for first execution (installing eggs)..." ./scripts/common_startup.sh > galaxy-common-startup.log 2>&1 +# reset galaxy psql db +echo "dropping galaxy_test database in psql" +echo "drop database if exists galaxy_test; create database galaxy_test;" | psql + echo "Configuring Galaxy." pushd config @@ -55,7 +40,8 @@ echo " database_connection: postgresql:///galaxy_test" | cat >> galaxy.yml sed -i 's/#admin_users:.*/admin_users: "irida@irida.ca"/' galaxy.yml # run galaxy on port 8888 instead of 8080; Tomcat runs on 8080 by default. -sed -i "s|http: 127.0.0.1:8080|http: 127.0.0.1:$galaxy_port|" galaxy.yml +sed -i "s|# bind: localhost:8080|bind: localhost:$galaxy_port|" galaxy.yml + popd popd echo "Galaxy has been installed" @@ -82,4 +68,4 @@ popd popd -echo "IRIDA, Galaxy, and the IRIDA export tool have been installed" +echo "Galaxy and the IRIDA export tool have been installed" diff --git a/irida_import/tests/integration/bash_scripts/install_irida.sh b/irida_import/tests/integration/bash_scripts/install_irida.sh new file mode 100755 index 00000000..7683ad0a --- /dev/null +++ b/irida_import/tests/integration/bash_scripts/install_irida.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +pushd /tmp/repos + +echo "Downloading IRIDA..." +if ! git clone https://github.com/phac-nml/irida.git --branch $1 +then + echo >&2 "Failed to clone" + exit 1 +else + pushd irida + echo "Preparing IRIDA for first excecution..." + + pushd lib + ./install-libs.sh + popd + popd + echo "IRIDA has been installed" +fi diff --git a/irida_import/tests/integration/create_client.sql b/irida_import/tests/integration/create_client.sql new file mode 100644 index 00000000..be68909f --- /dev/null +++ b/irida_import/tests/integration/create_client.sql @@ -0,0 +1,12 @@ +-- remote apis +INSERT INTO client_details (id,clientId,clientSecret,token_validity,refresh_validity,createdDate,modifiedDate) VALUES (1,"myClient","myClientSecret",100000,2592000,now(),now()); +INSERT INTO client_details_grant_types (client_details_id,grant_value) VALUES (1,"password"); +INSERT INTO client_details_scope (client_details_id,scope) VALUES (1,"read"); +INSERT INTO client_details_scope (client_details_id,scope) VALUES (1,"write"); +-- galaxy auth connection +INSERT INTO client_details (id,clientId,clientSecret,token_validity,refresh_validity,createdDate,modifiedDate,redirect_uri) VALUES (2,"auth_code_client","auth_code_secret",100000,2592000,now(),now(),"http://localhost:8080/galaxy/auth_code"); +INSERT INTO client_details_grant_types (client_details_id,grant_value) VALUES (2,"authorization_code"); +INSERT INTO client_details_scope (client_details_id,scope) VALUES (2,"read"); + +-- user -- password encryption of `password1` +INSERT INTO user (`createdDate`, `modifiedDate`, `email`, `firstName`, `lastName`, `locale`, `password`, `phoneNumber`, `username`, `enabled`, `system_role`, `credentialsNonExpired`) VALUES (now(), now() , 'jeffrey.thiessen@phac-aspc.gc.ca', 'Jeffrey', 'Thiessen', 'en', '$2a$10$yvzFLxWA9m2wNQmHpJtWT.MRZv8qV8Mo3EMB6HTkDnUbi9aBrbWWW', '0000', 'jeff', 1, 'ROLE_ADMIN', 1); diff --git a/irida_import/tests/integration/galaxy_data_setup.py b/irida_import/tests/integration/galaxy_data_setup.py new file mode 100644 index 00000000..13234dd0 --- /dev/null +++ b/irida_import/tests/integration/galaxy_data_setup.py @@ -0,0 +1,202 @@ +""" +This file is responsible for Managing an IRIDA instance, including all the parameters associated with testing and +database management. +""" + +import logging +import configparser +from os import path +from time import time, sleep +import sys +import subprocess +import inspect +import socket +from selenium import webdriver +from selenium.common import exceptions as selenium_exceptions +from bioblend import galaxy as bioblend_galaxy + +from ...irida_import import IridaImport + + +class SetupGalaxyData: + + def __init__(self, repo_dir, email, chrome_driver_path, chrome_driver_options): + """ + :param repo_dir: where to download and install galaxy + :param email: email for galaxy account (for connection to irida) + :param chrome_driver_path: where chromedriver is installed + :param chrome_driver_options: options for chrome while using chromedriver + """ + + self.log = self._setup_galaxy_logger() + + module_dir = path.dirname(path.abspath(__file__)) + self.SCRIPTS = path.join(module_dir, 'bash_scripts') + self.REPOS_PARENT = module_dir + self.REPOS = repo_dir + self.GALAXY_REPO = path.join(self.REPOS, 'galaxy') + self.TOOL_DIRECTORY = path.dirname(inspect.getfile(IridaImport)) + self.CONFIG_PATH = path.join(self.GALAXY_REPO, + 'tools', 'irida-galaxy-importer', 'irida_import', + 'config.ini') + self.INSTALL_EXEC = 'install_galaxy.sh' + self.GALAXY_DOMAIN = 'localhost' + self.GALAXY_PORT = 8888 + self.GALAXY_URL = 'http://' + self.GALAXY_DOMAIN + ':' + str( + self.GALAXY_PORT) + self.CONDA_INIT = '. $CONDA_PREFIX/etc/profile.d/conda.sh && conda activate _galaxy_ && ' + self.GALAXY_STOP = self.CONDA_INIT + 'bash run.sh --stop-daemon' + self.GALAXY_RUN = self.CONDA_INIT + 'bash run.sh --daemon' + + self.GALAXY_EMAIL = email + self.GALAXY_PASSWORD = 'Password1' + # defined in create_client.sql + self.REDIRECT_CLIENT_SECRET = "auth_code_secret" + + self.CHROME_DRIVER_PATH = chrome_driver_path + self.CHROME_DRIVER_OPTIONS = chrome_driver_options + + @staticmethod + def _setup_galaxy_logger(): + """ + setup logging to debug level for tests + :return: logger object + """ + log = logging.getLogger() + log.setLevel(logging.DEBUG) + log_out = logging.StreamHandler(sys.stdout) + log_out.setLevel(logging.DEBUG) + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + log_out.setFormatter(formatter) + log.addHandler(log_out) + return log + + def install_galaxy(self): + """ + Installs Galaxy and blocks until up + :return: + """ + # Install Galaxy and the IRIDA export tool: + exec_path = path.join(self.SCRIPTS, self.INSTALL_EXEC) + install = subprocess.Popen([exec_path, self.TOOL_DIRECTORY, + str(self.GALAXY_PORT)], + cwd=self.REPOS_PARENT) + install.wait() # Block until installed + + def stop_galaxy(self): + """ + Kill galaxy process + :return: + """ + self.log.info('Killing Galaxy') + subprocess.Popen(self.GALAXY_STOP, shell=True, cwd=self.GALAXY_REPO) + + def start_galaxy(self): + """ + Starts Galaxy and blocks until db migration is finihsed and web ui up + :return: + """ + self.log.info("Running galaxy") + subprocess.Popen(self.GALAXY_RUN, shell=True, cwd=self.GALAXY_REPO) + self.log.debug("Waiting for Galaxy database migration [%s]. Sleeping for [%s] seconds", self.GALAXY_URL, 360) + sleep(360) + self.log.debug("Galaxy database migration should have (hopefully) finished, checking if it is up") + self._wait_until_up( + self.GALAXY_DOMAIN, + self.GALAXY_PORT, + 600) + self.log.debug("Galaxy should now be up on [%s]", self.GALAXY_URL) + + def _wait_until_up(self, address, port, timeout): + """ + Wait until a port at an address is occupied, or time out + + :type address: str + :param address: e.g. 'localhost' or '127.0.0.1' + :type port: int + :param port: e.g. 8888 + :type timeout: int + :param timeout: the time to wait in seconds until throwing an exception + """ + + def check_up(addr, p): + """ + Find out if a port at an address is occupied + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((addr, p)) + sock.close() + if result == 0: + ans = True + else: + ans = False + return ans + + max_time = time() + timeout + up = False + while not up and time() < max_time: + self.log.debug("Checking if Galaxy is up...") + up = check_up(address, port) + + # If we query Galaxy immediately it may reset the connection: + sleep(10) + + if not up: + raise Exception('There was no response at {} on port {} for {} seconds' + .format(address, port, timeout)) + + def register_galaxy(self): + """Register with Galaxy, and then attempt to log in""" + driver = self._get_webdriver() + + try: + driver.get(self.GALAXY_URL) + driver.find_element_by_link_text("Login or Register").click() + driver.find_element_by_id("register-toggle").click() + driver.find_element_by_name("email").send_keys(self.GALAXY_EMAIL) + driver.find_element_by_name("password").send_keys(self.GALAXY_PASSWORD) + driver.find_element_by_name("confirm").send_keys(self.GALAXY_PASSWORD) + driver.find_element_by_name("username").send_keys("irida-test") + driver.find_element_by_name("create").click() + + driver.get(self.GALAXY_URL) + driver.find_element_by_link_text("Login or Register").click() + driver.find_element_by_name("login").send_keys(self.GALAXY_EMAIL) + driver.find_element_by_name("password").send_keys(self.GALAXY_PASSWORD) + driver.find_element_by_xpath("//button[@name='login']").click() + driver.find_element_by_name("login").click() + except selenium_exceptions.NoSuchElementException: + pass + finally: + driver.quit() + + def configure_galaxy_api_key(self): + """Make a new Galaxy admin API key and configure the tool to use it""" + gal = bioblend_galaxy.GalaxyInstance( + self.GALAXY_URL, + email=self.GALAXY_EMAIL, + password=self.GALAXY_PASSWORD) + self.configure_tool('Galaxy', 'admin_key', gal.key) + self.log.info('Galaxy admin_key:' + gal.key) + + def configure_irida_galaxy_connection(self, galaxy_url): + """configures the tool to know the redirect secret for the irida->galaxy connection""" + self.configure_tool('IRIDA', 'client_secret', self.REDIRECT_CLIENT_SECRET) + self.configure_tool('Galaxy', 'galaxy_url', galaxy_url) + + def configure_tool(self, section, option, value): + """Write tool configuration data""" + config = configparser.ConfigParser() + config.read(self.CONFIG_PATH) + config.set(section, option, value) + with open(self.CONFIG_PATH, 'w') as config_file: + config.write(config_file) + + def _get_webdriver(self): + """Set up the Selenium WebDriver""" + driver = webdriver.Chrome(self.CHROME_DRIVER_PATH, options=self.CHROME_DRIVER_OPTIONS) + driver.implicitly_wait(1) + driver.set_window_size(1024, 768) + + return driver diff --git a/irida_import/tests/integration/irida_data_setup.py b/irida_import/tests/integration/irida_data_setup.py new file mode 100644 index 00000000..8137085f --- /dev/null +++ b/irida_import/tests/integration/irida_data_setup.py @@ -0,0 +1,162 @@ +""" +This file is responsible for Managing an IRIDA instance, including all the parameters associated with testing and +database management. +""" +import requests +from requests import ConnectionError +from urllib.error import URLError +from os import path +from time import time, sleep +import sys +import subprocess +from http import HTTPStatus +from pathlib import Path + + +class SetupIridaData: + + def __init__(self, base_url, user, password, branch, db_host, db_port, repo_dir): + """ + :param base_url: url of the IRIDA instance's API + :param user: default admin username + :param password: default admin password + :param branch: the github branch to run the integration tests on (e.g. 'master' or 'development') + :param db_host: mysql database host + :param db_port: mysql database port + :param repo_dir: where to download and install irida + """ + + # Login info + self.base_url = base_url + self.user = user + self.password = password + self.branch = branch + + # Database info + self.DB_HOST = db_host + self.DB_PORT = db_port + self.DB_NAME = "irida_importer_test" + self.DB_USERNAME = "test" + self.DB_PASSWORD = "test" + self.DB_JDBC_URL = "jdbc:mysql://" + self.DB_HOST + ":" + self.DB_PORT + "/" + self.DB_NAME + + self.TIMEOUT = 600 # seconds + + self.IRIDA_DB_RESET = 'echo '\ + '"drop database if exists ' + self.DB_NAME + ';'\ + 'create database ' + self.DB_NAME + ';'\ + '"| mysql -h ' + self.DB_HOST + ' -P ' + self.DB_PORT + ' -u test -ptest' + + root_dir = Path(__file__).parent.absolute() + sql_file = Path(root_dir, "create_client.sql") + output_files = Path(root_dir, "tmp", "output-files") + reference_file = Path(root_dir, "tmp", "reference-files") + sequence_files = Path(root_dir, "tmp", "sequence-files") + assembly_files = Path(root_dir, "tmp", "assembly-files") + + self.IRIDA_DB_UPDATE = 'mysql -h ' + self.DB_HOST + ' -P ' + self.DB_PORT + ' -u test -ptest ' + \ + self.DB_NAME + ' < {}'.format(sql_file) + + self.IRIDA_CMD = \ + 'mvn clean compile spring-boot:start -DskipTests --quiet ' +\ + '-Dspring-boot.run.arguments=\"' +\ + '--spring.datasource.url={} '.format(self.DB_JDBC_URL) +\ + '--spring.datasource.username={} '.format(self.DB_USERNAME) +\ + '--spring.datasource.password={} '.format(self.DB_PASSWORD) +\ + '--liquibase.update.database.schema=true ' +\ + '--spring.jpa.hibernate.ddl-auto= ' +\ + '--spring.jpa.properties.hibernate.hbm2ddl.import_files= ' +\ + '--sequence.file.base.directory={} '.format(sequence_files) +\ + '--reference.file.base.directory={} '.format(reference_file) +\ + '--output.file.base.directory={} '.format(output_files) +\ + '--assembly.file.base.directory={} '.format(assembly_files) +\ + '--logging.pattern.console=' +\ + '\"' + + self.IRIDA_STOP = 'mvn spring-boot:stop' + + self.PATH_TO_MODULE = path.dirname(__file__) + if len(self.PATH_TO_MODULE) == 0: + self.PATH_TO_MODULE = "." + + self.SCRIPT_FOLDER = path.join(self.PATH_TO_MODULE, "bash_scripts") + self.INSTALL_IRIDA_EXEC = path.join( + self.SCRIPT_FOLDER, "install_irida.sh") + + self.REPO_PATH = repo_dir + self.IRIDA_PATH = path.join(self.REPO_PATH, "irida") + + def install_irida(self): + """ + Installs IRIDA + :return: + """ + install_proc = subprocess.Popen( + [self.INSTALL_IRIDA_EXEC, self.branch], cwd=self.PATH_TO_MODULE) + proc_res = install_proc.wait() + if proc_res == 1: # failed to execute + sys.exit(1) + + def reset_irida_db(self): + """ + Deletes the database and creates a new empty one + :return: + """ + db_reset_proc = subprocess.Popen(self.IRIDA_DB_RESET, shell=True) + proc_res = db_reset_proc.wait() + + if proc_res == 1: # failed to execute + print("Unable to execute:\n {cmd}".format(cmd=self.IRIDA_DB_RESET)) + sys.exit(1) + + def update_irida_db(self): + """ + Adds a user and a client for api operations to IRIDA database + :return: + """ + db_update_proc = subprocess.Popen(self.IRIDA_DB_UPDATE, shell=True) + proc_res = db_update_proc.wait() + + if proc_res == 1: # failed to execute + print("Unable to execute:\n {cmd}".format(cmd=self.IRIDA_DB_UPDATE)) + sys.exit(1) + + def run_irida(self): + """ + Runs IRIDA with first initialization + Waits until IRIDA is up to return + :return: + """ + subprocess.Popen( + self.IRIDA_CMD, cwd=self.IRIDA_PATH, shell=True) + self._wait_until_up() + + def _wait_until_up(self): + """ + Queries IRIDA until it responds with HTTP 200 + :return: + """ + start_time = time() + elapsed = 0 + status_code = -1 + print("Waiting for " + self.base_url) + + while status_code != HTTPStatus.OK and elapsed < self.TIMEOUT: + try: + print("Trying to connect...") + status_code = requests.api.get(self.base_url).status_code + elapsed = time() - start_time + + except (URLError, ConnectionError): + sleep(10) + print("Connected.") + + def stop_irida(self): + """ + Stops the IRIDA mvn process + This will sometimes dump errors into the log, but it is harmless + :return: + """ + stopper = subprocess.Popen( + self.IRIDA_STOP, cwd=self.IRIDA_PATH, shell=True) + stopper.wait() diff --git a/irida_import/tests/integration/start_integration_tests.py b/irida_import/tests/integration/start_integration_tests.py new file mode 100644 index 00000000..0168c1f1 --- /dev/null +++ b/irida_import/tests/integration/start_integration_tests.py @@ -0,0 +1,35 @@ +""" +This file is used to kick off the integration tests + +it can be used from command line like so: + $ python3 start_integration_tests.py +where is the IRIDA github branch to test against +""" + +import argparse +import sys + +import irida_import.tests.integration.tests_integration as tests_integration + +# parse a single argument that determines which IRIDA branch we are testing against +argument_parser = argparse.ArgumentParser(description='This program parses sequencing runs and uploads them to IRIDA.') +argument_parser.add_argument('branch', help='Which IRIDA branch to run integration tests against') +argument_parser.add_argument('db_host', help="Override database host", + nargs='?', action='store', default=False) +argument_parser.add_argument('db_port', help="Override database port", + nargs='?', action='store', default=False) + + +def main(): + # parse argument + args = argument_parser.parse_args() + + # Start integration tests on specified branch + if args.db_host and args.db_port: + return tests_integration.start(args.branch, args.db_host, args.db_port) + else: + return tests_integration.start(args.branch) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/irida_import/tests/integration/test_irida_import_int.py b/irida_import/tests/integration/test_irida_import_int.py deleted file mode 100644 index 99f87893..00000000 --- a/irida_import/tests/integration/test_irida_import_int.py +++ /dev/null @@ -1,437 +0,0 @@ -import socket -import getpass -import inspect -import time -import sys -import logging -import os -import configparser -import pytest -import subprocess -from tempfile import mkdtemp -from selenium import webdriver -from selenium.common.exceptions import NoSuchElementException -from selenium.common.exceptions import StaleElementReferenceException -from selenium.webdriver.common.by import By -from selenium.webdriver.support.wait import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from ...irida_import import IridaImport -from . import util -from requests_oauthlib import OAuth2Session -from oauthlib.oauth2 import LegacyApplicationClient -from bioblend import galaxy - - -# These variables are to stop Galaxy and Irida from being changed -# during script execution. This is required if you are using your -# own instance of Galaxy and Irida. -# os.environ['IRIDA_GALAXY_TOOL_TESTS_DONT_INSTALL'] = "1" -# os.environ['IRIDA_GALAXY_TOOL_TESTS_DONT_START_GALAXY'] = "1" -# os.environ['IRIDA_GALAXY_TOOL_TESTS_DONT_STOP_GALAXY'] = "1" -# os.environ['IRIDA_GALAXY_TOOL_TESTS_DONT_START_IRIDA'] = "1" - - -class TestIridaImportInt: - """ - Perform integration tests on the IRIDA import tool for Galaxy - - To use an already running instance of Galaxy on port 8888, installation - must be disabled, in addition to Galaxy starting/stopping - """ - - TIMEOUT = 600 # seconds - GALAXY_SLEEP_TIME = 360 - - USER = getpass.getuser() - EMAIL = 'irida@irida.ca' - - GALAXY_PASSWORD = 'Password1' - GALAXY_DOMAIN = 'localhost' - GALAXY_CMD = ['bash', 'run.sh', '--daemon'] - GALAXY_STOP = ['bash', 'run.sh', '--stop-daemon'] - GALAXY_DB_RESET = 'echo "drop database if exists galaxy_test; create database galaxy_test;" | psql' - - IRIDA_DOMAIN = 'localhost' - IRIDA_PORT = 8080 - IRIDA_URL = 'http://' + IRIDA_DOMAIN + ':' + str(IRIDA_PORT) - IRIDA_CMD = ['mvn', 'clean', 'jetty:run', - '-Djdbc.url=jdbc:mysql://localhost:3306/irida_test', - '-Djdbc.username=test', '-Djdbc.password=test', - '-Dliquibase.update.database.schema=true', - '-Dhibernate.hbm2ddl.auto=', - '-Dhibernate.hbm2ddl.import_files='] - IRIDA_STOP = 'mvn jetty:stop' - IRIDA_DB_RESET = 'echo ' \ - '"drop database if exists irida_test;' \ - 'create database irida_test;' \ - '"| mysql -u test -ptest' - IRIDA_PASSWORD_ID = 'password_client' - IRIDA_AUTH_CODE_ID = 'auth_code_client' - IRIDA_REDIRECT_URI = IRIDA_URL + '/galaxy/auth_code' - IRIDA_USER = 'admin' - IRIDA_PASSWORD = 'Password1!' - IRIDA_TOKEN_ENDPOINT = IRIDA_URL + '/api/oauth/token' - IRIDA_PROJECTS = IRIDA_URL + '/api/projects' - - IRIDA_GALAXY_MODAL = 'galaxy-modal' - WAIT = 120 - - INSTALL_EXEC = 'install.sh' - - # Sequence files accessed by IRIDA's REST API will not exist when the - # tool attempts to access them if they were not uploaded as valid sequence - # files - FASTQ_CONTENTS = ( - "@SRR566546.970 HWUSI-EAS1673_11067_FC7070M:4:1:2299:1109 length=50\n" + - "TTGCCTGCCTATCATTTTAGTGCCTGTGAGGTGGAGATGTGAGGATCAGT\n" + - "+SRR566546.970 HWUSI-EAS1673_11067_FC7070M:4:1:2299:1109 length=50\n" + - "hhhhhhhhhhghhghhhhhfhhhhhfffffe`ee[`X]b[d[ed`[Y[^Y") - - def setup_class(self): - """Initialize class variables, install IRIDA, Galaxy, and the tool""" - module_dir = os.path.dirname(os.path.abspath(__file__)) - self.SCRIPTS = os.path.join(module_dir, 'bash_scripts') - self.REPOS_PARENT = module_dir - self.REPOS = os.path.join(module_dir, 'repos') - self.TOOL_DIRECTORY = os.path.dirname(inspect.getfile(IridaImport)) - self.CONFIG_PATH = os.path.join(self.TOOL_DIRECTORY, 'tests', - 'integration', 'repos', 'galaxy', - 'tools', 'irida-galaxy-importer', 'irida_import', - 'config.ini') - - self.GALAXY = os.path.join(self.REPOS, 'galaxy') - self.IRIDA = os.path.join(self.REPOS, 'irida') - - log = logging.getLogger() - log.setLevel(logging.DEBUG) - log_out = logging.StreamHandler(sys.stdout) - log_out.setLevel(logging.DEBUG) - formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') - log_out.setFormatter(formatter) - log.addHandler(log_out) - self.log = log - - try: - os.environ['IRIDA_GALAXY_TOOL_TESTS_DONT_INSTALL'] - self.GALAXY_PORT = 8080 - self.GALAXY_URL = 'http://' + self.GALAXY_DOMAIN + ':' + str( - self.GALAXY_PORT) - except KeyError: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind(('', 0)) - self.GALAXY_PORT = sock.getsockname()[1] - self.GALAXY_URL = 'http://' + self.GALAXY_DOMAIN + ':' + str( - self.GALAXY_PORT) - - # Install IRIDA, Galaxy, and the IRIDA export tool: - exec_path = os.path.join(self.SCRIPTS, self.INSTALL_EXEC) - install = subprocess.Popen([exec_path, self.TOOL_DIRECTORY, - str(self.GALAXY_PORT)], - cwd=self.REPOS_PARENT) - install.wait() # Block untill installed - - @pytest.fixture(scope='class') - def driver(self, request): - """Set up the Selenium WebDriver""" - driver = webdriver.Chrome() - driver.implicitly_wait(1) - driver.set_window_size(1024, 768) - - def finalize_driver(): - driver.quit() - - request.addfinalizer(finalize_driver) - return driver - - @pytest.fixture(scope='class') - def setup_irida(self, request, driver): - """Set up IRIDA for tests (Start if required, register, log in)""" - - def stop_irida(): - print('Stopping IRIDA nicely') - stopper = subprocess.Popen(self.IRIDA_STOP, cwd=self.IRIDA, - shell=True) - stopper.wait() - - try: - os.environ['IRIDA_GALAXY_TOOL_TESTS_DONT_START_IRIDA'] - except KeyError: - stop_irida() - - # create temporary directories for IRIDA data - data_dir = mkdtemp(prefix='irida-tmp-') - sequence_file_dir = mkdtemp(prefix='sequence-files-', dir=data_dir) - reference_file_dir = mkdtemp(prefix='reference-files-', dir=data_dir) - output_file_dir = mkdtemp(prefix='output-files-', dir=data_dir) - self.IRIDA_CMD.append('-Dsequence.file.base.directory=' + sequence_file_dir) - self.IRIDA_CMD.append('-Dreference.file.base.directory=' + reference_file_dir) - self.IRIDA_CMD.append('-Doutput.file.base.directory=' + output_file_dir) - - subprocess.call(self.IRIDA_DB_RESET, shell=True) - FNULL = open(os.devnull, 'w') - subprocess.Popen(self.IRIDA_CMD, cwd=self.IRIDA, env=os.environ,stdout=FNULL) - util.wait_until_up(self.IRIDA_DOMAIN, self.IRIDA_PORT, - self.TIMEOUT) - - def finalize_irida(): - stop_irida() - - request.addfinalizer(finalize_irida) - self.register_irida(driver) - self.add_irida_client_password(driver) - self.add_irida_client_auth_code(driver) - self.configure_irida_client_secret(driver) - - # Return an OAuth 2.0 authorized session with IRIDA - return self.get_irida_oauth(driver) - - @pytest.fixture(scope='class') - def setup_galaxy(self, request, driver): - """Set up Galaxy for tests (Start if required, register, log in)""" - - def stop_galaxy(): - try: - os.environ['IRIDA_GALAXY_TOOL_TESTS_DONT_STOP_GALAXY'] - except KeyError: - print('Killing Galaxy') - subprocess.Popen(self.GALAXY_STOP, cwd=self.GALAXY) - - try: - os.environ['IRIDA_GALAXY_TOOL_TESTS_DONT_START_GALAXY'] - except KeyError: - stop_galaxy() - subprocess.call(self.GALAXY_DB_RESET, shell=True) - subprocess.Popen(self.GALAXY_CMD, cwd=self.GALAXY) - self.log.debug("Waiting for Galaxy database migration [%s]. Sleeping for [%s] seconds", self.GALAXY_URL, - self.GALAXY_SLEEP_TIME) - time.sleep(self.GALAXY_SLEEP_TIME) - self.log.debug("Galaxy database migration should have (hopefully) finished, checking if it is up") - util.wait_until_up( - self.GALAXY_DOMAIN, - self.GALAXY_PORT, - self.TIMEOUT) - self.log.debug("Galaxy should now be up on [%s]", self.GALAXY_URL) - - def finalize_galaxy(): - stop_galaxy() - - request.addfinalizer(finalize_galaxy) - self.register_galaxy(driver) - self.configure_galaxy_api_key(driver) - self.configure_tool('Galaxy', 'galaxy_url', self.GALAXY_URL) - - def test_galaxy_configured(self, setup_galaxy, driver): - """Verify that Galaxy is accessible""" - driver.get(self.GALAXY_URL) - - def test_irida_configured(self, setup_irida, driver): - """Verify that IRIDA is accessible""" - driver.get(self.IRIDA_URL) - - def test_tool_visible(self, setup_galaxy, driver): - """Make sure there is a link to the tool in Galaxy""" - driver.get(self.GALAXY_URL) - driver.find_element_by_xpath("//div[@id='Get Data']/a[span[contains(text(), 'Get Data')]]").click() - assert (driver.find_element_by_xpath("//a[contains(@class, 'irida_import')]")) - - def register_galaxy(self, driver): - """Register with Galaxy, and then attempt to log in""" - driver.get(self.GALAXY_URL) - driver.find_element_by_link_text("Login or Register").click() - driver.find_element_by_id("register-toggle").click() - driver.find_element_by_name("email").send_keys(self.EMAIL) - driver.find_element_by_name("password").send_keys("Password1") - driver.find_element_by_name("confirm").send_keys( - "Password1") - driver.find_element_by_name("username").send_keys("irida-test") - driver.find_element_by_name("create").click() - - try: - driver.get(self.GALAXY_URL) - driver.find_element_by_link_text("Login or Register").click() - driver.find_element_by_name("login").send_keys(self.EMAIL) - driver.find_element_by_name("password").send_keys("Password1") - driver.find_element_by_name("login").click() - except NoSuchElementException: - pass - - def configure_galaxy_api_key(self, driver): - """Make a new Galaxy admin API key and configure the tool to use it""" - gal = galaxy.GalaxyInstance(self.GALAXY_URL, - email=self.EMAIL, - password=self.GALAXY_PASSWORD) - self.configure_tool('Galaxy', 'admin_key', gal.key) - print('key:' + gal.key) - - def configure_tool(self, section, option, value): - """Write tool configuration data""" - config = configparser.ConfigParser() - config.read(self.CONFIG_PATH) - config.set(section, option, value) - with open(self.CONFIG_PATH, 'w') as config_file: - config.write(config_file) - - def register_irida(self, driver): - """Register with IRIDA if neccessary, and then log in""" - driver.get(self.IRIDA_URL) - self.login_irida(driver, 'admin', 'password1') - - # Set a new password if necessary - try: - driver.find_element_by_name( - "password").send_keys(self.IRIDA_PASSWORD) - driver.find_element_by_name( - "confirmPassword").send_keys(self.IRIDA_PASSWORD) - driver.find_element_by_xpath("//button[@type='submit']").click() - except NoSuchElementException: - self.login_irida(driver, self.IRIDA_USER, self.IRIDA_PASSWORD) - - def login_irida(self, driver, username, password): - """Log in to IRIDA (assumes the login page is opened by the driver)""" - try: - driver.find_element_by_name("username").send_keys(username) - driver.find_element_by_name( - "password").send_keys(password) - driver.find_element_by_xpath("//button[@type='submit']").click() - except NoSuchElementException: - # If already logged in - pass - - def add_irida_client_auth_code(self, driver): - driver.get(self.IRIDA_URL + '/clients/create') - driver.find_element_by_id("clientId").send_keys( - self.IRIDA_AUTH_CODE_ID) - driver.find_element_by_id('authorizedGrantTypes').click() - driver.find_element_by_xpath( - "//*[contains(text(), 'authorization_code')]").click() - driver.find_element_by_name("registeredRedirectUri").send_keys(self.IRIDA_REDIRECT_URI) - driver.find_element_by_id("scope_auto_read").click() - driver.find_element_by_id("create-client-submit").click() - - def add_irida_client_password(self, driver): - driver.get(self.IRIDA_URL + '/clients/create') - driver.find_element_by_id("clientId").send_keys(self.IRIDA_PASSWORD_ID) - driver.find_element_by_id("scope_write").click() - driver.find_element_by_id("create-client-submit").click() - - def get_irida_oauth(self, driver): - secret = self.get_irida_secret(driver, self.IRIDA_PASSWORD_ID) - client = LegacyApplicationClient(self.IRIDA_PASSWORD_ID) - irida_oauth = OAuth2Session(client=client) - irida_oauth.fetch_token( - self.IRIDA_TOKEN_ENDPOINT, - username=self.IRIDA_USER, - password=self.IRIDA_PASSWORD, - client_secret=secret) - return irida_oauth - - def get_irida_secret(self, driver, client_id): - """Get an IRIDA client's secret given its client ID """ - driver.get(self.IRIDA_URL + '/clients') - driver.find_element_by_xpath( - "//*[contains(text(), '" + client_id + "')]").click() - secret = driver.find_element_by_id( - 'client-secret').get_attribute('textContent') - return secret - - def configure_irida_client_secret(self, driver): - """Configure the client secret for the tool""" - secret = self.get_irida_secret(driver, self.IRIDA_AUTH_CODE_ID) - # It is assumed that the tests are being run from the repo's tool - # directory: - self.configure_tool('IRIDA', 'client_secret', secret) - - def get_href(self, response, rel): - """From a Requests response from IRIDA, get a href given a rel""" - links = response.json()['resource']['links'] - href = next(link['href'] for link in links if link['rel'] == rel) - return href - - def test_project_samples_import_single_end(self, setup_irida, setup_galaxy, - driver, tmpdir): - """Verify that sequence files can be imported from IRIDA to Galaxy""" - irida = setup_irida - project_name = 'ImportProjectSamples' - project = irida.post(self.IRIDA_PROJECTS, - json={'name': project_name}) - - samples = self.get_href(project, 'project/samples') - sample1 = irida.post(samples, json={'sampleName': 'PS_Sample1', - 'sequencerSampleId': 'PS_1'}) - sequences1 = self.get_href(sample1, 'sample/sequenceFiles') - - # Pytest manages the temporary directory - seq1 = tmpdir.join("seq1.fastq") - seq1.write(self.FASTQ_CONTENTS) - irida.post(sequences1, files={'file': open(str(seq1), - 'rb')}) - - seq2 = tmpdir.join("seq2.fastq") - seq2.write(self.FASTQ_CONTENTS) - irida.post(sequences1, files={'file': open(str(seq2), - 'rb')}) - - sample2 = irida.post(samples, json={'sampleName': 'PS_Sample2', - 'sequencerSampleId': 'PS_2'}) - sequences2 = self.get_href(sample2, 'sample/sequenceFiles') - seq3 = tmpdir.join("seq3.fastq") - seq3.write(self.FASTQ_CONTENTS) - irida.post(sequences2, files={'file': open(str(seq3), - 'rb')}) - - # Export to Galaxy using the button on the dropdown menu - driver.get(self.GALAXY_URL) - history_panel = driver.find_element_by_id('current-history-panel') - initially_succeeded = len(history_panel.find_elements_by_class_name( - 'state-ok')) - driver.find_element_by_xpath("//div[@id='Get Data']/a[span[contains(text(), 'Get Data')]]").click() - driver.find_element_by_xpath("//a[contains(@class, 'irida_import')]").click() - - # Sometimes a login is required - try: - self.login_irida(driver, self.IRIDA_USER, self.IRIDA_PASSWORD) - except NoSuchElementException: - pass - - # Pick the last matching project on this page - driver.find_elements_by_link_text(project_name)[-1].click() - - # These checkbox elements cannot be clicked directly - # Using IDs would complicate running the tests without restarting IRIDA - stale = True - timeout = 0 - while stale: - try: - checkboxes = driver.find_elements_by_xpath( - "//table[contains(@id, 'samplesTable')]/tbody/tr/td[1]/input[@type='checkbox']") - - checkboxes[0].click() - checkboxes[1].click() - - stale = False - except (StaleElementReferenceException, NoSuchElementException): - time.sleep(1) - timeout += 1 - - if timeout == 60: - raise - - driver.find_element_by_id("cart-add-btn").click() - driver.find_element_by_id("cart-show-btn").click() - - email_input = driver.find_element_by_xpath("//form[contains(@class, 'ant-form')]//input[@type='text']") - email_input.clear() - email_input.send_keys(self.EMAIL) - - # Click "Export Samples to Galaxy" button - driver.find_element_by_xpath("//button[span[text()='Export Samples to Galaxy']]").click() - - WebDriverWait(driver, self.WAIT).until( - EC.presence_of_element_located((By.ID, 'current-history-panel')) - ) - time.sleep(120) # Wait for import to complete - history_panel = driver.find_element_by_id('current-history-panel') - succeeded = len(history_panel.find_elements_by_class_name('state-ok')) - assert succeeded - initially_succeeded > 0, \ - "Import did not complete successfully" diff --git a/irida_import/tests/integration/test_irida_importer.py b/irida_import/tests/integration/test_irida_importer.py new file mode 100644 index 00000000..2b198aae --- /dev/null +++ b/irida_import/tests/integration/test_irida_importer.py @@ -0,0 +1,217 @@ +""" +Runs integration tests via selenium +""" +import time +import unittest +import ast +import tempfile +from selenium import webdriver +from selenium.common.exceptions import NoSuchElementException +from selenium.common.exceptions import StaleElementReferenceException +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.support.wait import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from urllib.parse import urljoin +from rauth import OAuth2Service + +import irida_import.tests.integration.tests_integration as tests_integration + + +class IridaImporterTestSuite(unittest.TestCase): + + def setUp(self): + print("\nStarting " + self.__module__ + ": " + self._testMethodName) + self.driver = self._init_driver() + self.irida_session = self._get_irida_oauth_session() + + def tearDown(self): + self.driver.quit() + return + + @staticmethod + def _init_driver(): + """Set up the Selenium WebDriver""" + driver = webdriver.Chrome(tests_integration.chrome_driver_path, options=tests_integration.chrome_driver_options) + driver.implicitly_wait(1) + driver.set_window_size(1024, 768) + + return driver + + @staticmethod + def _get_irida_oauth_session(): + def token_decoder(return_dict): + """ + safely parse given dictionary + + arguments: + return_dict -- access token dictionary + + returns evaluated dictionary + """ + # It is supposedly safer to decode bytes to string and then use ast.literal_eval than just use eval() + try: + irida_dict = ast.literal_eval(return_dict.decode("utf-8")) + except (SyntaxError, ValueError): + # SyntaxError happens when something that looks nothing like a token is returned (ex: 404 page) + # ValueError happens with the path returns something that looks like a token, but is invalid + # (ex: forgetting the /api/ part of the url) + raise ConnectionError("Unexpected response from server, URL may be incorrect") + return irida_dict + + access_token_url = urljoin(tests_integration.irida_base_url, "oauth/token") + oauth_service = OAuth2Service( + client_id=tests_integration.client_id, + client_secret=tests_integration.client_secret, + name="irida", + access_token_url=access_token_url, + base_url=tests_integration.irida_base_url + ) + + params = { + "data": { + "grant_type": "password", + "client_id": tests_integration.client_id, + "client_secret": tests_integration.client_secret, + "username": tests_integration.username, + "password": tests_integration.password + } + } + + access_token = oauth_service.get_access_token( + decoder=token_decoder, **params) + + return oauth_service.get_session(access_token) + + def test_irida_alive(self): + """ + Tests an endpoint on irida to check alive status + :return: + """ + try: + url = f"{tests_integration.irida_base_url}projects" + response = self.irida_session.get(url) + print(response) + except Exception as e: + self.fail("Could not verify that IRIDA is up: " + str(e)) + + def test_galaxy_alive(self): + """ + Tests an endpoint on galaxy to check alive status + :return: + """ + try: + response = self.driver.get(tests_integration.galaxy_url) + print(response) + except Exception as e: + self.fail("Could not verify that IRIDA is up: " + str(e)) + + def test_tool_visible(self): + """Make sure there is a link to the tool in Galaxy""" + self.driver.get(tests_integration.galaxy_url) + self.driver.find_element_by_xpath("//a[span[contains(text(), 'Get Data')]]").click() + self.assertIsNotNone(self.driver.find_element_by_xpath("//a[span[contains(text(), 'IRIDA')]]")) + + def test_project_samples_import_single_end(self): + """Verify that sequence files can be imported from IRIDA to Galaxy""" + + def _get_href(response, rel): + """From a Requests response from IRIDA, get a href given a rel""" + links = response.json()['resource']['links'] + href = next(link['href'] for link in links if link['rel'] == rel) + return href + + def _create_temp_file(file_name, file_contents): + """Create a fake file in memory with a wilename and file contents, returns file path""" + tmp = tempfile.NamedTemporaryFile(delete=False) + path = tmp.name + tmp.name = file_name + tmp.write(str.encode(file_contents)) + return path + + fastq_contents = ( + "@SRR566546.970 HWUSI-EAS1673_11067_FC7070M:4:1:2299:1109 length=50\n" + + "TTGCCTGCCTATCATTTTAGTGCCTGTGAGGTGGAGATGTGAGGATCAGT\n" + + "+SRR566546.970 HWUSI-EAS1673_11067_FC7070M:4:1:2299:1109 length=50\n" + + "hhhhhhhhhhghhghhhhhfhhhhhfffffe`ee[`X]b[d[ed`[Y[^Y") + + project_name = 'ImportProjectSamples' + project = self.irida_session.post(f"{tests_integration.irida_base_url}projects", + json={'name': project_name}) + + samples = _get_href(project, 'project/samples') + sample1 = self.irida_session.post(samples, json={'sampleName': 'PS_Sample1','sequencerSampleId': 'PS_1'}) + sequences1 = _get_href(sample1, 'sample/sequenceFiles') + + seq1 = _create_temp_file("seq1.fastq", fastq_contents) + self.irida_session.post(sequences1, files={'file': open(seq1, 'rb')}) + + seq2 = _create_temp_file("seq2.fastq", fastq_contents) + self.irida_session.post(sequences1, files={'file': open(seq2, 'rb')}) + + sample2 = self.irida_session.post(samples, json={'sampleName': 'PS_Sample2', 'sequencerSampleId': 'PS_2'}) + sequences2 = _get_href(sample2, 'sample/sequenceFiles') + + seq3 = _create_temp_file("seq3.fastq", fastq_contents) + self.irida_session.post(sequences2, files={'file': open(seq3, 'rb')}) + + # Export to Galaxy using the button on the dropdown menu + self.driver.get(tests_integration.galaxy_url) + history_panel = self.driver.find_element_by_id('current-history-panel') + initially_succeeded = len(history_panel.find_elements_by_class_name('state-ok')) + self.driver.find_element_by_xpath("//a[span[contains(text(), 'Get Data')]]").click() + self.driver.find_element_by_xpath("//a[span[contains(text(), 'IRIDA')]]").click() + + # Sometimes a login is required + try: + self.driver.find_element_by_name("username").send_keys(tests_integration.username) + self.driver.find_element_by_name("password").send_keys(tests_integration.password) + self.driver.find_element_by_xpath("//button[@type='submit']").click() + except NoSuchElementException: + # If already logged in + pass + + # Pick the last matching project on this page + self.driver.find_elements_by_link_text(project_name)[-1].click() + + stale = True + timeout = 0 + while stale: + try: + checkboxes = self.driver.find_elements_by_xpath("//input[@type='checkbox']") + # first checkbox is "select all samples checkbox" + checkboxes[0].click() + stale = False + except (StaleElementReferenceException, NoSuchElementException): + time.sleep(1) + timeout += 1 + + if timeout == 60: + raise + + self.driver.find_elements_by_class_name("t-add-cart-btn")[0].click() + time.sleep(10) # wait 10 seconds for popup to disappear + self.driver.find_element_by_xpath("//a[@href='/cart/galaxy']").click() + + email_input = self.driver.find_element_by_xpath("//form[contains(@class, 'ant-form')]//input[@type='text']") + email_input.send_keys(Keys.CONTROL, "a") + email_input.send_keys(tests_integration.galaxy_email) + + # Click "Export Samples to Galaxy" button + self.driver.find_element_by_xpath("//button[span[text()='Export Samples to Galaxy']]").click() + + # handle auth confirmation popup + if len(self.driver.window_handles) > 1: + self.driver.switch_to.window(self.driver.window_handles[1]) + self.driver.find_element_by_id('authorize-btn').click() + self.driver.switch_to.window(self.driver.window_handles[0]) + + WebDriverWait(self.driver, 120).until( + EC.presence_of_element_located((By.ID, 'current-history-panel')) + ) + time.sleep(120) # Wait for import to complete + history_panel = self.driver.find_element_by_id('current-history-panel') + succeeded = len(history_panel.find_elements_by_class_name('state-ok')) + # This currently fails, as the tool is broken because of this following issue + # https://github.com/galaxyproject/galaxy/issues/11066 + self.assertTrue(succeeded - initially_succeeded > 0, "Import did not complete successfully") diff --git a/irida_import/tests/integration/tests_integration.py b/irida_import/tests/integration/tests_integration.py new file mode 100644 index 00000000..2bc844ce --- /dev/null +++ b/irida_import/tests/integration/tests_integration.py @@ -0,0 +1,142 @@ +""" +This file is responsible for managing the execution of an IRIDA instance and Test Suites +""" +import unittest +from os import path + +from webdriver_manager.chrome import ChromeDriverManager +from webdriver_manager.core.utils import ChromeType +from selenium.webdriver.chrome.options import Options as ChromeDriverOptions + +from irida_import.tests.integration.irida_data_setup import SetupIridaData +from irida_import.tests.integration.galaxy_data_setup import SetupGalaxyData +from irida_import.tests.integration.test_irida_importer import IridaImporterTestSuite + +# Modules level variables that can/will be changed when the setup starts +irida_base_url = "http://localhost:8080/api/" +username = "jeff" +password = "password1" +client_id = "myClient" +client_secret = "myClientSecret" + +galaxy_url = None +galaxy_email = 'irida@irida.ca' + +# Deal with 'AF_UNIX path too long' errors +repo_path = path.join('/tmp', 'repos') + +# Have ChromeDriverManager handle chrome installation and driver for compatibility with github actions +chrome_driver_path = ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install() +chrome_driver_options = ChromeDriverOptions() +# When creating new tests, these can be commented out to have driver window display +chrome_driver_options.add_argument('--headless') +chrome_driver_options.add_argument('--disable-gpu') + + +def init_irida_setup(branch, db_host, db_port): + """ + Initializes the Irida setup object + :param branch: what branch from github to check out + :param db_host: database host override + :param db_port: database port override + :return: SetupIridaData object + """ + global irida_base_url + global username + global password + global client_id + global client_secret + + return SetupIridaData(irida_base_url[:irida_base_url.index("/api")], + username, password, branch, db_host, db_port, repo_dir=repo_path) + + +def init_galaxy_setup(): + """ + Initializes the Galaxy setup object + :return: + """ + return SetupGalaxyData( + repo_dir=repo_path, + email=galaxy_email, + chrome_driver_path=chrome_driver_path, + chrome_driver_options=chrome_driver_options, + ) + + +def create_test_suite(): + """ + Finds all the integration tests and adds them to a test suite + :return: unittest.TestSuite + """ + suite = unittest.TestSuite() + + # Include all the classes that contain tests + test_class_list = [IridaImporterTestSuite] + + # for each class, find the tests and add them to the test suite + for test_class in test_class_list: + for class_method in [*test_class.__dict__]: + if class_method.startswith("test_"): + suite.addTest(test_class(class_method)) + + return suite + + +def start(irida_branch="master", db_host="localhost", db_port="3306"): + """ + Start running the integration tests + + This is the entry point for the integration tests + :param irida_branch: what branch from github to check out + :param db_host: database host override + :param db_port: database port override + :return: + """ + exit_code = 0 + + # create a handler to manage a headless IRIDA instance + irida_handler = init_irida_setup(irida_branch, db_host, db_port) + + # Create handler for Galaxy + galaxy_handler = init_galaxy_setup() + global galaxy_url + galaxy_url = galaxy_handler.GALAXY_URL + + try: + # Install IRIDA packages + irida_handler.install_irida() + # Delete and recreate the database + irida_handler.reset_irida_db() + # Launch IRIDA + # Note: This initializes the database tables + # Note: This call waits until IRIDA is running + irida_handler.run_irida() + # Add data to database that the tests depend on + irida_handler.update_irida_db() + + # install an instance of galaxy + galaxy_handler.install_galaxy() + # start galaxy + galaxy_handler.start_galaxy() + + # configure galaxy post startup + galaxy_handler.register_galaxy() + galaxy_handler.configure_galaxy_api_key() + galaxy_handler.configure_irida_galaxy_connection(galaxy_url) + + # Run tests + full_suite = create_test_suite() + + runner = unittest.TextTestRunner() + res = runner.run(full_suite) + if not res.wasSuccessful(): + exit_code = 1 + except Exception as e: + raise e + finally: + # Make sure IRIDA and Galaxy are stopped even when an exception is raised + irida_handler.stop_irida() + galaxy_handler.stop_galaxy() + + return exit_code diff --git a/irida_import/tests/integration/util.py b/irida_import/tests/integration/util.py deleted file mode 100644 index 96661ac8..00000000 --- a/irida_import/tests/integration/util.py +++ /dev/null @@ -1,40 +0,0 @@ -import time -import socket - - -def wait_until_up(address, port, timeout): - """ - Wait until a port at an address is occupied, or time out - - :type address: str - :param address: e.g. 'localhost' or '127.0.0.1' - :type port: int - :param port: e.g. 8888 - :type timeout: int - :param timeout: the time to wait in seconds until throwing an exception - """ - max_time = time.time() + timeout - up = False - while not up and time.time() < max_time: - up = check_up(address, port) - - # If we query Galaxy immediately it may reset the connection: - time.sleep(10) - - if not up: - raise Exception('There was no response at {} on port {} for {} seconds' - .format(address, port, timeout)) - - -def check_up(address, port): - """ - Find out if a port at an address is occupied - """ - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((address, port)) - sock.close() - if result == 0: - ans = True - else: - ans = False - return ans diff --git a/irida_import/tests/unit/test_irida_import.py b/irida_import/tests/unit/test_irida_import.py index 635c14ec..92163c7a 100644 --- a/irida_import/tests/unit/test_irida_import.py +++ b/irida_import/tests/unit/test_irida_import.py @@ -5,10 +5,10 @@ import logging import pprint import pytest -import mock +import unittest.mock as mock from requests_oauthlib import OAuth2Session -from mock import Mock +from unittest.mock import Mock from bioblend import galaxy from bioblend.galaxy.objects import (GalaxyInstance, Library, Folder, client) from bioblend.galaxy.objects.wrappers import LibraryContentInfo @@ -47,7 +47,7 @@ def setup_logging(self): def setup_json(self): """Create a json string from a text file""" logging.debug("Opening a test json string") - test_path = 'tests/unit/data/test.dat' + test_path = 'irida_import/tests/unit/data/test.dat' # Pytest messes up the paths to modules and packages. # I haven't found a way to get around it without hardcoding paths. # Reading a file is neccessary to avoid writing large PEP8 commpliant @@ -379,9 +379,9 @@ def test_assign_ownership_if_nec(self, imp): # TODO: write the functionality for this to test return True - def test_import_to_galaxy(self, setup_json, mocker): + def test_import_to_galaxy(self, setup_json): """Test reading a file and running apropriate methods""" - mocker.patch('bioblend.galaxy.objects.GalaxyInstance', autospec=True) + mock.patch('bioblend.galaxy.objects.GalaxyInstance', autospec=True) mocked_open_function = mock.mock_open(read_data=setup_json) open_function_name = "builtins.open" diff --git a/run-tests.sh b/run-tests.sh deleted file mode 100755 index 1e7c2789..00000000 --- a/run-tests.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -cd `dirname $0` -SCRIPT_DIR=`pwd` - -CHROMEDRIVER_VERSION='' - -PYTHON_VERSION=${PYTHON_VERSION:-3.6.7} - -exit_error() { - echo $1 - exit 1 -} - -check_dependencies() { - mvn --version 1>/dev/null 2>/dev/null - if [ $? -ne 0 ]; - then - exit_error "Command 'mvn' does not exist. Please install Maven (e.g., 'apt-get install maven') to continue." - fi - - mysql --version 1>/dev/null 2>/dev/null - if [ $? -ne 0 ]; - then - exit_error "Command 'mysql' does not exist. Please install MySQL/MariaDB (e.g., 'apt-get install mariadb-client mariadb-server') to continue." - fi - - psql --version 1>/dev/null 2>/dev/null - if [ $? -ne 0 ]; - then - exit_error "Command 'psql' does not exist. Please install PostgreSQL (e.g., 'apt-get install postgresql') to continue." - fi - - git --version 1>/dev/null 2>/dev/null - if [ $? -ne 0 ]; - then - exit_error "Command 'git' does not exist. Please install git (e.g., 'apt-get install git') to continue." - fi - - chromedriver --version 1>/dev/null 2>/dev/null - if [ $? -ne 0 ]; - then - exit_error "Command 'chromedriver' does not exist. Please install Chromedriver (e.g., 'apt-get install chromium-chromedriver') to continue." - else - CHROMEDRIVER_VERSION=`chromedriver --version | sed -e 's/^ChromeDriver //' -e 's/ (.*//' 2>/dev/null` - fi - - xvfb-run -h 1>/dev/null 2>/dev/null - if [ $? -ne 0 ]; - then - exit_error "Command 'xvfb' does not exist. Please install xvfb (e.g., 'apt-get install xvfb') to continue." - fi -} - -################# -#### MAIN ####### -################# - -check_dependencies - -source $SCRIPT_DIR/.ci/install_deps.sh $CHROMEDRIVER_VERSION $PYTHON_VERSION - -pushd irida_import - -exit_code=1 - -case "$1" in - integration) - # If xvfb is already started, don't bother with 'xvfb-run'. Useful for TravisCI tests. - if [ "$XVFB_STARTED" = "1" ]; - then - pytest tests/integration/*.py - else - xvfb-run pytest tests/integration/*.py - fi - exit_code=$? - ;; - unit) - pytest tests/unit/*.py - exit_code=$? - ;; - *) - # If xvfb is already started, don't bother with 'xvfb-run'. Useful for TravisCI tests. - if [ "$XVFB_STARTED" = "1" ]; - then - py.test -s - else - xvfb-run py.test -s - fi - exit_code=$? - ;; -esac - -exit $exit_code \ No newline at end of file diff --git a/setup.py b/setup.py index db6a7047..41feaffc 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ def readme(): with open('README.md') as f: return f.read() + setup(name='irida_import', version='2.0.0', description='A tool for importing data from IRIDA into Galaxy', @@ -38,6 +39,19 @@ def readme(): 'requests-oauthlib', 'simplejson' ], + extras_require={ + "TEST": [ + "pytest", + "pytest-cov", + "pytest-mock", + "mock", + "subprocess32", + "selenium", + "rauth", + "argparse", + "webdriver-manager", + ], + }, zip_safe=False, classifiers=[ 'License :: OSI Approved :: Apache Software License', From ee1ba446ceac928aab53311a1f070972e24f4ca0 Mon Sep 17 00:00:00 2001 From: Jeff Thiessen Date: Fri, 9 Sep 2022 14:01:27 -0500 Subject: [PATCH 2/7] switch integration tests to launch irida via gradle (#27) * switch integration tests to launch irida via gradle * update readme and changelog * pin selenium so tests stop breaking every update --- CHANGELOG.md | 1 + README.md | 4 ++-- .../integration/bash_scripts/install_irida.sh | 8 +------ .../tests/integration/irida_data_setup.py | 23 +++++++++++-------- setup.py | 2 +- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7ebd641..53ce1333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to irida-galaxy-importer will be documeted in this file. ## 2.0.1 * Switched from travisCI tests to Github Actions +* Switched integration tests to launch irida via gradle ## 2.0.0 diff --git a/README.md b/README.md index 952cb83c..ce4e53e7 100644 --- a/README.md +++ b/README.md @@ -246,7 +246,7 @@ If you wish to make additions to the code, the below instructions can be used to The script `run-tests.sh` can be used to run the tests. This should check for some of the dependencies and let you know which is missing. However, you will have to have the following dependencies installed: * Java 11 -* Maven +* Gradle * MySQL/MariaDB (Server and Client) * PostgreSQL (Server and Client) * Git @@ -256,7 +256,7 @@ The script `run-tests.sh` can be used to run the tests. This should check for so On Ubuntu, you can install these with: ```bash -sudo apt-get install openjdk-11-jdk maven mariadb-client mariadb-server postgresql git chromium-chromedriver xvfb +sudo apt-get install openjdk-11-jdk gradle mariadb-client mariadb-server postgresql git chromium-chromedriver xvfb ``` MySQL must be configured to grant all privileges to the user `test` with password `test` for the databases `irida_test`. MySQL must also be configured to disable `ONLY_FULL_GROUP_BY` mode. diff --git a/irida_import/tests/integration/bash_scripts/install_irida.sh b/irida_import/tests/integration/bash_scripts/install_irida.sh index 7683ad0a..47f0ead9 100755 --- a/irida_import/tests/integration/bash_scripts/install_irida.sh +++ b/irida_import/tests/integration/bash_scripts/install_irida.sh @@ -1,4 +1,5 @@ #!/bin/bash +# This file creates a repos directory and pulls the irida branch with the name in $1 pushd /tmp/repos @@ -8,12 +9,5 @@ then echo >&2 "Failed to clone" exit 1 else - pushd irida - echo "Preparing IRIDA for first excecution..." - - pushd lib - ./install-libs.sh - popd - popd echo "IRIDA has been installed" fi diff --git a/irida_import/tests/integration/irida_data_setup.py b/irida_import/tests/integration/irida_data_setup.py index 8137085f..9eb99d6d 100644 --- a/irida_import/tests/integration/irida_data_setup.py +++ b/irida_import/tests/integration/irida_data_setup.py @@ -58,8 +58,8 @@ def __init__(self, base_url, user, password, branch, db_host, db_port, repo_dir) self.DB_NAME + ' < {}'.format(sql_file) self.IRIDA_CMD = \ - 'mvn clean compile spring-boot:start -DskipTests --quiet ' +\ - '-Dspring-boot.run.arguments=\"' +\ + './gradlew -q clean bootRun -x check ' +\ + '--args=\"' +\ '--spring.datasource.url={} '.format(self.DB_JDBC_URL) +\ '--spring.datasource.username={} '.format(self.DB_USERNAME) +\ '--spring.datasource.password={} '.format(self.DB_PASSWORD) +\ @@ -73,8 +73,6 @@ def __init__(self, base_url, user, password, branch, db_host, db_port, repo_dir) '--logging.pattern.console=' +\ '\"' - self.IRIDA_STOP = 'mvn spring-boot:stop' - self.PATH_TO_MODULE = path.dirname(__file__) if len(self.PATH_TO_MODULE) == 0: self.PATH_TO_MODULE = "." @@ -86,6 +84,9 @@ def __init__(self, base_url, user, password, branch, db_host, db_port, repo_dir) self.REPO_PATH = repo_dir self.IRIDA_PATH = path.join(self.REPO_PATH, "irida") + # Handles the irida subprocess + self.irida_subprocess = None + def install_irida(self): """ Installs IRIDA @@ -127,8 +128,11 @@ def run_irida(self): Waits until IRIDA is up to return :return: """ - subprocess.Popen( - self.IRIDA_CMD, cwd=self.IRIDA_PATH, shell=True) + self.irida_subprocess = subprocess.Popen( + "exec " + self.IRIDA_CMD, # assign new process image + stdout=subprocess.PIPE, # makes subprocess.Popen return pid of command, and not shell (default behaviour) + cwd=self.IRIDA_PATH, + shell=True) self._wait_until_up() def _wait_until_up(self): @@ -153,10 +157,9 @@ def _wait_until_up(self): def stop_irida(self): """ - Stops the IRIDA mvn process + Kill the IRIDA gradlew process This will sometimes dump errors into the log, but it is harmless :return: """ - stopper = subprocess.Popen( - self.IRIDA_STOP, cwd=self.IRIDA_PATH, shell=True) - stopper.wait() + self.irida_subprocess.kill() + self.irida_subprocess = None diff --git a/setup.py b/setup.py index 41feaffc..fc7512fa 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def readme(): "pytest-mock", "mock", "subprocess32", - "selenium", + "selenium==4.2.0", "rauth", "argparse", "webdriver-manager", From a4c0345cb0bda211a82a31d0a0afa9a717684803 Mon Sep 17 00:00:00 2001 From: Jeffrey Thiessen Date: Tue, 13 Sep 2022 18:04:24 -0500 Subject: [PATCH 3/7] trying 127.0.0.1 over localhost, and pinning galaxy version --- README.md | 4 ++-- irida_import/extras/apache2/index.html | 6 +++--- irida_import/extras/nginx/nginx.conf | 2 +- irida_import/irida_import.xml.sample | 2 +- irida_import/sample.json | 2 +- .../tests/integration/bash_scripts/install_galaxy.sh | 6 +++--- irida_import/tests/integration/create_client.sql | 2 +- irida_import/tests/integration/galaxy_data_setup.py | 4 ++-- irida_import/tests/integration/test_irida_importer.py | 2 +- irida_import/tests/integration/tests_integration.py | 3 ++- irida_import/tests/unit/data/test.dat | 2 +- irida_import/tests/unit/test_irida_import.py | 4 ++-- 12 files changed, 20 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index ce4e53e7..d7c51331 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ You will next want to make the appropriate changes with your connection informat ### MODIFY THESE ### admin_key: d9c54f0b38b75dd6513035e4dd786a0b -galaxy_url: http://localhost:48888 +galaxy_url: http://127.0.0.1:48888 #################### illumina_path: /illumina_reads @@ -196,7 +196,7 @@ client_http_retry_delay: 30 ### MODIFY THESE ### client_secret: qlB82t7Ct917127lL7oQ82bd9o2iAP8bT0rJohpz7s client_id: galaxy -irida_url: http://localhost:8080 +irida_url: http://127.0.0.1:8080 #################### initial_endpoint_suffix: /projects diff --git a/irida_import/extras/apache2/index.html b/irida_import/extras/apache2/index.html index 07acc8ba..6a92e231 100644 --- a/irida_import/extras/apache2/index.html +++ b/irida_import/extras/apache2/index.html @@ -27,7 +27,7 @@ }).config(function($sceDelegateProvider) { $sceDelegateProvider.resourceUrlWhitelist([ 'self', - 'http://localhost/**' + 'http://127.0.0.1/**' ]); }).controller('galaxyController', ['$scope', '$http', '$window', '$location','$timeout', galaxyController]); @@ -73,9 +73,9 @@ sampleFilePaths = [ "http://www.dgfdg.com/sample1file1", - "file://localhost/home/jthiessen/lib_imp_dir/test/test.fastq" + "file://127.0.0.1/home/jthiessen/lib_imp_dir/test/test.fastq" ]; - addSample("thisissample1'sname","http://localhost/some_IRIDA_API_path/Projects/1/Samples/1",sampleFilePaths); + addSample("thisissample1'sname","http://127.0.0.1/some_IRIDA_API_path/Projects/1/Samples/1",sampleFilePaths); $timeout( function(){ document.getElementById("samplesSubmit").submit(); diff --git a/irida_import/extras/nginx/nginx.conf b/irida_import/extras/nginx/nginx.conf index 4d9cb7e1..c35b8e34 100644 --- a/irida_import/extras/nginx/nginx.conf +++ b/irida_import/extras/nginx/nginx.conf @@ -2,7 +2,7 @@ events {} http { upstream galaxy_app { - server localhost:8888; + server 127.0.0.1:8888; } server { diff --git a/irida_import/irida_import.xml.sample b/irida_import/irida_import.xml.sample index 6c82ac14..7f140d9d 100644 --- a/irida_import/irida_import.xml.sample +++ b/irida_import/irida_import.xml.sample @@ -7,7 +7,7 @@ --log-file $log_file --history-id "$__app__.security.encode_id($output.history.id)" ]]> - + import data from IRIDA to Galaxy diff --git a/irida_import/sample.json b/irida_import/sample.json index b8b73150..21375d60 100644 --- a/irida_import/sample.json +++ b/irida_import/sample.json @@ -1 +1 @@ -{"param_dict": {"__new_file_path__": "/home/jthiessen/galaxy-dist/database/tmp", "runtool_btn": "Execute", "__user_id__": "1", "output": "/home/jthiessen/galaxy-dist/database/files/000/dataset_992.dat", "GALAXY_ROOT_DIR": "/home/jthiessen/galaxy-dist", "chromInfo": "/home/jthiessen/galaxy-dist/tool-data/shared/ucsc/chrom/?.len", "__tool_data_path__": "/home/jthiessen/galaxy-dist/tool-data", "userId": "1", "userEmail": "joel.thiessen@phac-aspc.gc.ca", "dbkey": "?", "__user_name__": "jthiessen", "__datatypes_config__": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp", "__app__": "galaxy.app:UniverseApplication", "__user__": "galaxy.model:User", "GALAXY_DATA_INDEX_DIR": "/home/jthiessen/galaxy-dist/tool-data", "__root_dir__": "/home/jthiessen/galaxy-dist", "__user_email__": "joel.thiessen@phac-aspc.gc.ca", "__admin_users__": "joel.thiessen@phac-aspc.gc.ca", "json_params": "{\"_embedded\":{\"library\":{\"name\":\"Project 5-fbristow\"},\"user\":{\"email\":\"franklin.bristow@phac-aspc.gc.ca\"},\"oauth2\":{\"code\":\"4DEPzF\",\"redirect\":\"/galaxy/auth_code\"},\"samples\":[{\"name\":\"01-1111\",\"_links\":{\"self\":{\"href\":\"\"}},\"_embedded\":{\"sample_files\":[{\"_links\":{\"self\":{\"href\":\"http://localhost:8080/api/projects/4/samples/51/sequenceFiles/3\"}}},{\"_links\":{\"self\":{\"href\":\"http://localhost:8080/api/projects/4/samples/51/sequenceFiles/4\"}}},{\"_links\":{\"self\":{\"href\":\"http://localhost:8080/api/projects/4/samples/51/sequenceFiles/9\"}}},{\"_links\":{\"self\":{\"href\":\"http://localhost:8080/api/projects/4/samples/51/sequenceFiles/10\"}}}]}}]}}", "__get_data_table_entry__": "", "GALAXY_DATATYPES_CONF_FILE": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp"}, "output_data": [{"extra_files_path": "/home/jthiessen/galaxy-dist/database/job_working_directory/000/992/dataset_992_files", "ext": "auto", "out_data_name": "output", "hda_id": 801, "file_name": "/home/jthiessen/galaxy-dist/database/files/000/dataset_992.dat", "dataset_id": 992}], "job_config": {"GALAXY_ROOT_DIR": "/home/jthiessen/galaxy-dist", "TOOL_PROVIDED_JOB_METADATA_FILE": "galaxy.json", "GALAXY_DATATYPES_CONF_FILE": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp"}} \ No newline at end of file +{"param_dict": {"__new_file_path__": "/home/jthiessen/galaxy-dist/database/tmp", "runtool_btn": "Execute", "__user_id__": "1", "output": "/home/jthiessen/galaxy-dist/database/files/000/dataset_992.dat", "GALAXY_ROOT_DIR": "/home/jthiessen/galaxy-dist", "chromInfo": "/home/jthiessen/galaxy-dist/tool-data/shared/ucsc/chrom/?.len", "__tool_data_path__": "/home/jthiessen/galaxy-dist/tool-data", "userId": "1", "userEmail": "joel.thiessen@phac-aspc.gc.ca", "dbkey": "?", "__user_name__": "jthiessen", "__datatypes_config__": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp", "__app__": "galaxy.app:UniverseApplication", "__user__": "galaxy.model:User", "GALAXY_DATA_INDEX_DIR": "/home/jthiessen/galaxy-dist/tool-data", "__root_dir__": "/home/jthiessen/galaxy-dist", "__user_email__": "joel.thiessen@phac-aspc.gc.ca", "__admin_users__": "joel.thiessen@phac-aspc.gc.ca", "json_params": "{\"_embedded\":{\"library\":{\"name\":\"Project 5-fbristow\"},\"user\":{\"email\":\"franklin.bristow@phac-aspc.gc.ca\"},\"oauth2\":{\"code\":\"4DEPzF\",\"redirect\":\"/galaxy/auth_code\"},\"samples\":[{\"name\":\"01-1111\",\"_links\":{\"self\":{\"href\":\"\"}},\"_embedded\":{\"sample_files\":[{\"_links\":{\"self\":{\"href\":\"http://127.0.0.1:8080/api/projects/4/samples/51/sequenceFiles/3\"}}},{\"_links\":{\"self\":{\"href\":\"http://127.0.0.1:8080/api/projects/4/samples/51/sequenceFiles/4\"}}},{\"_links\":{\"self\":{\"href\":\"http://127.0.0.1:8080/api/projects/4/samples/51/sequenceFiles/9\"}}},{\"_links\":{\"self\":{\"href\":\"http://127.0.0.1:8080/api/projects/4/samples/51/sequenceFiles/10\"}}}]}}]}}", "__get_data_table_entry__": "", "GALAXY_DATATYPES_CONF_FILE": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp"}, "output_data": [{"extra_files_path": "/home/jthiessen/galaxy-dist/database/job_working_directory/000/992/dataset_992_files", "ext": "auto", "out_data_name": "output", "hda_id": 801, "file_name": "/home/jthiessen/galaxy-dist/database/files/000/dataset_992.dat", "dataset_id": 992}], "job_config": {"GALAXY_ROOT_DIR": "/home/jthiessen/galaxy-dist", "TOOL_PROVIDED_JOB_METADATA_FILE": "galaxy.json", "GALAXY_DATATYPES_CONF_FILE": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp"}} \ No newline at end of file diff --git a/irida_import/tests/integration/bash_scripts/install_galaxy.sh b/irida_import/tests/integration/bash_scripts/install_galaxy.sh index 931d492a..31517d09 100755 --- a/irida_import/tests/integration/bash_scripts/install_galaxy.sh +++ b/irida_import/tests/integration/bash_scripts/install_galaxy.sh @@ -9,7 +9,7 @@ pushd /tmp/repos echo "Downloading Galaxy..." git clone https://github.com/galaxyproject/galaxy/ > galaxy-clone.log 2>&1 pushd galaxy -git checkout master > galaxy-checkout.log 2>&1 +git checkout v22.01 > galaxy-checkout.log 2>&1 git fetch git reset --hard git clean -fd @@ -40,7 +40,7 @@ echo " database_connection: postgresql:///galaxy_test" | cat >> galaxy.yml sed -i 's/#admin_users:.*/admin_users: "irida@irida.ca"/' galaxy.yml # run galaxy on port 8888 instead of 8080; Tomcat runs on 8080 by default. -sed -i "s|# bind: localhost:8080|bind: localhost:$galaxy_port|" galaxy.yml +sed -i "s|# bind: localhost:8080|bind: 127.0.0.1:$galaxy_port|" galaxy.yml popd popd @@ -54,7 +54,7 @@ echo "Initializing the tool's configuration file." pushd galaxy/tools/irida-galaxy-importer/ cp irida_import/config.ini.sample irida_import/config.ini sed -i "s/^max_waits: .*$/max_waits: 1/" irida_import/config.ini -sed -i "s|^galaxy_url: http://localhost:8888$|galaxy_url: http://localhost:$galaxy_port|" irida_import/config.ini +sed -i "s|^galaxy_url: http://localhost:8888$|galaxy_url: http://127.0.0.1:$galaxy_port|" irida_import/config.ini echo "Configuring the tool's XML file" python -m irida_import.main --generate_xml diff --git a/irida_import/tests/integration/create_client.sql b/irida_import/tests/integration/create_client.sql index be68909f..6b9d0e9a 100644 --- a/irida_import/tests/integration/create_client.sql +++ b/irida_import/tests/integration/create_client.sql @@ -4,7 +4,7 @@ INSERT INTO client_details_grant_types (client_details_id,grant_value) VALUES (1 INSERT INTO client_details_scope (client_details_id,scope) VALUES (1,"read"); INSERT INTO client_details_scope (client_details_id,scope) VALUES (1,"write"); -- galaxy auth connection -INSERT INTO client_details (id,clientId,clientSecret,token_validity,refresh_validity,createdDate,modifiedDate,redirect_uri) VALUES (2,"auth_code_client","auth_code_secret",100000,2592000,now(),now(),"http://localhost:8080/galaxy/auth_code"); +INSERT INTO client_details (id,clientId,clientSecret,token_validity,refresh_validity,createdDate,modifiedDate,redirect_uri) VALUES (2,"auth_code_client","auth_code_secret",100000,2592000,now(),now(),"http://127.0.0.1:8080/galaxy/auth_code"); INSERT INTO client_details_grant_types (client_details_id,grant_value) VALUES (2,"authorization_code"); INSERT INTO client_details_scope (client_details_id,scope) VALUES (2,"read"); diff --git a/irida_import/tests/integration/galaxy_data_setup.py b/irida_import/tests/integration/galaxy_data_setup.py index 13234dd0..a805cccb 100644 --- a/irida_import/tests/integration/galaxy_data_setup.py +++ b/irida_import/tests/integration/galaxy_data_setup.py @@ -40,7 +40,7 @@ def __init__(self, repo_dir, email, chrome_driver_path, chrome_driver_options): 'tools', 'irida-galaxy-importer', 'irida_import', 'config.ini') self.INSTALL_EXEC = 'install_galaxy.sh' - self.GALAXY_DOMAIN = 'localhost' + self.GALAXY_DOMAIN = '127.0.0.1' self.GALAXY_PORT = 8888 self.GALAXY_URL = 'http://' + self.GALAXY_DOMAIN + ':' + str( self.GALAXY_PORT) @@ -113,7 +113,7 @@ def _wait_until_up(self, address, port, timeout): Wait until a port at an address is occupied, or time out :type address: str - :param address: e.g. 'localhost' or '127.0.0.1' + :param address: e.g. '127.0.0.1' :type port: int :param port: e.g. 8888 :type timeout: int diff --git a/irida_import/tests/integration/test_irida_importer.py b/irida_import/tests/integration/test_irida_importer.py index 2b198aae..2e4f17c5 100644 --- a/irida_import/tests/integration/test_irida_importer.py +++ b/irida_import/tests/integration/test_irida_importer.py @@ -154,7 +154,7 @@ def _create_temp_file(file_name, file_contents): seq3 = _create_temp_file("seq3.fastq", fastq_contents) self.irida_session.post(sequences2, files={'file': open(seq3, 'rb')}) - + import pdb; pdb.set_trace() # Export to Galaxy using the button on the dropdown menu self.driver.get(tests_integration.galaxy_url) history_panel = self.driver.find_element_by_id('current-history-panel') diff --git a/irida_import/tests/integration/tests_integration.py b/irida_import/tests/integration/tests_integration.py index 2bc844ce..d2639795 100644 --- a/irida_import/tests/integration/tests_integration.py +++ b/irida_import/tests/integration/tests_integration.py @@ -13,7 +13,7 @@ from irida_import.tests.integration.test_irida_importer import IridaImporterTestSuite # Modules level variables that can/will be changed when the setup starts -irida_base_url = "http://localhost:8080/api/" +irida_base_url = "http://127.0.0.1:8080/api/" username = "jeff" password = "password1" client_id = "myClient" @@ -125,6 +125,7 @@ def start(irida_branch="master", db_host="localhost", db_port="3306"): galaxy_handler.configure_galaxy_api_key() galaxy_handler.configure_irida_galaxy_connection(galaxy_url) + # import pdb;pdb.set_trace() # Run tests full_suite = create_test_suite() diff --git a/irida_import/tests/unit/data/test.dat b/irida_import/tests/unit/data/test.dat index 6da657d6..e7c3d9ef 100644 --- a/irida_import/tests/unit/data/test.dat +++ b/irida_import/tests/unit/data/test.dat @@ -1 +1 @@ -{"param_dict": {"__new_file_path__": "/home/jthiessen/galaxy-dist/database/tmp", "runtool_btn": "Execute", "__user_id__": "1", "output": "/home/jthiessen/galaxy-dist/database/files/000/dataset_992.dat", "GALAXY_ROOT_DIR": "/home/jthiessen/galaxy-dist", "chromInfo": "/home/jthiessen/galaxy-dist/tool-data/shared/ucsc/chrom/?.len", "__tool_data_path__": "/home/jthiessen/galaxy-dist/tool-data", "userId": "1", "userEmail": "joel.thiessen@phac-aspc.gc.ca", "dbkey": "?", "__user_name__": "jthiessen", "__datatypes_config__": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp", "__app__": "galaxy.app:UniverseApplication", "__user__": "galaxy.model:User", "GALAXY_DATA_INDEX_DIR": "/home/jthiessen/galaxy-dist/tool-data", "__root_dir__": "/home/jthiessen/galaxy-dist", "__user_email__": "joel.thiessen@phac-aspc.gc.ca", "__admin_users__": "joel.thiessen@phac-aspc.gc.ca", "json_params": "{\"_embedded\":{\"library\":{\"name\":\"Project 5-fbristow\"},\"user\":{\"email\":\"franklin.bristow@phac-aspc.gc.ca\"},\"addtohistory\":true,\"includeAssemblies\":false,\"makepairedcollection\":true,\"oauth2\":{\"code\":\"4DEPzF\",\"redirect\":\"/galaxy/auth_code\"},\"samples\":[{\"name\":\"01-1111\",\"_links\":{\"self\":{\"href\":\"\"}},\"_embedded\":{\"sample_files\":[{\"_links\":{\"self\":{\"href\":\"http://localhost:8080/api/projects/4/samples/51/sequenceFiles/3\"}}},{\"_links\":{\"self\":{\"href\":\"http://localhost:8080/api/projects/4/samples/51/sequenceFiles/4\"}}},{\"_links\":{\"self\":{\"href\":\"http://localhost:8080/api/projects/4/samples/51/sequenceFiles/9\"}}},{\"_links\":{\"self\":{\"href\":\"http://localhost:8080/api/projects/4/samples/51/sequenceFiles/10\"}}}]}}]}}", "__get_data_table_entry__": "", "GALAXY_DATATYPES_CONF_FILE": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp"}, "output_data": [{"extra_files_path": "/home/jthiessen/galaxy-dist/database/job_working_directory/000/992/dataset_992_files", "ext": "auto", "out_data_name": "output", "hda_id": 801, "file_name": "/home/jthiessen/galaxy-dist/database/files/000/dataset_992.dat", "dataset_id": 992}], "job_config": {"GALAXY_ROOT_DIR": "/home/jthiessen/galaxy-dist", "TOOL_PROVIDED_JOB_METADATA_FILE": "galaxy.json", "GALAXY_DATATYPES_CONF_FILE": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp"}} +{"param_dict": {"__new_file_path__": "/home/jthiessen/galaxy-dist/database/tmp", "runtool_btn": "Execute", "__user_id__": "1", "output": "/home/jthiessen/galaxy-dist/database/files/000/dataset_992.dat", "GALAXY_ROOT_DIR": "/home/jthiessen/galaxy-dist", "chromInfo": "/home/jthiessen/galaxy-dist/tool-data/shared/ucsc/chrom/?.len", "__tool_data_path__": "/home/jthiessen/galaxy-dist/tool-data", "userId": "1", "userEmail": "joel.thiessen@phac-aspc.gc.ca", "dbkey": "?", "__user_name__": "jthiessen", "__datatypes_config__": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp", "__app__": "galaxy.app:UniverseApplication", "__user__": "galaxy.model:User", "GALAXY_DATA_INDEX_DIR": "/home/jthiessen/galaxy-dist/tool-data", "__root_dir__": "/home/jthiessen/galaxy-dist", "__user_email__": "joel.thiessen@phac-aspc.gc.ca", "__admin_users__": "joel.thiessen@phac-aspc.gc.ca", "json_params": "{\"_embedded\":{\"library\":{\"name\":\"Project 5-fbristow\"},\"user\":{\"email\":\"franklin.bristow@phac-aspc.gc.ca\"},\"addtohistory\":true,\"includeAssemblies\":false,\"makepairedcollection\":true,\"oauth2\":{\"code\":\"4DEPzF\",\"redirect\":\"/galaxy/auth_code\"},\"samples\":[{\"name\":\"01-1111\",\"_links\":{\"self\":{\"href\":\"\"}},\"_embedded\":{\"sample_files\":[{\"_links\":{\"self\":{\"href\":\"http://127.0.0.1:8080/api/projects/4/samples/51/sequenceFiles/3\"}}},{\"_links\":{\"self\":{\"href\":\"http://127.0.0.1:8080/api/projects/4/samples/51/sequenceFiles/4\"}}},{\"_links\":{\"self\":{\"href\":\"http://127.0.0.1:8080/api/projects/4/samples/51/sequenceFiles/9\"}}},{\"_links\":{\"self\":{\"href\":\"http://127.0.0.1:8080/api/projects/4/samples/51/sequenceFiles/10\"}}}]}}]}}", "__get_data_table_entry__": "", "GALAXY_DATATYPES_CONF_FILE": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp"}, "output_data": [{"extra_files_path": "/home/jthiessen/galaxy-dist/database/job_working_directory/000/992/dataset_992_files", "ext": "auto", "out_data_name": "output", "hda_id": 801, "file_name": "/home/jthiessen/galaxy-dist/database/files/000/dataset_992.dat", "dataset_id": 992}], "job_config": {"GALAXY_ROOT_DIR": "/home/jthiessen/galaxy-dist", "TOOL_PROVIDED_JOB_METADATA_FILE": "galaxy.json", "GALAXY_DATATYPES_CONF_FILE": "/home/jthiessen/galaxy-dist/database/tmp/tmpaNhVAp"}} diff --git a/irida_import/tests/unit/test_irida_import.py b/irida_import/tests/unit/test_irida_import.py index 92163c7a..75b173af 100644 --- a/irida_import/tests/unit/test_irida_import.py +++ b/irida_import/tests/unit/test_irida_import.py @@ -21,7 +21,7 @@ class MockConfig: def __init__(self): self.ADMIN_KEY = "09008eb345c9d5a166b0d8f301b1e72c" - self.GALAXY_URL = "http://localhost:8888/" + self.GALAXY_URL = "http://127.0.0.1:8888/" self.ILLUMINA_PATH = '/illumina_reads' self.REFERENCE_PATH = '/references' self.MAX_WAITS = 1 @@ -30,7 +30,7 @@ def __init__(self): self.CLIENT_RETRY_DELAY = 30 self.CLIENT_ID = 'webClient' self.CLIENT_SECRET = 'webClientSecret' - self.TOKEN_ENDPOINT = 'http://localhost:8080/api/oauth/token' + self.TOKEN_ENDPOINT = 'http://127.0.0.1:8080/api/oauth/token' class TestIridaImport: From 6c81752e82ecf3964aae5a24c4efb8ec9d4e296d Mon Sep 17 00:00:00 2001 From: Eric Enns Date: Mon, 5 Dec 2022 11:42:41 -0600 Subject: [PATCH 4/7] chore: switch to chrome from chromium, and ensure irida base url is http://127.0.0.1:8080 --- irida_import/tests/integration/bash_scripts/install_galaxy.sh | 1 + irida_import/tests/integration/irida_data_setup.py | 1 + irida_import/tests/integration/test_irida_importer.py | 1 - irida_import/tests/integration/tests_integration.py | 3 +-- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/irida_import/tests/integration/bash_scripts/install_galaxy.sh b/irida_import/tests/integration/bash_scripts/install_galaxy.sh index 31517d09..528a4cfb 100755 --- a/irida_import/tests/integration/bash_scripts/install_galaxy.sh +++ b/irida_import/tests/integration/bash_scripts/install_galaxy.sh @@ -55,6 +55,7 @@ pushd galaxy/tools/irida-galaxy-importer/ cp irida_import/config.ini.sample irida_import/config.ini sed -i "s/^max_waits: .*$/max_waits: 1/" irida_import/config.ini sed -i "s|^galaxy_url: http://localhost:8888$|galaxy_url: http://127.0.0.1:$galaxy_port|" irida_import/config.ini +sed -i "s|^irida_url: http://localhost:8080$|irida_url: http://127.0.0.1:8080|" irida_import/config.ini echo "Configuring the tool's XML file" python -m irida_import.main --generate_xml diff --git a/irida_import/tests/integration/irida_data_setup.py b/irida_import/tests/integration/irida_data_setup.py index 9eb99d6d..8536f592 100644 --- a/irida_import/tests/integration/irida_data_setup.py +++ b/irida_import/tests/integration/irida_data_setup.py @@ -63,6 +63,7 @@ def __init__(self, base_url, user, password, branch, db_host, db_port, repo_dir) '--spring.datasource.url={} '.format(self.DB_JDBC_URL) +\ '--spring.datasource.username={} '.format(self.DB_USERNAME) +\ '--spring.datasource.password={} '.format(self.DB_PASSWORD) +\ + '--server.base.url=http://127.0.0.1:8080 ' +\ '--liquibase.update.database.schema=true ' +\ '--spring.jpa.hibernate.ddl-auto= ' +\ '--spring.jpa.properties.hibernate.hbm2ddl.import_files= ' +\ diff --git a/irida_import/tests/integration/test_irida_importer.py b/irida_import/tests/integration/test_irida_importer.py index 2e4f17c5..be9369d7 100644 --- a/irida_import/tests/integration/test_irida_importer.py +++ b/irida_import/tests/integration/test_irida_importer.py @@ -154,7 +154,6 @@ def _create_temp_file(file_name, file_contents): seq3 = _create_temp_file("seq3.fastq", fastq_contents) self.irida_session.post(sequences2, files={'file': open(seq3, 'rb')}) - import pdb; pdb.set_trace() # Export to Galaxy using the button on the dropdown menu self.driver.get(tests_integration.galaxy_url) history_panel = self.driver.find_element_by_id('current-history-panel') diff --git a/irida_import/tests/integration/tests_integration.py b/irida_import/tests/integration/tests_integration.py index d2639795..94b26717 100644 --- a/irida_import/tests/integration/tests_integration.py +++ b/irida_import/tests/integration/tests_integration.py @@ -5,7 +5,6 @@ from os import path from webdriver_manager.chrome import ChromeDriverManager -from webdriver_manager.core.utils import ChromeType from selenium.webdriver.chrome.options import Options as ChromeDriverOptions from irida_import.tests.integration.irida_data_setup import SetupIridaData @@ -26,7 +25,7 @@ repo_path = path.join('/tmp', 'repos') # Have ChromeDriverManager handle chrome installation and driver for compatibility with github actions -chrome_driver_path = ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install() +chrome_driver_path = ChromeDriverManager().install() chrome_driver_options = ChromeDriverOptions() # When creating new tests, these can be commented out to have driver window display chrome_driver_options.add_argument('--headless') From 2947f1a4d2b217fb7aaa62ef59c2419aaa90c535 Mon Sep 17 00:00:00 2001 From: deepsidhu85 Date: Fri, 3 Feb 2023 14:07:42 -0600 Subject: [PATCH 5/7] Object store support (#30) * Added function to download temp file and upload it to galaxy. * Updated to download file if not locally found * Updated logic to decide when to upload file vs link * Fixed incorrect header format. Added exception error message * Refactored logic * Added missing href to samplefile object instantiation * Added print statements * Added print statements to help debug * Added print statements to help debug * Moved setting of library dataset id for a sample file into the if/else statements * Refactored logic * Changed variable iniiialization to None from boolean * Reverted refactoring * Removed variable initialization not required * Added break to for loop once href is found for sample file * Added function to check if sha256 of downloaded file matches the expected sha256 * Updated get_file function to add upload sha 256 to sample file * Updated to get the actual hash * Added missing parameter * Updated logic to still upload a file to galaxy is there was no upload sha set in IRIDA * Updated logic to get file size from response if it exists * Converted bytes to non si unit * Refactored logic * Removed print statement * Updated parameter type comment * Changed param type, removed sha256 hash when creating a new SampleFile * Added comment to clarify that fastq and fast5 files are downloaded using the accept header for fastq files * Updated to explicitly name every argument, formatted for readability, updated param description for size, updated to comparing to None to use is instead of ==, fixed param names in function comments * Updated sample_file to contain content type of file * Added missing accept header * Moved getting content type into sample_file instead of passing content type in and updated to detect file type from file href and returning the content type based on this * Added success tests for checking if correct sample_file object is created and the content type is set correctly using detection of type from href * Removed variable * Fixed path for fast5 file * Added try/except for if an invalid content type is set for a file. Added test for invalid content type * Updated called to assertRaises) * Fixed assert statement for href * Updated to make test class a subclass of unittest so we could inherit assertRaises * Removed inheritance from unittest since it breaks the current setup of the tests. Updated to use a try/catch and assert if the content type threw and error * Attempting to get error message using get * Another attempt to get the error message from valueerror * Yet another attempt to get the error message from the valueerror exception --- irida_import/irida_import.py | 215 +++++++++++++++---- irida_import/sample_file.py | 24 ++- irida_import/tests/unit/test_irida_import.py | 72 ++++++- 3 files changed, 262 insertions(+), 49 deletions(-) diff --git a/irida_import/irida_import.py b/irida_import/irida_import.py index 53084965..c1c5202c 100755 --- a/irida_import/irida_import.py +++ b/irida_import/irida_import.py @@ -5,9 +5,11 @@ import logging import os.path import pprint -import re -import sys import time +import tempfile +import shutil +import sys +import hashlib from bioblend import galaxy from bioblend.galaxy.objects import GalaxyInstance @@ -181,8 +183,28 @@ def get_sample_file(self, file_dict): resource = file_dict name = resource['fileName'] path = resource['file'] - - return SampleFile(name, path) + links = file_dict['links'] + for link in links: + if link['rel'] == 'self': + href = link['href'] + break + upload_sha_256 = None + file_size = None + + if 'uploadSha256' in resource.keys(): + upload_sha_256 = resource['uploadSha256'] + + if 'fileSizeBytes' in resource.keys(): + file_size = resource['fileSizeBytes'] + + + return SampleFile( + name=name, + path=path, + href=href, + file_size=file_size, + upload_sha_256=upload_sha_256, + ) def get_first_or_make_lib(self, desired_lib_name, email): """" @@ -305,7 +327,7 @@ def exists_in_lib(self, item_type, item_attr_name, desired_attr_value): - def existing_file(self, sample_file_path, galaxy_name): + def existing_file(self, sample_file_path, galaxy_name, size): """ Find out dataset id for an existing file @@ -314,6 +336,10 @@ def existing_file(self, sample_file_path, galaxy_name): :type galaxy_name: str :param galaxy_name: the full path to the sample file as it exists in Galaxy + :type size: long + :param size: The size in bytes of the file to check against a + previously uploaded file. Size will be `None` if api call to + get sample file details does not return a value for the size :rtype: Boolean :return: Return file unique ID otherwise Boolean False """ @@ -321,7 +347,8 @@ def existing_file(self, sample_file_path, galaxy_name): "Getting dataset ID for existing file: " + galaxy_name) found = False - size = os.path.getsize(sample_file_path) + if size is None: + size = os.path.getsize(sample_file_path) # check cache before fetching from galaxy. # current state of the library should only change between irida_import.py invocation @@ -335,6 +362,7 @@ def existing_file(self, sample_file_path, galaxy_name): if datasets: for data_id in datasets: item = self.reg_gi.libraries.show_dataset(self.library.id,data_id) + if item['file_size'] in (size, size + 1) and item['state'] == 'ok': found = item['id'] break @@ -561,39 +589,58 @@ def _add_file(self, added_to_galaxy=None, sample_folder_path=None,sample_folder_ :return: dataset object or the id of an existing dataset """ galaxy_sample_file_name = sample_folder_path + '/' + sample_file.name - if os.path.isfile(sample_file.path): - if sample_file.library_dataset_id == None: - #grab dataset_id if it does exist, if not will be given False - dataset_id = self.existing_file(sample_file.path,galaxy_sample_file_name) - - if dataset_id: - # Return dataset id of existing file - added_to_galaxy = [{'id': dataset_id}] - self.print_logged(time.strftime("[%D %H:%M:%S]:") + - ' Skipped file with Galaxy path: ' + - galaxy_sample_file_name) - sample_file.verified = True - self.skipped_files_log.append( - {'galaxy_name': galaxy_sample_file_name}) - else: - self.logger.debug( - " Sample file does not exist so uploading/linking it") - added = self.link( - sample_file, sample_folder_id) - if(added): - added_to_galaxy = added - self.print_logged(time.strftime("[%D %H:%M:%S]:") + - ' Imported file with Galaxy path: ' + - galaxy_sample_file_name) - self.uploaded_files_log.append( - {'galaxy_name': galaxy_sample_file_name}) - - sample_file.library_dataset_id = added_to_galaxy[0]['id'] + file_exists_locally = os.path.isfile(sample_file.path) - else: - error = ("File not found:\n Galaxy path:{0}\nLocal path:{1}" - ).format(galaxy_sample_file_name, sample_file.path) - raise ValueError(error) + if sample_file.library_dataset_id is None: + + #grab dataset_id if it does exist, if not will be given False + dataset_id = self.existing_file( + sample_file_path=sample_file.path, + galaxy_name=galaxy_sample_file_name, + size=sample_file.file_size + ) + + if dataset_id: + # Return dataset id of existing file + added_to_galaxy = [{'id': dataset_id}] + self.print_logged(time.strftime("[%D %H:%M:%S]:") + + ' Skipped file with Galaxy path: ' + + galaxy_sample_file_name) + sample_file.verified = True + self.skipped_files_log.append( + {'galaxy_name': galaxy_sample_file_name}) + else: + try: + if file_exists_locally: + self.logger.debug( + " Sample file does not exist so linking it") + added = self.link( + sample_file, sample_folder_id) + if(added): + added_to_galaxy = added + self.print_logged(time.strftime("[%D %H:%M:%S]:") + + ' Imported file with Galaxy path: ' + + galaxy_sample_file_name) + self.uploaded_files_log.append( + {'galaxy_name': galaxy_sample_file_name}) + else: + self.logger.debug( + " Sample file does not exist so uploading it") + added = self.upload_file_to_galaxy( + sample_file, sample_folder_id) + if(added): + added_to_galaxy = added + self.print_logged(time.strftime("[%D %H:%M:%S]:") + + ' Imported file with Galaxy path: ' + + galaxy_sample_file_name) + self.uploaded_files_log.append( + {'galaxy_name': galaxy_sample_file_name}) + except: + error = ("File not found:\n Galaxy path:{0}\nLocal path:{1}" + ).format(galaxy_sample_file_name, sample_file.path) + raise ValueError(error) + + sample_file.library_dataset_id = added_to_galaxy[0]['id'] return added_to_galaxy @@ -604,11 +651,11 @@ def link(self, sample_file, folder_id): :type sample_file: SampleFile :param sample_file: the sample file to link :type folder_id: ID of folder to link file to - :param sample_folder_path: the folder in Galaxy to store the file in + :param folder_id: the folder in Galaxy to store the file in :return: a list containing a single dict with the file's url, id, and name. """ - self.logger.debug(' Attempting to upload a file') + self.logger.debug('Attempting to link to file') added = None file_path = sample_file.path self.logger.debug( @@ -628,6 +675,94 @@ def link(self, sample_file, folder_id): return added + def upload_file_to_galaxy(self, sample_file, folder_id): + """ + Upload a sample file to Galaxy + + :type sample_file: SampleFile + :param sample_file: the sample file to upload + :type folder_id: ID of folder to upload file to + :param folder_id: the folder in Galaxy to store the file in + :return: a list containing a single dict with the file's + url, id, and name. + """ + self.logger.debug('Attempting to upload file') + added = None + file_path = sample_file.path + self.logger.debug( + " Sample file's local path is" + file_path) + file_type = 'auto' + file_ext = os.path.splitext(file_path)[1] + + # Assume fastq files are fastqsanger: + if file_ext == '.fastq': + file_type = 'fastqsanger' + + tmp_dir = tempfile.mkdtemp() + tmp_file_mode = 'w+b' + + try: + tmp_file = tempfile.NamedTemporaryFile(mode=tmp_file_mode, prefix=sample_file.name, dir=tmp_dir) + tmp_file.name = tmp_dir + "/" + sample_file.name + + try: + headers={'Accept': sample_file.get_content_type()} + + # Open the file for writing. + with open(tmp_file.name, tmp_file_mode) as f: + try: + # Write the content to the file + with self.irida.get(sample_file.href, headers=headers, stream=False) as resp: + f.write(resp.content) + except: + error = ("Unable to download file as it was not found:\nLocal path:{0}" + ).format(sample_file.path) + raise ValueError(error) + + if (sample_file.upload_sha_256 is None) or (self.check_file_hash_valid(tmp_file.name, sample_file.upload_sha_256)): + # Copies the file into the galaxy library + added = self.reg_gi.libraries.upload_file_from_local_path( + library_id=self.library.id, + file_local_path=tmp_file.name, + folder_id=folder_id, + file_type=file_type + ) + else: + error = ("Downloaded file sha256 does not match the original sha256:\nLocal path:{0}" + ).format(sample_file.path) + raise ValueError(error) + except ValueError as content_type_error: + raise ValueError(content_type_error) + + # closes and removes the temp file + tmp_file.close() + finally: + # Remove the temp directory + shutil.rmtree(tmp_dir) + + return added + + def check_file_hash_valid(self, temp_file_path, expected_sha_256): + """ + Check if the downloaded file sha256 matches the original uploaded file sha256 + + :type temp_file_path: Temporary file + :param temp_file_path: the downloaded file path + :type expected_sha_256: str + :param expected_sha_256: the original uploaded file sha256 + :return: if the downloaded file sha256 matches the original uploaded file sha256 + url, id, and name. + """ + BUF_SIZE = 32768 # Read file in 32kb chunks + sha256 = hashlib.sha256() + with open(temp_file_path, 'rb') as f: + while True: + data = f.read(BUF_SIZE) + if not data: + break + sha256.update(data) + return sha256.hexdigest() == expected_sha_256 + def print_summary(self, failed=False): """ Print a final summary of the tool's activity diff --git a/irida_import/sample_file.py b/irida_import/sample_file.py index b40eba2a..93272de4 100644 --- a/irida_import/sample_file.py +++ b/irida_import/sample_file.py @@ -20,7 +20,7 @@ class SampleFile: """A representation of a sample file obtained from IRIDA""" - def __init__(self, name, path): + def __init__(self, name, path, href, file_size=None, upload_sha_256=None): """ Create a sample file instance. @@ -28,10 +28,19 @@ def __init__(self, name, path): :param name: the name of the sample file :type path: str :param path: the URI of the sample file + :type href: str + :param href: the url of the sample file + :type file_size: long + :param file_size: the size of the file in bytes + :type upload_sha_256: str + :param upload_sha_256: the hash of the uploaded file """ self.path = path self.name = name + self.href = href + self.file_size = file_size + self.upload_sha_256 = upload_sha_256 self.library_dataset_id = None self.verified = False @@ -47,10 +56,21 @@ def __eq__(self, sample_file): return equal def __repr__(self): - return self.name + " @ " + self.path + return self.name + " @ " + self.path + " @ " + self.href def state(self, gi, library_id): return gi.libraries.show_dataset(library_id, self.library_dataset_id)['state'] def delete(self, gi, library_id): return gi.libraries.delete_library_dataset(library_id, self.library_dataset_id, purged=True)['deleted'] + + def get_content_type(self): + content_type_fastq = "application/fastq" + content_type_fasta = "application/fasta" + if 'assemblies' in self.href: + return content_type_fasta + elif 'fast5' in self.href or 'sequenceFiles' in self.href: + return content_type_fastq + else: + error = ("Unable to detect type of file and set content type. href is:{0}") + raise ValueError(error) \ No newline at end of file diff --git a/irida_import/tests/unit/test_irida_import.py b/irida_import/tests/unit/test_irida_import.py index 75b173af..d47917e3 100644 --- a/irida_import/tests/unit/test_irida_import.py +++ b/irida_import/tests/unit/test_irida_import.py @@ -17,7 +17,6 @@ from ...sample_file import SampleFile from ...sample_pair import SamplePair - class MockConfig: def __init__(self): self.ADMIN_KEY = "09008eb345c9d5a166b0d8f301b1e72c" @@ -129,7 +128,7 @@ def test_get_samples(self, imp, setup_json): """ param_dict = json.loads(setup_json)['param_dict'] json_params = json.loads(param_dict['json_params']) - sample_file = SampleFile('nameish', 'pathish') + sample_file = SampleFile(name='nameish', path='pathish', href="http://127.0.0.1/api/samples/1/pairs/1/files/1") imp.get_sample_file = Mock(return_value=sample_file) samples = imp.get_samples(json_params['_embedded']['samples'],False,False) @@ -140,6 +139,65 @@ def test_get_samples(self, imp, setup_json): assert isinstance(sample, Sample), 'The list must contain samples' assert len(samples) == 1, 'Number of samples is incorrect' + def test_get_fastq_file(self, imp): + """ + Test if correct sample_file object is created and the content type + application/fastq is returned for sequence files + """ + sample_file = SampleFile(name='test_file.fastq', path='/path/to/test_file.fastq', href="http://127.0.0.1/api/samples/1/sequenceFiles/1") + imp.get_sample_file = Mock(return_value=sample_file) + assert isinstance(sample_file, SampleFile), 'sample_file not an instance of SampleFile' + assert sample_file.name == "test_file.fastq" + assert sample_file.path == "/path/to/test_file.fastq" + assert sample_file.href == "http://127.0.0.1/api/samples/1/sequenceFiles/1" + assert sample_file.get_content_type() == "application/fastq" + + def test_get_fasta_file(self, imp): + """ + Test if correct sample_file object is created and the content type + application/fasta is returned for assemblies + """ + sample_file = SampleFile(name='test_file.fasta', path='/path/to/test_file.fasta', href="http://127.0.0.1/api/samples/54/assemblies/1") + imp.get_sample_file = Mock(return_value=sample_file) + assert isinstance(sample_file, SampleFile), 'sample_file not an instance of SampleFile' + assert sample_file.name == "test_file.fasta" + assert sample_file.path == "/path/to/test_file.fasta" + assert sample_file.href == "http://127.0.0.1/api/samples/54/assemblies/1" + assert sample_file.get_content_type() == "application/fasta" + + def test_get_fast5_file(self, imp): + """ + Test if correct sample_file object is created and the content type + application/fastq is returned for fast5 files. + """ + sample_file = SampleFile(name='test_file.fast5', path='/path/to/test_file.fast5', href="http://127.0.0.1/api/samples/1/fast5/1/files/1") + imp.get_sample_file = Mock(return_value=sample_file) + assert isinstance(sample_file, SampleFile), 'sample_file not an instance of SampleFile' + # Currently fast5 files are retrieved from IRIDA using an + # accept header of application/fastq + assert sample_file.name == "test_file.fast5" + assert sample_file.path == "/path/to/test_file.fast5" + assert sample_file.href == "http://127.0.0.1/api/samples/1/fast5/1/files/1" + assert sample_file.get_content_type() == "application/fastq" + + def test_get_content_type_invalid_href(self, imp): + """ + Test if an error is raised when attempting to check if the correct + content type is set for a file with an 'invalid' href + """ + sample_file = SampleFile(name='test_file.fast5', path='/path/to/test_file.fast5', href="http://127.0.0.1/api/samples/1/unknowntype/1/files/1") + imp.get_sample_file = Mock(return_value=sample_file) + assert isinstance(sample_file, SampleFile), 'sample_file not an instance of SampleFile' + # Currently fast5 files are retrieved from IRIDA using an + # accept header of application/fastq + assert sample_file.name == "test_file.fast5" + assert sample_file.path == "/path/to/test_file.fast5" + assert sample_file.href == "http://127.0.0.1/api/samples/1/unknowntype/1/files/1" + try: + sample_file.get_content_type() + except ValueError as content_type_error: + assert "Unable to detect type of file and set content type" in str(content_type_error) + def test_get_first_or_make_lib_empty(self, imp): """Test library creation if there are no preexisting libraries""" wanted_name = 'boblib' @@ -279,8 +337,8 @@ def test_add_samples_if_nec(self, imp, file_list): os.path.isfile = Mock(return_value=True) imp.unique_file = Mock(return_value=True) - sampleFile1 = SampleFile('file1', "/imaginary/path/file1.fasta") - sampleFile2 = SampleFile('file2', "/imaginary/path/file2.fasta") + sampleFile1 = SampleFile(name='file1', path="/imaginary/path/file1.fasta", href="http://127.0.0.1/api/samples/1/pairs/1/files/1") + sampleFile2 = SampleFile(name='file2', path="/imaginary/path/file2.fasta", href="http://127.0.0.1/api/samples/1/pairs/1/files/2") samplePair1 = SamplePair( 'pair1', sampleFile1, @@ -321,8 +379,8 @@ def test_add_samples_to_history(self, imp, file_list): history = imp.reg_gi.histories.create_history() - sampleFile1 = SampleFile('file1', "/imaginary/path/file1.fasta") - sampleFile2 = SampleFile('file2', "/imaginary/path/file2.fasta") + sampleFile1 = SampleFile(name='file1', path="/imaginary/path/file1.fasta", href="http://127.0.0.1/api/samples/1/pairs/1/files/1") + sampleFile2 = SampleFile(name='file2', path="/imaginary/path/file2.fasta", href="http://127.0.0.1/api/samples/1/pairs/1/files/2") samplePair1 = SamplePair( 'pair1', sampleFile1, @@ -370,7 +428,7 @@ def test_link(self, imp, folder_list): imp.reg_gi.libraries.upload_from_galaxy_filesystem = Mock( return_value=single_file_list) - sample_file = SampleFile('file1', 'file:///imaginary/path/file1.fasta') + sample_file = SampleFile(name='file1', path='file:///imaginary/path/file1.fasta', href="http://127.0.0.1/api/samples/1/pairs/1/files/1") sample_folder_path = '/bobfolder1/bobfolder2/bobfolder3' uploaded = imp.link(sample_file, sample_folder_path) assert uploaded == single_file_list, 'The correct file must be made' From cc6c7be87d87c09bcb7c90025f2a6fe0a7038c58 Mon Sep 17 00:00:00 2001 From: deepsidhu85 Date: Fri, 10 Feb 2023 11:03:50 -0600 Subject: [PATCH 6/7] Updated version number (#31) * Updated version number * Updated wording of changelog entry * Updated branch names from master to main --- .github/workflows/ci-tests.yml | 10 +++++----- CHANGELOG.md | 3 +++ Makefile | 2 +- README.md | 6 +++--- irida_import/irida_import.xml.sample | 2 +- irida_import/tests/integration/irida_data_setup.py | 2 +- irida_import/tests/integration/tests_integration.py | 2 +- setup.py | 2 +- 8 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 9924e9e5..d496f823 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -3,14 +3,14 @@ name: Integration Tests on: pull_request: # Run on all pull requests push: - branches: # Run on any push to development or master + branches: # Run on any push to development or main - development - - master - schedule: # Run weekly on development and master + - main + schedule: # Run weekly on development and main - cron: 0 2 * * 1 branches: development - cron: 0 2 * * 1 - branches: master + branches: main jobs: @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: False #Setting so that if one of the test suites fail, the other will continue matrix: - branch: ['master','development'] # IRIDA Branches to test against + branch: ['main','development'] # IRIDA Branches to test against steps: - uses: actions/checkout@v2 #Checkout the project from git diff --git a/CHANGELOG.md b/CHANGELOG.md index 53ce1333..ead0412b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All notable changes to irida-galaxy-importer will be documeted in this file. +## 2.1.0 +* Added in support for importing IRIDA files that are not available locally (i.e. in the cloud) + ## 2.0.1 * Switched from travisCI tests to Github Actions * Switched integration tests to launch irida via gradle diff --git a/Makefile b/Makefile index f23bf182..0504e310 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SHELL=/bin/bash -IRIDA_VERSION?=master +IRIDA_VERSION?=main requirements: clean env source .virtualenv/bin/activate diff --git a/README.md b/README.md index d7c51331..09c5e716 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ The Galaxy [tools/][galaxy-tools] directory contains tools that come with the Ga ```bash cd galaxy/tools/ -git clone -b master https://github.com/phac-nml/irida-galaxy-importer.git +git clone -b main https://github.com/phac-nml/irida-galaxy-importer.git cd irida-galaxy-importer # Optional. Checkout specific release from https://github.com/phac-nml/irida-galaxy-importer/releases @@ -277,9 +277,9 @@ make unittests Integration tests can be run with ```bash -make integrationtests branch=master +make integrationtests branch=main ``` -This tests against the `master` branch of IRIDA +This tests against the `main` branch of IRIDA [galaxy]: https://galaxyproject.org/ [irida]: https://www.irida.ca/ diff --git a/irida_import/irida_import.xml.sample b/irida_import/irida_import.xml.sample index 7f140d9d..ffa3a303 100644 --- a/irida_import/irida_import.xml.sample +++ b/irida_import/irida_import.xml.sample @@ -1,4 +1,4 @@ - + server Date: Fri, 10 Feb 2023 14:53:58 -0600 Subject: [PATCH 7/7] Update CHANGELOG.md --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ead0412b..8923cd27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,6 @@ All notable changes to irida-galaxy-importer will be documeted in this file. ## 2.1.0 * Added in support for importing IRIDA files that are not available locally (i.e. in the cloud) - -## 2.0.1 * Switched from travisCI tests to Github Actions * Switched integration tests to launch irida via gradle