METplus-Internal #14 Initial release

internal_tests/scanning/environment/development.docker

@@ -0,0 +1,10 @@
+# Define the development environment for NCAR project machine seneca
+# Based on settings in /usr/local/src/met/README.snat
+
+# Top-level MET project directory
+MET_PROJ_DIR=`ls -1d /met/MET*`
+
+# SonarQube
+export SONARQUBE_DIR=/d1/projects/SonarQube/
+export SONARQUBE_WRAPPER_BIN=$SONARQUBE_DIR/build-wrapper-linux-x86
+export SONARQUBE_SCANNER_BIN=$SONARQUBE_DIR/sonar-scanner-4.6.2.2472-linux/bin

internal_tests/scanning/environment/development.seneca

@@ -0,0 +1,10 @@
+# Define the development environment for NCAR project machine seneca
+# Based on settings in /usr/local/src/met/README.snat
+
+# Top-level MET project directory
+MET_PROJ_DIR=/d1/projects/MET
+
+# SonarQube
+#export SONARQUBE_DIR=/d1/projects/SonarQube/
+#export SONARQUBE_WRAPPER_BIN=$SONARQUBE_DIR/build-wrapper-linux-x86
+#export SONARQUBE_SCANNER_BIN=$SONARQUBE_DIR/sonar-scanner-4.6.2.2472-linux/bin

internal_tests/scanning/sornaqube/run_nightly.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+#
+# Run nightly SonarQube scan
+#=======================================================================
+#
+# This run_nightly.sh script calls the run_sonarqube.sh script.
+# It is intented to be run nightly through cron. Output should be
+# directed to the LOGFILE, per cron convention. To run this script, use
+# the following commands:
+#
+#    git clone https://github.com/dtcenter/MET
+#    MET/scripts/sosnarqube/run_nightly.sh name
+#
+# Usage: run_nightly.sh name
+#    where "name" specifies a branch, tag, or hash
+#
+# For example, scan the develop branch:
+#    run_nightly.sh develop
+#
+#=======================================================================
+
+# Constants
+#EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu jpresto@ucar.edu linden@ucar.edu mccabe@ucar.edu"
+EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu mccabe@ucar.edu"
+#EMAIL_LIST="hsoh@ucar.edu"
+KEEP_DAYS=5
+
+function usage {
+  echo
+  echo "USAGE: run_nightly.sh name"
+  echo "   where \"name\" specifies a branch, tag, or hash."
+  echo
+}
+
+# Check for arguments
+if [ $# -lt 1 ]; then usage; exit 1; fi
+
+# Store the full path to the scripts directory
+SCRIPT_DIR=`dirname $0`
+if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi 
+
+# Define the development environment
+ENV_FILE=${SCRIPT_DIR}/../environment/development.`hostname`
+if [[ ! -e ${ENV_FILE} ]]; then
+  echo "$0: ERROR -> Development environment file missing: ${ENV_FILE}"
+  exit 1
+fi
+source ${ENV_FILE}
+
+SONARQUBE_WORK_DIR=${MET_PROJ_DIR}/MET_regression/sonarqube_METplus
+
+# Delete old directories
+find ${SONARQUBE_WORK_DIR} -mtime +${KEEP_DAYS} -name "NB*" | \
+     xargs rm -rf
+
+# Create and switch to a run directory
+TODAY=`date +%Y%m%d`
+YESTERDAY=`date -d "1 day ago" +%Y%m%d`
+RUN_DIR=${SONARQUBE_WORK_DIR}/NB${TODAY}
+[[ -e ${RUN_DIR} ]] && rm -rf ${RUN_DIR}
+mkdir -p ${RUN_DIR}
+cd ${RUN_DIR}
+
+# Create a logfile
+LOGFILE=${RUN_DIR}/run_sonarqube_${TODAY}.log
+
+# Run scan and check for bad return status
+${SCRIPT_DIR}/run_sonarqube.sh ${1} >& ${LOGFILE}
+if [[ $? -ne 0 ]]; then
+  echo "$0: The nightly SonarQube scan for METplus FAILED in `basename ${RUN_DIR}`." >> ${LOGFILE}
+  cat ${LOGFILE} | mail -s "METplus SonarQube scan Failed for ${1} in `basename ${RUN_DIR}` (autogen msg)" ${EMAIL_LIST}
+  exit 1
+fi
+
+# Convert SonarQube report from pdf to html
+
+exit 0

internal_tests/scanning/sornaqube/run_sonarqube.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+#
+# Run SonarQube Source Code Analyzer on a specified revision of MET
+#=======================================================================
+#
+# This run_sonarqube.sh script will check out the specified version
+# of MET and run the SonarQube Source Code Analyzer on it.  First,
+# go to the directory where you would like the SCA output written and
+# then run:
+#
+#    git clone https://github.com/dtcenter/MET
+#    MET/scripts/sonarqube/run_sonarqube_sca.sh name
+#
+# Usage: run_sonarqube_sca.sh name
+#    Test the specified branched version of MET:
+#       run_sonarqube_sca.sh {branch name}
+#    Test the specified tagged version of MET:
+#       run_sonarqube_sca.sh {tag name}
+#
+#=======================================================================
+
+# Constants
+GIT_REPO="https://github.com/dtcenter/METplus"
+
+function usage {
+        echo
+        echo "USAGE: $(basename $0) name"
+        echo "   where \"name\" specifies a branch, tag, or hash."
+        echo
+}
+
+# Check for arguments
+if [[ $# -lt 1 ]]; then usage; exit; fi
+
+# Check that SONARQUBE_WRAPPER_BIN is defined
+if [ -z ${SONARQUBE_WRAPPER_BIN} ]; then
+  which build-wrapper-linux-x86-64 2> /dev/null
+  if [ $? -eq 0 ]; then
+    SONARQUBE_WRAPPER_BIN=$(which build-wrapper-linux-x86-64 2> /dev/null)
+  else
+    which build-wrapper 2> /dev/null
+    if [ $? -eq 0 ]; then
+      SONARQUBE_WRAPPER_BIN=$(which build-wrapper 2> /dev/null)
+    else
+      echo "ERROR: SONARQUBE_WRAPPER_BIN must be set"
+      exit 1
+    fi
+  fi
+fi
+if [ ! -e ${SONARQUBE_WRAPPER_BIN} ]; then
+  echo "ERROR: SONARQUBE_WRAPPER_BIN (${SONARQUBE_WRAPPER_BIN}) does not exist"
+  exit 1
+fi
+
+# Check that SONARQUBE_SCANNER_BIN is defined
+if [ -z ${SONARQUBE_SCANNER_BIN} ]; then
+  which sonar-scanner 2> /dev/null
+  if [ $? -eq 0 ]; then
+    SONARQUBE_SCANNER_BIN=$(which sonar-scanner 2> /dev/null)
+  else
+    echo "ERROR: SONARQUBE_SCANNER_BIN must be set"
+    exit 1
+  fi
+fi
+if [ ! -e ${SONARQUBE_SCANNER_BIN} ]; then
+  echo "ERROR: SONARQUBE_SCANNER_BIN (${SONARQUBE_SCANNER_BIN}) does not exist"
+  exit 1
+fi
+
+# Sub-routine for running a command and checking return status
+function run_command() {
+
+  # Print the command being called
+  echo "CALLING: $1"
+
+  # Run the command and store the return status
+  $1
+  STATUS=$?
+
+  # Check return status
+  if [[ ${STATUS} -ne 0 ]]; then
+     echo "ERROR: Command returned with non-zero status ($STATUS): $1"
+     exit ${STATUS}
+  fi
+
+  return ${STATUS}
+}
+
+
+# Store the full path to the scripts directory
+SCRIPT_DIR=`dirname $0`
+if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi 
+
+# Clone repo into a sub-directory and checkout the requested version
+REPO_DIR="METplus-${1}"
+
+if [ -e ${REPO_DIR} ]; then
+  run_command "rm -rf ${REPO_DIR}"
+fi
+run_command "git clone ${GIT_REPO} ${REPO_DIR}"
+run_command "cd ${REPO_DIR}"
+run_command "git checkout ${1}"
+
+# Set the build id
+#BUILD_ID="MET-${1}"
+
+SONAR_PROPERTIES=sonar-project.properties
+
+# Copy sonar-project.properties for Python code
+[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES
+cp -p $SCRIPT_DIR/sonar-project.properties $SONAR_PROPERTIES
+
+# Run SonarQube scan for Python code
+run_command "${SONARQUBE_SCANNER_BIN}/sonar-scanner"
+
+# Run SonarQube report generator to make a PDF file
+#TODAY=`date +%Y%m%d`

internal_tests/scanning/sornaqube/sonar-project.properties

@@ -0,0 +1,17 @@
+sonar.projectKey=org.sonarqube:METplus_python_NB
+sonar.projectName=METplus Nightly Build
+sonar.projectVersion=1.0
+
+sonar.sources=docs,internal_tests,manage_externals,metplus,parm,produtil,ush
+
+# The build-wrapper output dir
+
+# Encoding of the source files
+sonar.sourceEncoding=UTF-8
+
+#----- Default SonarQube server
+#sonar.host.url=http://localhost:9000
+sonar.host.url=http://mandan:9000
+
+sonar.login=met
+sonar.password=met@sonar.ucar