Other: bootstrap script for Amazon Linux AMI

setup/bootstrap_amazon_linux.sh

@@ -0,0 +1,199 @@
+#!/bin/bash
+set -eu
+
+REDASH_BASE_PATH=/opt/redash
+FILES_BASE_URL=https://raw.githubusercontent.com/EverythingMe/redash/docs_setup/setup/files/
+FILE_BASE_URL_FOR_AMAZON_LINUX=https://raw.githubusercontent.com/EverythingMe/redash/master/setup/files/
+# Verify running as root:
+if [ "$(id -u)" != "0" ]; then
+    if [ $# -ne 0 ]; then
+        echo "Failed running with sudo. Exiting." 1>&2
+        exit 1
+    fi
+    echo "This script must be run as root. Trying to run with sudo."
+    sudo bash $0 --with-sudo
+    exit 0
+fi
+
+# Base packages
+yum update
+yum install -y python-pip python-devel nginx curl
+yes | yum groupinstall -y "Development Tools"
+yum install -y libffi-devel openssl-devel
+
+# redash user
+# TODO: check user doesn't exist yet?
+if [-x $(adduser --system --no-create-home --comment "" redash)]; then
+  echo "redash user have already registered."
+fi
+add_service() {
+    service_name=$1
+    service_command="/etc/init.d/$service_name"
+
+    echo "Adding service: $service_name (/etc/init.d/$service_name)."
+    chmod +x $service_command
+
+    if command -v chkconfig >/dev/null 2>&1; then
+        # we're chkconfig, so lets add to chkconfig and put in runlevel 345
+        chkconfig --add $service_name && echo "Successfully added to chkconfig!"
+        chkconfig --level 345 $service_name on && echo "Successfully added to runlevels 345!"
+    elif command -v update-rc.d >/dev/null 2>&1; then
+        #if we're not a chkconfig box assume we're able to use update-rc.d
+        update-rc.d $service_name defaults && echo "Success!"
+    else
+        echo "No supported init tool found."
+    fi
+
+    $service_command start
+}
+
+# PostgreSQL
+pg_available=0
+psql --version || pg_available=$?
+if [ $pg_available -ne 0 ]; then
+    # wget $FILES_BASE_URL"postgres_apt.sh" -O /tmp/postgres_apt.sh
+    # bash /tmp/postgres_apt.sh
+    yum update
+    yum -y install postgresql93-server postgresql93-devel
+    service postgresql93 initdb
+    add_service "postgresql93"
+fi
+
+
+
+# Redis
+redis_available=0
+redis-cli --version || redis_available=$?
+if [ $redis_available -ne 0 ]; then
+    wget http://download.redis.io/releases/redis-2.8.17.tar.gz
+    tar xzf redis-2.8.17.tar.gz
+    rm redis-2.8.17.tar.gz
+    cd redis-2.8.17
+    make
+    make install
+
+    # Setup process init & configuration
+
+    REDIS_PORT=6379
+    REDIS_CONFIG_FILE="/etc/redis/$REDIS_PORT.conf"
+    REDIS_LOG_FILE="/var/log/redis_$REDIS_PORT.log"
+    REDIS_DATA_DIR="/var/lib/redis/$REDIS_PORT"
+
+    mkdir -p `dirname "$REDIS_CONFIG_FILE"` || die "Could not create redis config directory"
+    mkdir -p `dirname "$REDIS_LOG_FILE"` || die "Could not create redis log dir"
+    mkdir -p "$REDIS_DATA_DIR" || die "Could not create redis data directory"
+
+    wget -O /etc/init.d/redis_6379 $FILES_BASE_URL"redis_init"
+    wget -O $REDIS_CONFIG_FILE $FILES_BASE_URL"redis.conf"
+
+    add_service "redis_$REDIS_PORT"
+
+    cd ..
+    rm -rf redis-2.8.17
+fi
+
+
+if [ ! -d "$REDASH_BASE_PATH" ]; then
+    sudo mkdir /opt/redash
+    sudo chown redash /opt/redash
+    sudo -u redash mkdir /opt/redash/logs
+fi
+
+# Default config file
+if [ ! -f "/opt/redash/.env" ]; then
+    sudo -u redash wget $FILES_BASE_URL"env" -O /opt/redash/.env
+fi
+
+# Install latest version
+REDASH_VERSION=${REDASH_VERSION-0.6.3.b906}
+LATEST_URL="https://github.com/EverythingMe/redash/releases/download/v${REDASH_VERSION}/redash.$REDASH_VERSION.tar.gz"
+VERSION_DIR="/opt/redash/redash.$REDASH_VERSION"
+REDASH_TARBALL=/tmp/redash.tar.gz
+REDASH_TARBALL=/tmp/redash.tar.gz
+
+if [ ! -d "$VERSION_DIR" ]; then
+    sudo -u redash wget $LATEST_URL -O $REDASH_TARBALL
+    sudo -u redash mkdir $VERSION_DIR
+    sudo -u redash tar -C $VERSION_DIR -xvf $REDASH_TARBALL
+    ln -nfs $VERSION_DIR /opt/redash/current
+    ln -nfs /opt/redash/.env /opt/redash/current/.env
+
+    cd /opt/redash/current
+
+    # TODO: venv?
+    pip install -r requirements.txt
+fi
+
+# InfluxDB dependencies:
+pip install influxdb==2.6.0
+
+# BigQuery dependencies:
+pip install google-api-python-client==1.2 pyOpenSSL==0.14 oauth2client==1.2
+
+# MySQL dependencies:
+yum install -y mysql-devel
+pip install MySQL-python==1.2.5
+
+# Mongo dependencies:
+pip install pymongo==2.7.2
+
+# Setup supervisord + sysv init startup script
+sudo -u redash mkdir -p /opt/redash/supervisord
+pip install supervisor==3.1.2 # TODO: move to requirements.txt
+
+# Create database / tables
+pg_user_exists=0
+sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='redash'" | grep -q 1 || pg_user_exists=$?
+if [ $pg_user_exists -ne 0 ]; then
+    echo "Creating redash postgres user & database."
+    sudo -u postgres createuser redash --no-superuser --no-createdb --no-createrole
+    sudo -u postgres createdb redash --owner=redash
+
+    cd /opt/redash/current
+    sudo -u redash bin/run ./manage.py database create_tables
+fi
+
+# Create default admin user
+cd /opt/redash/current
+# TODO: make sure user created only once
+# TODO: generate temp password and print to screen
+sudo -u redash bin/run ./manage.py users create --admin --password admin "Admin" "admin"
+
+# Create re:dash read only pg user & setup data source
+pg_user_exists=0
+sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='redash_reader'" | grep -q 1 || pg_user_exists=$?
+if [ $pg_user_exists -ne 0 ]; then
+    echo "Creating redash reader postgres user."
+
+    sudo yum install expect
+
+    REDASH_READER_PASSWORD=$(mkpasswd)
+    sudo -u postgres psql -c "CREATE ROLE redash_reader WITH PASSWORD '$REDASH_READER_PASSWORD' NOCREATEROLE NOCREATEDB NOSUPERUSER LOGIN"
+    sudo -u redash psql -c "grant select(id,name,type) ON data_sources to redash_reader;" redash
+    sudo -u redash psql -c "grant select on activity_log, events, queries, dashboards, widgets, visualizations, query_results to redash_reader;" redash
+
+    cd /opt/redash/current
+    sudo -u redash bin/run ./manage.py ds new -n "re:dash metadata" -t "pg" -o "{\"user\": \"redash_reader\", \"password\": \"$REDASH_READER_PASSWORD\", \"host\": \"localhost\", \"dbname\": \"redash\"}"
+fi
+
+
+# Get supervisord startup script
+sudo -u redash wget -O /opt/redash/supervisord/supervisord.conf $FILE_BASE_URL_FOR_AMAZON_LINUX"supervisord_for_amazon_linux.conf"
+
+# install start-stop-daemon
+wget http://developer.axis.com/download/distribution/apps-sys-utils-start-stop-daemon-IR1_9_18-2.tar.gz
+tar xvzf apps-sys-utils-start-stop-daemon-IR1_9_18-2.tar.gz
+cd apps/sys-utils/start-stop-daemon-IR1_9_18-2/
+gcc start-stop-daemon.c -o start-stop-daemon
+cp start-stop-daemon /sbin/
+
+wget -O /etc/init.d/redash_supervisord $FILE_BASE_URL_FOR_AMAZON_LINUX"redash_supervisord_init_for_amazon_linux"
+add_service "redash_supervisord"
+
+# Nginx setup
+if [-x $(mkdir /etc/nginx/sites-available)]; then
+  echo "/etc/nginx/sites-available exists."
+fi
+wget -O /etc/nginx/sites-available/redash $FILES_BASE_URL"nginx_redash_site"
+ln -nfs /etc/nginx/sites-available/redash /etc/nginx/conf.d/redash.conf
+service nginx restart

setup/redash_supervisord_init_for_amazon_linux

@@ -0,0 +1,125 @@
+#!/bin/sh
+# /etc/init.d/redash_supervisord
+### BEGIN INIT INFO
+# Provides:          supervisord
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: process supervisor
+### END INIT INFO
+
+# Author: Ron DuPlain <ron.duplain@gmail.com>
+
+# Do NOT "set -e"
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin
+NAME=supervisord
+DESC="process supervisor"
+DAEMON=/usr/local/bin/$NAME
+DAEMON_ARGS="--configuration /opt/redash/supervisord/supervisord.conf "
+PIDFILE=/opt/redash/supervisord/supervisord.pid
+SCRIPTNAME=/etc/init.d/redash_supervisord
+USER=redash
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/default/$NAME ] && . /etc/default/$NAME
+
+. /etc/rc.d/init.d/functions
+
+#
+# Function that starts the daemon/service
+#
+
+do_start()
+{
+    # Return
+    #   0 if daemon has been started
+    #   1 if daemon was already running
+    #   2 if daemon could not be started
+    start-stop-daemon --start --quiet --pidfile $PIDFILE --user $USER --chuid $USER --exec $DAEMON --test > /dev/null \
+        || return 1
+    start-stop-daemon --start --quiet --pidfile $PIDFILE --user $USER --chuid $USER --exec $DAEMON -- \
+        $DAEMON_ARGS \
+        || return 2
+    # Add code here, if necessary, that waits for the process to be ready
+    # to handle requests from services started subsequently which depend
+    # on this one.  As a last resort, sleep for some time.
+}
+
+#
+# Function that stops the daemon/service
+#
+
+do_stop()
+{
+    # Return
+    #   0 if daemon has been stopped
+    #   1 if daemon was already stopped
+    #   2 if daemon could not be stopped
+    #   other if a failure occurred
+    start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --user $USER --chuid $USER --name $NAME
+    RETVAL="$?"
+    [ "$RETVAL" = 2 ] && return 2
+    # Wait for children to finish too if this is a daemon that forks
+    # and if the daemon is only ever run from this initscript.
+    # If the above conditions are not satisfied then add some other code
+    # that waits for the process to drop all resources that could be
+    # needed by services started subsequently.  A last resort is to
+    # sleep for some time.
+    start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --user $USER --chuid $USER --exec $DAEMON
+    [ "$?" = 2 ] && return 2
+    # Many daemons don't delete their pidfiles when they exit.
+    rm -f $PIDFILE
+    return "$RETVAL"
+}
+
+case "$1" in
+  start)
+    [ "$VERBOSE" != no ] && echo "Starting $DESC" "$NAME"
+    do_start
+    case "$?" in
+        0|1) [ "$VERBOSE" != no ] && echo 0 ;;
+        2) [ "$VERBOSE" != no ] && echo 1 ;;
+    esac
+    ;;
+  stop)
+    [ "$VERBOSE" != no ] && echo "Stopping $DESC" "$NAME"
+    do_stop
+    case "$?" in
+        0|1) [ "$VERBOSE" != no ] && echo 0 ;;
+        2) [ "$VERBOSE" != no ] && echo 1 ;;
+    esac
+    ;;
+  status)
+       status -p "$STASH_PID" stash
+       # status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
+       ;;
+  restart)
+    echo "Restarting $DESC" "$NAME"
+    do_stop
+    case "$?" in
+      0|1)
+        do_start
+        case "$?" in
+            0) echo 0 ;;
+            1) echo 1 ;; # Old process is still running
+            *) echo 1 ;; # Failed to start
+        esac
+        ;;
+      *)
+        # Failed to stop
+        echo 1
+        ;;
+    esac
+    ;;
+  *)
+    echo "Usage: $SCRIPTNAME {start|stop|status|restart}" >&2
+    exit 3
+    ;;
+esac
+

setup/supervisord_for_amazon_linux.conf

@@ -0,0 +1,31 @@
+
+[supervisord]
+nodaemon=false
+logfile=/opt/redash/logs/supervisord.log
+pidfile=/opt/redash/supervisord/supervisord.pid
+directory=/opt/redash/current
+
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:redash_server]
+command=/opt/redash/current/bin/run /usr/local/bin/gunicorn -b 127.0.0.1:5000 --name redash -w 4 redash.wsgi:app
+process_name=redash_server
+numprocs=1
+priority=999
+autostart=true
+autorestart=true
+stdout_logfile=/opt/redash/logs/api.log
+stderr_logfile=/opt/redash/logs/api_error.log
+
+[program:redash_celery]
+command=/opt/redash/current/bin/run /usr/local/bin/celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries
+process_name=redash_celery
+numprocs=1
+priority=999
+autostart=true
+autorestart=true
+stdout_logfile=/opt/redash/logs/celery.log