Merge pull request #152 from Zarquan/20200718-zrq-kubernetes

20200718 zrq kubernetes

experiments/zrq/pyspark/Dockermod

@@ -0,0 +1,34 @@
+#
+# <meta:header>
+#   <meta:licence>
+#     Copyright (c) 2020, ROE (http://www.roe.ac.uk/)
+#
+#     This information is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     This information is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#   </meta:licence>
+# </meta:header>
+#
+
+ARG basetag
+
+FROM aglais/pyspark:${basetag}
+MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
+
+ARG buildtag
+ARG buildtime
+
+LABEL maintainer="Dave Morris <docker-admin@metagrid.co.uk>"
+LABEL buildtag="${buildtag}"
+LABEL buildtime="${buildtime}"
+LABEL gitrepo="https://github.com/wfau/aglais"
+

experiments/zrq/spark/Dockermod

@@ -0,0 +1,34 @@
+#
+# <meta:header>
+#   <meta:licence>
+#     Copyright (c) 2020, ROE (http://www.roe.ac.uk/)
+#
+#     This information is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     This information is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#   </meta:licence>
+# </meta:header>
+#
+
+ARG basetag
+
+FROM aglais/spark:${basetag}
+MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
+
+ARG buildtag
+ARG buildtime
+
+LABEL maintainer="Dave Morris <docker-admin@metagrid.co.uk>"
+LABEL buildtag="${buildtag}"
+LABEL buildtime="${buildtime}"
+LABEL gitrepo="https://github.com/wfau/aglais"
+

experiments/zrq/terraform/modules/cluster/variables.tf

@@ -62,7 +62,7 @@ variable "zrq_cluster_template_name" {
 
 variable "zrq_worker_flavor_name" {
     description = "Flavor for the worker nodes"
-    default = "general.v1.tiny"
+    default = "general.v1.small"
     type = string
     }
 

experiments/zrq/zeppelin/docker/Dockerbase

@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:16.04 AS zeppelinbase
+MAINTAINER Apache Software Foundation <dev@zeppelin.apache.org>
+
+ARG buildtag
+ARG buildtime
+
+LABEL maintainer="Dave Morris <docker-admin@metagrid.co.uk>"
+LABEL buildtag="${buildtag}"
+LABEL buildtime="${buildtime}"
+LABEL gitrepo="https://github.com/wfau/aglais"
+
+ENV LOG_TAG="[ZEPPELIN]:"
+ENV LANG=en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
+
+RUN echo "$LOG_TAG update and install basic packages" && \
+    apt-get -y update && \
+    apt-get install -y locales && \
+    locale-gen $LANG && \
+    apt-get install -y software-properties-common && \
+    apt -y autoclean && \
+    apt -y dist-upgrade && \
+    apt-get install -y build-essential
+
+RUN echo "$LOG_TAG install tini related packages" && \
+    apt-get install -y wget curl grep sed dpkg && \
+    TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \
+    curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \
+    dpkg -i tini.deb && \
+    rm tini.deb
+
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+RUN echo "$LOG_TAG Install java8" && \
+    apt-get -y update && \
+    apt-get install -y openjdk-8-jdk && \
+    rm -rf /var/lib/apt/lists/*
+
+# should install conda first before numpy, matploylib since pip and python will be installed by conda
+RUN echo "$LOG_TAG Install miniconda3 related packages" && \
+    apt-get -y update && \
+    apt-get install -y bzip2 ca-certificates \
+    libglib2.0-0 libxext6 libsm6 libxrender1 \
+    git mercurial subversion && \
+    echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \
+    wget --quiet https://repo.continuum.io/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh
+
+ENV PATH /opt/conda/bin:$PATH
+
+RUN echo "$LOG_TAG Install python related packages" && \
+    apt-get -y update && \
+    apt-get install -y python-dev python-pip && \
+    apt-get install -y gfortran && \
+    # numerical/algebra packages
+    apt-get install -y libblas-dev libatlas-dev liblapack-dev && \
+    # font, image
+    apt-get install -y libpng-dev libfreetype6-dev libxft-dev && \
+    # for tkinter
+    apt-get install -y python-tk libxml2-dev libxslt-dev zlib1g-dev && \
+    hash -r && \
+    conda config --set always_yes yes --set changeps1 no && \
+    conda update -q conda && \
+    conda info -a && \
+    conda config --add channels conda-forge && \
+    pip install -q pycodestyle==2.5.0 && \
+    pip install -q numpy==1.17.3 pandas==0.25.0 scipy==1.3.1 grpcio==1.19.0 bkzep==0.6.1 hvplot==0.5.2 protobuf==3.10.0 pandasql==0.7.3 ipython==7.8.0 matplotlib==3.0.3 ipykernel==5.1.2 jupyter_client==5.3.4 bokeh==1.3.4 panel==0.6.0 holoviews==1.12.3 seaborn==0.9.0 plotnine==0.5.1 intake==0.5.3 intake-parquet==0.2.2 altair==3.2.0 pycodestyle==2.5.0 apache_beam==2.15.0
+
+RUN echo "$LOG_TAG Install R related packages" && \
+    echo "PATH: $PATH" && \
+    ls /opt/conda/bin && \
+    echo "deb http://cran.rstudio.com/bin/linux/ubuntu xenial/" | tee -a /etc/apt/sources.list && \
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 51716619E084DAB9 && \
+    apt-get -y update && \
+    apt-get -y --allow-unauthenticated install r-base r-base-dev && \
+    R -e "install.packages('evaluate', repos = 'https://cloud.r-project.org')" && \
+    R -e "install.packages('knitr', repos='http://cran.us.r-project.org')" && \
+    R -e "install.packages('ggplot2', repos='http://cran.us.r-project.org')" && \
+    R -e "install.packages('googleVis', repos='http://cran.us.r-project.org')" && \
+    R -e "install.packages('data.table', repos='http://cran.us.r-project.org')" && \
+    R -e "install.packages('IRkernel', repos = 'https://cloud.r-project.org');IRkernel::installspec()" && \
+    R -e "install.packages('shiny', repos = 'https://cloud.r-project.org')" && \
+    # for devtools, Rcpp
+    apt-get -y install libcurl4-gnutls-dev libssl-dev && \
+    R -e "install.packages('devtools', repos='http://cran.us.r-project.org')" && \
+    R -e "install.packages('Rcpp', repos='http://cran.us.r-project.org')" && \
+    Rscript -e "library('devtools'); library('Rcpp'); install_github('ramnathv/rCharts')"
+
+RUN echo "$LOG_TAG Cleanup" && \
+    apt-get autoclean && \
+    apt-get clean
+
+
+

experiments/zrq/zeppelin/docker/Dockerfile

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG buildsrc
+
+FROM aglais/zeppelinbase:${buildsrc}
+MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
+
+ARG buildsrc
+ARG buildtag
+ARG buildtime
+
+LABEL maintainer="Dave Morris <docker-admin@metagrid.co.uk>"
+LABEL buildsrc="${buildsrc}"
+LABEL buildtag="${buildtag}"
+LABEL buildtime="${buildtime}"
+LABEL gitrepo="https://github.com/wfau/aglais"
+
+ENV Z_HOME="/zeppelin"
+ENV ZEPPELIN_ADDR="0.0.0.0"
+
+COPY . ${Z_HOME}/
+RUN chown -R root:root ${Z_HOME} && \
+    mkdir -p ${Z_HOME}/logs ${Z_HOME}/run ${Z_HOME}/webapps && \
+    # Allow process to edit /etc/passwd, to create a user entry for zeppelin
+    chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
+    # Give access to some specific folders
+    chmod -R 775 "${Z_HOME}/logs" "${Z_HOME}/run" "${Z_HOME}/notebook" "${Z_HOME}/conf" && \
+    # Allow process to create new folders (e.g. webapps)
+    chmod 775 ${Z_HOME}
+
+COPY log4j.properties ${Z_HOME}/conf/
+
+USER 1000
+
+EXPOSE 8080
+
+ENTRYPOINT [ "/usr/bin/tini", "--" ]
+WORKDIR ${Z_HOME}
+CMD ["bin/zeppelin.sh"]

experiments/zrq/zeppelin/docker/Dockermod

@@ -0,0 +1,37 @@
+#
+# <meta:header>
+#   <meta:licence>
+#     Copyright (c) 2020, ROE (http://www.roe.ac.uk/)
+#
+#     This information is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     This information is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#   </meta:licence>
+# </meta:header>
+#
+
+ARG buildtag
+
+FROM aglais/zeppelin:${buildtag}
+MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
+
+ARG buildtag
+ARG buildtime
+
+LABEL maintainer="Dave Morris <docker-admin@metagrid.co.uk>"
+LABEL buildtag="${buildtag}"
+LABEL buildtime="${buildtime}"
+LABEL gitrepo="https://github.com/wfau/aglais"
+
+COPY . ${Z_HOME}/
+
+

experiments/zrq/zeppelin/docker/log4j.properties

@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+log4j.rootLogger = INFO, stdout
+
+log4j.appender.stdout = org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n

notes/zrq/20200622-02-project-plan.txt

@@ -18,11 +18,6 @@
 #   </meta:licence>
 # </meta:header>
 #
-#zrq-notes-time
-#zrq-notes-indent
-#zrq-notes-crypto
-#zrq-notes-ansible
-#zrq-notes-osformat
 #
 
     # Jun20 Project milestone wiki page

notes/zrq/20200703-01-project-plan.txt

@@ -18,11 +18,6 @@
 #   </meta:licence>
 # </meta:header>
 #
-#zrq-notes-time
-#zrq-notes-indent
-#zrq-notes-crypto
-#zrq-notes-ansible
-#zrq-notes-osformat
 #
 
 

notes/zrq/20200706-03-k8s-dashboard.txt

@@ -39,6 +39,7 @@
         --publish 127.0.0.1:8443:8443 \
         --env "clouduser=${AGLAIS_USER:?}" \
         --env "cloudname=${AGLAIS_CLOUD:?}" \
+        --env "clustername=${CLUSTER_NAME:?}" \
         --volume "${HOME:?}/clouds.yaml:/etc/openstack/clouds.yaml:ro,z" \
         --volume "${AGLAIS_CODE}/experiments/zrq/kubernetes:/kubernetes:z" \
         atolmis/openstack-client \
@@ -48,9 +49,9 @@
 # Set the cloud and cluster names.
 #[user@kubernator]
 
-    cloudname=gaia-prod
-    clustername=drupal-one
-
+    # Deprecated - inherited from aglais.env
+    # cloudname=gaia-prod
+    # clustername=Tiberius
 
 # -----------------------------------------------------
 # Get the connection details for our cluster.
@@ -64,6 +65,18 @@
                 --force \
                 --dir "${HOME}/.kube/${clustername:?}"
 
+    # TODO - why does this always return '1'
+    #      adding -v -v to the call ends with
+
+    > ....
+    > ....
+    > 'SHELL'
+    > clean_up ConfigCluster: 'SHELL'
+    > END return value: 1
+
+    # Is this related ?
+    # https://bugs.launchpad.net/python-magnumclient/+bug/1807104
+
 
 # -----------------------------------------------------
 # Deploy the dashboard (recomended method).
@@ -143,12 +156,16 @@
 # https://github.com/stedolan/jq/issues/47#issuecomment-374179653
 #[root@dashboard]
 
-    kubectl \
-        --output json \
-        --kubeconfig "${HOME}/.kube/${clustername:?}/config" \
-        --namespace kubernetes-dashboard \
-            get secret \
-    | jq -r '.items[] | select(.metadata.annotations."kubernetes.io/service-account.name" == "dashboard-user") | .data.token | @base64d'
+    dashtoken=$(
+        kubectl \
+            --output json \
+            --kubeconfig "${HOME}/.kube/${clustername:?}/config" \
+            --namespace kubernetes-dashboard \
+                get secret \
+        | jq -r '.items[] | select(.metadata.annotations."kubernetes.io/service-account.name" == "dashboard-user") | .data.token | @base64d'
+        )
+
+    echo ${dashtoken:?}
 
     >   eyJhbGci........98iBhXew