From 6d6442fdcc351ac164ec4e471b4eaf3bcb994279 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Tue, 12 Dec 2023 20:01:50 +0100 Subject: [PATCH] In Centos7 CONDA_DIR is different from Ubuntu - /usr/local --- docker/Dockerfile.cc-analysis-centos7 | 46 ++++------ docker/Dockerfile.cc-base-centos7 | 121 +++++++++++--------------- 2 files changed, 68 insertions(+), 99 deletions(-) diff --git a/docker/Dockerfile.cc-analysis-centos7 b/docker/Dockerfile.cc-analysis-centos7 index 85d0fe2e..84c93630 100644 --- a/docker/Dockerfile.cc-analysis-centos7 +++ b/docker/Dockerfile.cc-analysis-centos7 @@ -17,18 +17,21 @@ ARG XCACHE_HOST="red-xcache1.unl.edu" # Configure environment +ENV CONDA_DIR /usr/local ENV CERT_DIR $CERT_DIR ENV XCACHE_HOST $XCACHE_HOST ENV BEARER_TOKEN_FILE $BEARER_TOKEN_FILE ENV SEC_TOKEN_SYSTEM_DIRECTORY $SEC_TOKEN_SYSTEM_DIRECTORY -ENV CONDA_DIR /opt/conda ENV SHELL /bin/bash ENV NB_USER $NB_USER ENV USER $NB_USER ENV NB_UID $NB_UID ENV NB_GID $NB_GID ENV HOME /home/$NB_USER -ENV PATH "/opt/conda/bin/:$PATH" +ENV PATH "${CONDA_DIR}/bin/:$PATH" +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 # Install all OS dependencies for notebook server that starts but lacks all # features (e.g., download as all possible file formats) @@ -146,29 +149,13 @@ RUN cd /tmp && \ ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/" ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClXcachePlugin-5.so" -# REMOVE THIS BLOCK AFTER TEST: -# ------- xrootd-authz-plugin ------------------------------- -#RUN cd /tmp && \ -# # ------- xrdcl-authz-plugin ------------------------------- -# git clone https://github.com/bbockelm/xrdcl-authz-plugin.git && \ -# cd xrdcl-authz-plugin && \ -# mkdir build && \ -# cd build && \ -# cmake /tmp/xrdcl-authz-plugin -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} && \ -# make && \ -# make install && \ -# ln -s ${CONDA_DIR}/lib/libXrdClAuthzPlugin-5.so ${CONDA_DIR}/lib/libXrdClAuthzPlugin.so - -#ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/" -#ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClAuthzPlugin.so" - RUN chmod 755 /etc/grid-security/certificates COPY certs/hcc-flatiron.pem /etc/grid-security/certificates/ RUN ln -s /etc/grid-security/certificates/hcc-flatiron.pem /etc/grid-security/certificates/80d1fda9.0 # TODO: RETEST IF WE STILL NEED THIS -ENV LD_LIBRARY_PATH="/opt/conda/lib/:$LD_LIBRARY_PATH" -ENV PATH="/opt/conda/bin/:$PATH" +ENV LD_LIBRARY_PATH="${CONDA_DIR}/lib/:$LD_LIBRARY_PATH" +ENV PATH="${CONDA_DIR}/bin/:$PATH" USER root # Setup supervisord files @@ -181,8 +168,8 @@ RUN groupadd -r condor && \ # FIXME: merge PRs open in distributed.git (oshadura) # Distributed: we need to install patched version of distributed version -COPY dask/distributed /opt/conda/lib/python3.9/site-packages/distributed -RUN cd /opt/conda/lib/python3.9/site-packages/distributed && \ +COPY dask/distributed ${CONDA_DIR}/lib/python3.10/site-packages/distributed +RUN cd ${CONDA_DIR}/lib/python3.10/site-packages/distributed && \ patch -p2 < 0001-Patch-from-bbockelman-adaptive-scaling.patch && \ patch -p2 < 0002-Allow-scheduler-to-preserve-worker-hostnames.patch # && patch -p2 < 0003-Activate-patch.patch @@ -191,22 +178,23 @@ RUN cd /opt/conda/lib/python3.9/site-packages/distributed && \ # FIXME: we have a wrong path, let's make a link. # cms-jovyan@jupyter-oksana-2eshadura-40cern-2ech:~$ echo $PATH -# /opt/conda/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games -RUN rm -rf /opt/conda/condabin && ln -s /opt/conda/bin /opt/conda/condabin +# ${CONDA_DIR}/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games +RUN rm -rf ${CONDA_DIR}/condabin && ln -s ${CONDA_DIR}/bin ${CONDA_DIR}/condabin # Cleanup RUN rm -rf /tmp/* \ && rm -rf $HOME/.cache/.pip/* \ - && find /opt/conda/ -type f,l -name '*.a' -delete \ - && find /opt/conda/ -type f,l -name '*.pyc' -delete \ - && find /opt/conda/ -type f,l -name '*.js.map' -delete \ - && rm -rf /opt/conda/pkgs + && mamba clean --all -f -y \ + && find ${CONDA_DIR} -type f -name '*.a' -delete \ + && find ${CONDA_DIR} -type f -name '*.pyc' -delete \ + && find ${CONDA_DIR} -type f -name '*.js.map' -delete \ + && (find ${CONDA_DIR}/lib/python3.10/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete || echo "no bokeh static files to cleanup") \ + && rm -rf ${CONDA_DIR}/pkgs # Mkdir CVMFS mount directory. Mount done upstream in configs. Harmless # if not utilized, crucial if CVMFS is desired RUN mkdir /cvmfs - # Prepare HTCondor spec. environment and execute dask-worker command # If we using this container as a sidecar, we don't setup any HTCondor spec. environment # nor and execute dask-worker command diff --git a/docker/Dockerfile.cc-base-centos7 b/docker/Dockerfile.cc-base-centos7 index 7975bed6..3181aeb9 100644 --- a/docker/Dockerfile.cc-base-centos7 +++ b/docker/Dockerfile.cc-base-centos7 @@ -37,14 +37,14 @@ ARG WORKER_IMAGE="${REGISTRY}/${PROJECT}/cc-analysis-ubuntu" ARG GITHUB_ACTIONS="false" # Configure environment -ENV CONDA_DIR /opt/conda +ENV CONDA_DIR /usr/local ENV SHELL /bin/bash ENV NB_USER $NB_USER ENV USER $NB_USER ENV NB_UID $NB_UID ENV NB_GID $NB_GID ENV HOME /home/$NB_USER -ENV PATH "/opt/conda/bin/:$PATH" +ENV PATH "${CONDA_DIR}/bin/:$PATH" ENV DASK_ROOT_CONFIG $DASK_ROOT_CONFIG ENV LABEXTENTION_CLUSTER $LABEXTENTION_CLUSTER ENV LABEXTENTION_FACTORY_CLASS $LABEXTENTION_FACTORY_CLASS @@ -136,14 +136,20 @@ RUN mkdir "/home/${NB_USER}/work" && \ # files across image layers when the permissions change RUN mamba install --quiet --yes \ -c conda-forge \ - 'notebook' \ - 'jupyterhub' \ - 'jupyterlab' \ + 'notebook'==6.5.3 \ + 'jupyterhub'==3.1.1 \ + 'jupyterlab'==3.6.1 \ + jupyterlab_widgets==3.0.7 \ + jupyterlab-git==0.41.0 \ + dask=2023.11.0 \ + dask_labextension==6.2.0 \ + jupyter_client==8.1.0 \ jupyterlab_widgets \ - jupyterlab-git \ - dask_labextension && \ + nodejs=14 \ + ipywidgets==8.0.6 && \ mamba clean \ --all \ + --force-pkgs-dirs \ --yes && \ npm cache clean --force && \ jupyter notebook --generate-config && \ @@ -167,36 +173,6 @@ RUN sed -re "s/c.NotebookApp/c.ServerApp/g" \ /etc/jupyter/jupyter_notebook_config.py > /etc/jupyter/jupyter_server_config.py && \ fix-permissions /etc/jupyter/ -# Install Jupyter Notebook, Lab, and Hub -# Generate a notebook server config -# Cleanup temporary files -# Correct permissions -# Do all this in a single RUN command to avoid duplicating all of the -# files across image layers when the permissions change -RUN mamba install --quiet --yes \ - -c conda-forge \ - 'notebook'==6.5.3 \ - 'jupyterhub'==3.1.1 \ - 'jupyterlab'==3.6.1 \ - jupyterlab_widgets==3.0.7 \ - jupyterlab-git==0.41.0 \ - dask=2023.11.0 \ - dask_labextension==6.2.0 \ - jupyter_client==8.1.0 \ - jupyterlab_widgets \ - nodejs=14 \ - ipywidgets==8.0.6 && \ - mamba clean \ - --all \ - --force-pkgs-dirs \ - --yes && \ - npm cache clean --force && \ - jupyter notebook --generate-config && \ - jupyter lab clean && \ - rm -rf "/home/${NB_USER}/.cache/yarn" - #fix-permissions "${CONDA_DIR}" && \ - #fix-permissions "/home/${NB_USER}" - # Enable the serverextensions that do not use the conf.d approach and # build JupyterLab. RUN jupyter serverextension enable dask_labextension jupyterlab_git && \ @@ -214,17 +190,6 @@ RUN mkdir -p ${DASK_ROOT_CONFIG} && chown -R "${NB_USER}:${NB_GID}" ${DASK_ROOT mkdir -p /var/lib/condor && \ mkdir -p /etc/condor/config.d -# Setup HTCondor user/group and change group for user $NB_USER -# Fix error (submitting jobs as user/group 0 (root) is not allowed for security reasons) and -# it configured from kubernetes side and updated in docker container to match it -RUN groupadd -r condor && \ - useradd -r -g condor -d /var/lib/condor -s /sbin/nologin condor - -# Fix permissions for Dask/Ceph config files -RUN chown -R "${NB_USER}:${NB_GID}" ${DASK_ROOT_CONFIG}/*.yaml -ENV LD_LIBRARY_PATH="/opt/conda/lib/:$LD_LIBRARY_PATH" -ENV PATH="/opt/conda/bin/:$PATH" - USER ${NB_UID} RUN mamba install --yes \ @@ -247,6 +212,40 @@ RUN mamba install --yes \ --force-pkgs-dirs \ --yes +USER ${NB_UID} +# Dask, Labextention and coffea-casa setup +COPY dask/dask.yaml dask/labextension.yaml ${DASK_ROOT_CONFIG}/ + +USER root +# Add HTCondor configuration files that not needed to be edited +COPY condor/condor_config /etc/condor/ +COPY condor/config.d /etc/condor/config.d/ + +RUN curl -L https://github.com/opensciencegrid/osg-vo-config/archive/refs/heads/master.tar.gz | \ + tar -xz --strip-components=1 --directory=/etc/grid-security --wildcards */vomses */vomsdir && \ + mv /etc/grid-security/vomses /etc + +# Setup HTCondor user/group and change group for user $NB_USER +# Fix error (submitting jobs as user/group 0 (root) is not allowed for security reasons) and +# it configured from kubernetes side and updated in docker container to match it +RUN groupadd -r condor && \ + useradd -r -g condor -d /var/lib/condor -s /sbin/nologin condor + +# Fix permissions for Dask/Ceph config files +RUN chown -R "${NB_USER}:${NB_GID}" ${DASK_ROOT_CONFIG}/*.yaml +ENV LD_LIBRARY_PATH="${CONDA_DIR}/lib/:$LD_LIBRARY_PATH" +ENV PATH="${CONDA_DIR}/bin/:$PATH" + +# FIXME: we have a wrong path, let's make a link. +# cms-jovyan@jupyter-oksana-2eshadura-40cern-2ech:~$ echo $PATH +# ${CONDA_DIR}/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games +RUN rm -rf ${CONDA_DIR}/condabin && ln -s ${CONDA_DIR}/bin ${CONDA_DIR}/condabin + +# Mkdir CVMFS mount directory. Mount done upstream in configs. Harmless +# if not utilized, crucial if CVMFS is desired +RUN mkdir /cvmfs + +USER ${NB_USER} RUN pip install --no-cache-dir \ correctionlib \ funcx \ @@ -289,44 +288,27 @@ COPY dask/jobqueue-coffea-casa.yaml dask/dask_tls.yaml ${DASK_ROOT_CONFIG}/ USER root # Distributed: we need to install patched version of distributed version -COPY dask/distributed ${CONDA_DIR}/lib/python3.9/site-packages/distributed -RUN cd ${CONDA_DIR}/lib/python3.9/site-packages/distributed && \ +COPY dask/distributed ${CONDA_DIR}/lib/python3.10/site-packages/distributed +RUN cd ${CONDA_DIR}/lib/python3.10/site-packages/distributed && \ patch -p2 < 0001-Patch-from-bbockelman-adaptive-scaling.patch && \ patch -p2 < 0002-Allow-scheduler-to-preserve-worker-hostnames.patch # && patch -p2 < 0003-Activate-patch.patch # && patch -p2 < 0004-Add-possibility-to-setup-external_adress-for-schedul.patch # && patch -p2 < 0005-Add-nanny-patch.patch - # Cleanup RUN rm -rf /tmp/* \ && rm -rf $HOME/.cache/.pip/* \ + && mamba clean --all -f -y \ && jupyter lab clean \ && jlpm cache clean \ && npm cache clean --force \ && find ${CONDA_DIR} -type f -name '*.a' -delete \ && find ${CONDA_DIR} -type f -name '*.pyc' -delete \ && find ${CONDA_DIR} -type f -name '*.js.map' -delete \ - && (find ${CONDA_DIR}/lib/python*/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete || echo "no bokeh static files to cleanup") \ + && (find ${CONDA_DIR}/lib/python3.10/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete || echo "no bokeh static files to cleanup") \ && rm -rf ${CONDA_DIR}/pkgs -# Fix permissions for Dask files -RUN chown -R "${NB_USER}:${NB_GID}" ${DASK_ROOT_CONFIG}/*.yaml -# xcache setup -#ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/" -ENV LD_LIBRARY_PATH="${CONDA_DIR}/lib/:$LD_LIBRARY_PATH" -#ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClAuthzPlugin.so" -ENV PATH="${CONDA_DIR}/bin/:$PATH" - -# FIXME: we have a wrong path, let's make a link. -# cms-jovyan@jupyter-oksana-2eshadura-40cern-2ech:~$ echo $PATH -# /opt/conda/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games -RUN rm -rf ${CONDA_DIR}/condabin && ln -s ${CONDA_DIR}/bin ${CONDA_DIR}/condabin - -# Mkdir CVMFS mount directory. Mount done upstream in configs. Harmless -# if not utilized, crucial if CVMFS is desired -RUN mkdir /cvmfs - # FIXME: add better layering for preparation of env ADD prepare-env/prepare-env-cc.sh /usr/local/bin/prepare-env.sh RUN chmod ugo+x /usr/local/bin/prepare-env.sh @@ -340,4 +322,3 @@ ENTRYPOINT ["tini", "-g", "--", "/usr/local/bin/prepare-env.sh"] # Use bash login shell for entrypoint in order # to automatically source user's .bashrc CMD ["start-notebook.sh"] -