diff --git a/Builds/CMake/RippledRelease.cmake b/Builds/CMake/RippledRelease.cmake index 3f23d792720..645ec6da620 100644 --- a/Builds/CMake/RippledRelease.cmake +++ b/Builds/CMake/RippledRelease.cmake @@ -54,6 +54,7 @@ if (is_root_project) Builds/containers/shared/update-rippled.sh Builds/containers/packaging/rpm/rippled.spec Builds/containers/packaging/rpm/build_rpm.sh + bin/getRippledInfo ) exclude_from_default (rpm_container) add_custom_target (rpm @@ -64,6 +65,7 @@ if (is_root_project) -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out "$<$:--volume=/etc/passwd:/etc/passwd;--volume=/etc/group:/etc/group;--user=${DOCKER_USER_ID}:${DOCKER_GROUP_ID}>" -t rippled-rpm-builder:${container_label} + /bin/bash -c "cp -pu rippled/Builds/containers/packaging/rpm/build_rpm.sh . && ./build_rpm.sh" VERBATIM USES_TERMINAL COMMAND_EXPAND_LISTS @@ -115,6 +117,7 @@ if (is_root_project) Builds/containers/packaging/dpkg/debian/rippled.preinst Builds/containers/packaging/dpkg/debian/rippled.prerm Builds/containers/packaging/dpkg/debian/rules + bin/getRippledInfo ) exclude_from_default (dpkg_container) add_custom_target (dpkg @@ -125,6 +128,7 @@ if (is_root_project) -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out "$<$:--volume=/etc/passwd:/etc/passwd;--volume=/etc/group:/etc/group;--user=${DOCKER_USER_ID}:${DOCKER_GROUP_ID}>" -t rippled-dpkg-builder:${container_label} + /bin/bash -c "cp -pu rippled/Builds/containers/packaging/dpkg/build_dpkg.sh . && ./build_dpkg.sh" VERBATIM USES_TERMINAL COMMAND_EXPAND_LISTS @@ -177,9 +181,6 @@ if (is_root_project) Builds/containers/ubuntu-builder/Dockerfile Builds/containers/ubuntu-builder/ubuntu_setup.sh Builds/containers/shared/build_deps.sh - Builds/containers/shared/rippled.service - Builds/containers/shared/update_sources.sh - Builds/containers/shared/update-rippled.sh ) exclude_from_default (ci_container) else () diff --git a/Builds/containers/centos-builder/Dockerfile b/Builds/containers/centos-builder/Dockerfile index 4e321430e73..26da564b525 100644 --- a/Builds/containers/centos-builder/Dockerfile +++ b/Builds/containers/centos-builder/Dockerfile @@ -34,9 +34,6 @@ RUN source scl_source enable devtoolset-7 python27 && \ # prep files for package building RUN mkdir -m 777 -p /opt/rippled_bld/pkg WORKDIR /opt/rippled_bld/pkg - -COPY packaging/rpm/rippled.spec ./ -COPY shared/update_sources.sh ./ RUN mkdir -m 777 ./rpmbuild RUN mkdir -m 777 ./rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} diff --git a/Builds/containers/gitlab-ci/pkgbuild.yml b/Builds/containers/gitlab-ci/pkgbuild.yml index 402a23582da..555b9d00333 100644 --- a/Builds/containers/gitlab-ci/pkgbuild.yml +++ b/Builds/containers/gitlab-ci/pkgbuild.yml @@ -11,10 +11,10 @@ variables: # these containers are built manually using the rippled # cmake build (container targets) and tagged/pushed so they # can be used here - RPM_CONTAINER_TAG: "2020-01-08" + RPM_CONTAINER_TAG: "2020-02-10" RPM_CONTAINER_NAME: "rippled-rpm-builder" RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}" - DPKG_CONTAINER_TAG: "2020-01-08" + DPKG_CONTAINER_TAG: "2020-02-10" DPKG_CONTAINER_NAME: "rippled-dpkg-builder" DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}" ARTIFACTORY_HOST: "artifactory.ops.ripple.com" diff --git a/Builds/containers/packaging/dpkg/build_dpkg.sh b/Builds/containers/packaging/dpkg/build_dpkg.sh index 74e469e59c4..feee2ce4f4d 100755 --- a/Builds/containers/packaging/dpkg/build_dpkg.sh +++ b/Builds/containers/packaging/dpkg/build_dpkg.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash set -ex +# make sure pkg source files are up to date with repo +cd /opt/rippled_bld/pkg +cp -pru rippled/Builds/containers/packaging/dpkg/debian/. debian/ +cp -pu rippled/Builds/containers/shared/rippled.service debian/ +cp -pu rippled/Builds/containers/shared/update_sources.sh . source update_sources.sh # Build the dpkg diff --git a/Builds/containers/packaging/dpkg/debian/rippled.install b/Builds/containers/packaging/dpkg/debian/rippled.install index eefdd152806..3ce9f60fb3a 100644 --- a/Builds/containers/packaging/dpkg/debian/rippled.install +++ b/Builds/containers/packaging/dpkg/debian/rippled.install @@ -1,6 +1,7 @@ opt/ripple/bin/rippled opt/ripple/bin/validator-keys opt/ripple/bin/update-rippled.sh +opt/ripple/bin/getRippledInfo opt/ripple/etc/rippled.cfg opt/ripple/etc/validators.txt opt/ripple/etc/update-rippled-cron diff --git a/Builds/containers/packaging/dpkg/debian/rules b/Builds/containers/packaging/dpkg/debian/rules index d076d846d13..a084a748ba6 100755 --- a/Builds/containers/packaging/dpkg/debian/rules +++ b/Builds/containers/packaging/dpkg/debian/rules @@ -32,6 +32,7 @@ override_dh_auto_install: cd bld && DESTDIR=../debian/tmp cmake --build . --target install -- -v install -D bld/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys install -D Builds/containers/shared/update-rippled.sh debian/tmp/opt/ripple/bin/update-rippled.sh + install -D bin/getRippledInfo debian/tmp/opt/ripple/bin/getRippledInfo install -D Builds/containers/shared/update-rippled-cron debian/tmp/opt/ripple/etc/update-rippled-cron install -D Builds/containers/shared/rippled-logrotate debian/tmp/etc/logrotate.d/rippled rm -rf debian/tmp/opt/ripple/lib64/cmake/date diff --git a/Builds/containers/packaging/rpm/build_rpm.sh b/Builds/containers/packaging/rpm/build_rpm.sh index df2455f6a1d..3a0b39d14db 100755 --- a/Builds/containers/packaging/rpm/build_rpm.sh +++ b/Builds/containers/packaging/rpm/build_rpm.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -ex +cd /opt/rippled_bld/pkg +cp -pu rippled/Builds/containers/packaging/rpm/rippled.spec . +cp -pu rippled/Builds/containers/shared/update_sources.sh . source update_sources.sh # Build the rpm diff --git a/Builds/containers/packaging/rpm/rippled.spec b/Builds/containers/packaging/rpm/rippled.spec index aeed6910f40..2edb23704dc 100644 --- a/Builds/containers/packaging/rpm/rippled.spec +++ b/Builds/containers/packaging/rpm/rippled.spec @@ -51,6 +51,7 @@ install -D rippled/bld.release/validator-keys/validator-keys ${RPM_BUILD_ROOT}%{ install -D ./rippled/Builds/containers/shared/rippled.service ${RPM_BUILD_ROOT}/usr/lib/systemd/system/rippled.service install -D ./rippled/Builds/containers/packaging/rpm/50-rippled.preset ${RPM_BUILD_ROOT}/usr/lib/systemd/system-preset/50-rippled.preset install -D ./rippled/Builds/containers/shared/update-rippled.sh ${RPM_BUILD_ROOT}%{_bindir}/update-rippled.sh +install -D ./rippled/bin/getRippledInfo ${RPM_BUILD_ROOT}%{_bindir}/getRippledInfo install -D ./rippled/Builds/containers/shared/update-rippled-cron ${RPM_BUILD_ROOT}%{_prefix}/etc/update-rippled-cron install -D ./rippled/Builds/containers/shared/rippled-logrotate ${RPM_BUILD_ROOT}/etc/logrotate.d/rippled install -d $RPM_BUILD_ROOT/var/log/rippled @@ -79,6 +80,7 @@ chown -R root:$GROUP_NAME %{_prefix}/etc/update-rippled-cron %{_bindir}/rippled /usr/local/bin/rippled %{_bindir}/update-rippled.sh +%{_bindir}/getRippledInfo %{_prefix}/etc/update-rippled-cron %{_bindir}/validator-keys %config(noreplace) %{_prefix}/etc/rippled.cfg diff --git a/Builds/containers/ubuntu-builder/Dockerfile b/Builds/containers/ubuntu-builder/Dockerfile index 35e54bbe439..e78c6b60e1f 100644 --- a/Builds/containers/ubuntu-builder/Dockerfile +++ b/Builds/containers/ubuntu-builder/Dockerfile @@ -27,15 +27,10 @@ ENV BOOST_ROOT="/opt/local/boost/_INSTALLED_" ENV OPENSSL_ROOT="/opt/local/openssl" # prep files for package building -RUN mkdir -m 777 -p /opt/rippled_bld/pkg +RUN mkdir -m 777 -p /opt/rippled_bld/pkg/debian +RUN update-alternatives --set gcc /usr/bin/gcc-8 WORKDIR /opt/rippled_bld/pkg -COPY packaging/dpkg/debian /opt/rippled_bld/pkg/debian/ -COPY shared/update_sources.sh ./ -COPY shared/rippled.service /opt/rippled_bld/pkg/debian/ - COPY packaging/dpkg/build_dpkg.sh ./ -RUN update-alternatives --set gcc /usr/bin/gcc-8 - CMD ./build_dpkg.sh diff --git a/bin/getInfoRippled.sh b/bin/getInfoRippled.sh deleted file mode 100755 index 240eed642e3..00000000000 --- a/bin/getInfoRippled.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -rippled_exe=/opt/ripple/bin/rippled -conf_file=/etc/opt/ripple/rippled.cfg - -while getopts ":e:c:" opt; do - case $opt in - e) - rippled_exe=${OPTARG} - ;; - c) - conf_file=${OPTARG} - ;; - \?) - echo "Invalid option: -$OPTARG" - esac -done - -tmp_loc=$(mktemp -d --tmpdir ripple_info.XXXX) -cd /tmp -chmod 751 ripple_info.* -cd ~ -echo ${tmp_loc} - -cleaned_conf=${tmp_loc}/cleaned_rippled_cfg.txt - -if [[ -f ${conf_file} ]] -then - db=$(sed -r -e 's/\/secretsecretsecretsecretmaybe/g' ${conf_file} |\ - awk -v OUT_FILE=${cleaned_conf} ' - BEGIN {skip=0; db_path="";print > OUT_FILE} - /^\[validation_seed\]/ {skip=1; next} - /^\[node_seed\]/ {skip=1; next} - /^\[validation_manifest\]/ {skip=1; next} - /^\[validator_token\]/ {skip=1; next} - /^\[.*\]/ {skip=0} - skip==1 {next} - save==1 {save=0;db_path=$0} - /^\[database_path\]/ {save=1} - {print >> OUT_FILE} - END {print db_path} - ') -fi - -echo "database_path: ${db}" -df ${db} > ${tmp_loc}/db_path_df.txt -echo - -# Send output from this script to a log file -## this captures any messages -## or errors from the script itself - -log_file=${tmp_loc}/get_info.log -exec 3>&1 1>>${log_file} 2>&1 - -## Send all stdout files to /tmp - -if [[ -x ${rippled_exe} ]] -then - pgrep rippled && \ - ${rippled_exe} --conf ${conf_file} \ - -- server_info > ${tmp_loc}/server_info.txt -fi - -df -h > ${tmp_loc}/free_disk_space.txt -cat /proc/meminfo > ${tmp_loc}/amount_mem.txt -cat /proc/swaps > ${tmp_loc}/swap_space.txt -ulimit -a > ${tmp_loc}/reported_current_limits.txt - -for dev_path in $(df | awk '$1 ~ /^\/dev\// {print $1}'); do - # strip numbers from end and remove '/dev/' - dev=$(basename ${dev_path%%[0-9]}) - if [[ "$(cat /sys/block/${dev}/queue/rotational)" = 0 ]] - then - echo "${dev} : SSD" >> ${tmp_loc}/is_ssd.txt - else - echo "${dev} : NO SSD" >> ${tmp_loc}/is_ssd.txt - fi -done - -pushd ${tmp_loc} -tar -czvf info-package.tar.gz *.txt *.log -popd - -echo "Use the following command on your local machine to download from your rippled instance: scp @:${tmp_loc}/info-package.tar.gz "| tee /dev/fd/3 - diff --git a/bin/getRippledInfo b/bin/getRippledInfo new file mode 100755 index 00000000000..4c7b1d8e820 --- /dev/null +++ b/bin/getRippledInfo @@ -0,0 +1,143 @@ +#!/usr/bin/env bash + +rippled_exe=/opt/ripple/bin/rippled +conf_file=/etc/opt/ripple/rippled.cfg + +while getopts ":e:c:" opt; do + case $opt in + e) + rippled_exe=${OPTARG} + ;; + c) + conf_file=${OPTARG} + ;; + \?) + echo "Invalid option: -$OPTARG" + exit -1 + esac +done + +tmp_loc=$(mktemp -d --tmpdir ripple_info.XXXXX) +chmod 751 ${tmp_loc} +awk_prog=${tmp_loc}/cfg.awk +summary_out=${tmp_loc}/rippled_info.md +printf "# rippled report info\n\n> generated at %s\n" "$(date -R)" > ${summary_out} + +function log_section { + printf "\n## %s\n" "$*" >> ${summary_out} + + while read -r l; do + echo " $l" >> ${summary_out} + done > ${awk_prog} + BEGIN {FS="[[:space:]]*=[[:space:]]*"; skip=0; db_path=""; print > OUT_FILE; split(exl,exa,"|")} + /^#/ {next} + save==2 && /^[[:space:]]*$/ {next} + /^\[.+\]$/ { + section=tolower(gensub(/^\[[[:space:]]*([a-zA-Z_]+)[[:space:]]*\]$/, "\\1", "g")) + skip = 0 + for (i in exa) { + if (section == exa[i]) + skip = 1 + } + if (section == "database_path") + save = 1 + } + skip==1 {next} + save==2 {save=0; db_path=$0} + save==1 {save=2} + $1 ~ /password/ {$0=$1"="} + {print >> OUT_FILE} + END {print db_path} +EOP + + db=$(\ + sed -r -e 's/\//g;s/^[[:space:]]*//;s/[[:space:]]*$//' ${conf_file} |\ + awk -v OUT_FILE=${cleaned_conf} -v exl="$(join_by '|' "${exclude[@]}")" -f ${awk_prog}) + rm ${awk_prog} + cat ${cleaned_conf} | log_section "cleaned config file" + rm ${cleaned_conf} + echo "${db}" | log_section "database path" + df ${db} | log_section "df: database" +fi + +# Send output from this script to a log file +## this captures any messages +## or errors from the script itself + +log_file=${tmp_loc}/get_info.log +exec 3>&1 1>>${log_file} 2>&1 + +## Send all stdout files to /tmp + +if [[ -x ${rippled_exe} ]] ; then + pgrep rippled && \ + ${rippled_exe} --conf ${conf_file} \ + -- server_info | log_section "server info" +fi + +cat /proc/meminfo | log_section "meminfo" +cat /proc/swaps | log_section "swap space" +ulimit -a | log_section "ulimit" + +if command -v lshw >/dev/null 2>&1 ; then + lshw 2>/dev/null | log_section "hardware info" +else + lscpu > ${tmp_loc}/hw_info.txt + hwinfo >> ${tmp_loc}/hw_info.txt + lspci >> ${tmp_loc}/hw_info.txt + lsblk >> ${tmp_loc}/hw_info.txt + cat ${tmp_loc}/hw_info.txt | log_section "hardware info" + rm ${tmp_loc}/hw_info.txt +fi + +if command -v iostat >/dev/null 2>&1 ; then + iostat -t -d -x 2 6 | log_section "iostat" +fi + +df -h | log_section "free disk space" +drives=($(df | awk '$1 ~ /^\/dev\// {print $1}' | xargs -n 1 basename)) +block_devs=($(ls /sys/block/)) +for d in "${drives[@]}"; do + for dev in "${block_devs[@]}"; do + #echo "D: [$d], DEV: [$dev]" + if [[ $d =~ $dev ]]; then + # this file (if exists) has 0 for SSD and 1 for HDD + if [[ "$(cat /sys/block/${dev}/queue/rotational 2>/dev/null)" == 0 ]] ; then + echo "${d} : SSD" >> ${tmp_loc}/is_ssd.txt + else + echo "${d} : NO SSD" >> ${tmp_loc}/is_ssd.txt + fi + fi + done +done + +if [[ -f ${tmp_loc}/is_ssd.txt ]] ; then + cat ${tmp_loc}/is_ssd.txt | log_section "SSD" + rm ${tmp_loc}/is_ssd.txt +fi + +cat ${log_file} | log_section "script log" + +cat << MSG | tee /dev/fd/3 +#################################################### + rippled info has been gathered. Please copy the + contents of ${summary_out} + to a github gist at https://gist.github.com/ + + PLEASE REVIEW THIS FILE FOR ANY SENSITIVE DATA + BEFORE POSTING! We have tried our best to omit + any sensitive information from this file, but you + should verify before posting. +#################################################### +MSG +