From 5946393bc3c5d807ee6f1ca47a90f6c27d8dedb0 Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Fri, 4 Sep 2020 20:04:54 +0530 Subject: [PATCH 01/10] chore(build): build and push images from libcstor Signed-off-by: mittachaitu --- .travis.yml | 37 +++-- Makefile.am | 2 +- build_image.sh | 124 +++++++++++++++ changelogs/unreleased/70-mittachaitu | 1 + cmd/Makefile.am | 1 + cmd/zrepl/Makefile.am | 26 +++ cmd/zrepl/zrepl.c | 226 +++++++++++++++++++++++++++ configure.ac | 3 + docker/Dockerfile | 24 +++ docker/Dockerfile.arm64 | 24 +++ docker/Dockerfile.base | 32 ++++ docker/Dockerfile.base.arm64 | 32 ++++ docker/entrypoint-poolimage.sh | 39 +++++ m4/headers.m4 | 1 + m4/user-frame-larger-than.m4 | 26 +++ 15 files changed, 585 insertions(+), 13 deletions(-) create mode 100755 build_image.sh create mode 100644 changelogs/unreleased/70-mittachaitu create mode 100644 cmd/Makefile.am create mode 100644 cmd/zrepl/Makefile.am create mode 100644 cmd/zrepl/zrepl.c create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.arm64 create mode 100644 docker/Dockerfile.base create mode 100644 docker/Dockerfile.base.arm64 create mode 100644 docker/entrypoint-poolimage.sh create mode 100644 m4/user-frame-larger-than.m4 diff --git a/.travis.yml b/.travis.yml index 6c6cac9..60c755a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,20 @@ env: global: # Travis limits maximum log size, we have to cut tests output - CSTOR_TEST_TRAVIS_LOG_MAX_LENGTH=800 +jobs: + include: + - os: linux + arch: amd64 + env: + - RUN_UZFS_TESTS=1 + - RELEASE_TAG_DOWNSTREAM=1 + - os: linux + arch: arm64 + env: + - RUN_UZFS_TESTS=0 + - RUN_ZTESTS=0 + - RELEASE_TAG_DOWNSTREAM=0 + before_install: - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - sudo apt-get update -qq @@ -55,6 +69,9 @@ install: - cd cstor - if [ ${TRAVIS_BRANCH} == "master" ]; then git checkout develop; else git checkout ${TRAVIS_BRANCH} || git checkout develop; fi - git branch + - sh autogen.sh + - ./configure --with-config=user --enable-debug --enable-uzfs=yes --with-jemalloc --with-fio=$PWD/../fio --with-libcstor=$PWD/../libcstor/include + - make -j4; - cd .. # return to libcstor code - cd libcstor @@ -63,22 +80,18 @@ install: - make -j4 - sudo make install - sudo ldconfig - - cd .. - - cd cstor - - sh autogen.sh - - ./configure --with-config=user --enable-debug --enable-uzfs=yes --with-jemalloc --with-fio=$PWD/../fio --with-libcstor=$PWD/../libcstor/include - - make -j4; - # return to libcstor code to run lint checks - - cd .. - - cd libcstor - make -f ../cstor/Makefile cstyle CSTORDIR=$PWD/../cstor # back to cstor for running further tests. - cd .. - cd cstor script: - - export FIO_SRCDIR=$PWD/../fio; - - sudo bash ./print_debug_info.sh & - - sudo bash ../libcstor/tests/cstor/script/test_uzfs.sh -T all || travis_terminate 1; + # run ztest and test supported zio backends + - if [ $RUN_UZFS_TESTS = 1 ]; then + export FIO_SRCDIR=$PWD/../fio; + sudo bash ./print_debug_info.sh & + sudo bash ../libcstor/tests/cstor/script/test_uzfs.sh -T all || travis_terminate 1; + fi + - ./build_image.sh || travis_terminate 1; # If this build is running due to travis release tag, then # go ahead and tag the dependent repo. # @@ -113,7 +126,7 @@ script: REL_SUFFIX="-$TAG_SUFFIX"; fi; REL_BRANCH=$(echo $(echo "$TRAVIS_TAG" | cut -d'-' -f1 | rev | cut -d'.' -f2- | rev).x$REL_SUFFIX) ; - ./buildscripts/git-release "$REPO_ORG/cstor" "$TRAVIS_TAG" "$REL_BRANCH" || travis_terminate 1; + ./buildscripts/git-release "$REPO_ORG/istgt" "$TRAVIS_TAG" "$REL_BRANCH" || travis_terminate 1; fi after_failure: - find /var/tmp/test_results/current/log -type f -name '*' -printf "%f\n" -exec cut -c -$CSTOR_TEST_TRAVIS_LOG_MAX_LENGTH {} \; diff --git a/Makefile.am b/Makefile.am index 1f665e3..f992334 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,4 @@ ACLOCAL_AMFLAGS= -I m4 AUTOMAKE_OPTIONS = foreign -SUBDIRS = src include +SUBDIRS = src include cmd diff --git a/build_image.sh b/build_image.sh new file mode 100755 index 0000000..399477b --- /dev/null +++ b/build_image.sh @@ -0,0 +1,124 @@ +#!/bin/bash +set -e + +pwd + +# Determine the arch/os we're building for +ARCH=$(uname -m) + +# Build cstor +cd ../cstor +make clean +sh autogen.sh +./configure --enable-uzfs=yes --with-config=user --with-jemalloc --with-libcstor=$PWD/../libcstor/include +make clean +make -j$(nproc) + +# Build libcstor +cd ../libcstor +make clean +sh autogen.sh +./configure --with-zfs-headers=$PWD/../cstor/include --with-spl-headers=$PWD/../cstor/lib/libspl/include +make -j$(nproc) +sudo make install +sudo ldconfig + +# The images can be pushed to any docker/image registeries +# like docker hub, quay. The registries are specified in +# the `build/push` script. +# +# The images of a project or company can then be grouped +# or hosted under a unique organization key like `openebs` +# +# Each component (container) will be pushed to a unique +# repository under an organization. +# Putting all this together, an unique uri for a given +# image comprises of: +# //: +# +# IMAGE_ORG can be used to customize the organization +# under which images should be pushed. +# By default the organization name is `openebs`. + +if [ -z "${IMAGE_ORG}" ]; then + IMAGE_ORG="openebs" +fi + +# Specify the date of build +DBUILD_DATE=$(date +'%Y-%m-%dT%H:%M:%SZ') + +# Specify the docker arg for repository url +if [ -z "${DBUILD_REPO_URL}" ]; then + DBUILD_REPO_URL="https://github.com/openebs/cstor" +fi + +# Specify the docker arg for website url +if [ -z "${DBUILD_SITE_URL}" ]; then + DBUILD_SITE_URL="https://openebs.io" +fi + +DBUILD_ARGS="--build-arg DBUILD_DATE=${DBUILD_DATE} --build-arg DBUILD_REPO_URL=${DBUILD_REPO_URL} --build-arg DBUILD_SITE_URL=${DBUILD_SITE_URL} --build-arg ARCH=${ARCH}" + + +if [ "${ARCH}" = "x86_64" ]; then + REPO_NAME="$IMAGE_ORG/cstor-base" + DOCKERFILE_BASE="Dockerfile.base" + DOCKERFILE="Dockerfile" +elif [ "${ARCH}" = "aarch64" ]; then + REPO_NAME="$IMAGE_ORG/cstor-base-arm64" + DOCKERFILE_BASE="Dockerfile.base.arm64" + DOCKERFILE="Dockerfile.arm64" +else + echo "${ARCH} is not supported" + exit 1 +fi + +mkdir -p ./docker/zfs/bin +mkdir -p ./docker/zfs/lib + +cp cmd/zrepl/.libs/zrepl ./docker/zfs/bin +cp ../cstor/cmd/zpool/.libs/zpool ./docker/zfs/bin +cp ../cstor/cmd/zfs/.libs/zfs ./docker/zfs/bin +cp ../cstor/cmd/zstreamdump/.libs/zstreamdump ./docker/zfs/bin + +cp ../cstor/lib/libzpool/.libs/*.so* ./docker/zfs/lib +cp ../cstor/lib/libuutil/.libs/*.so* ./docker/zfs/lib +cp ../cstor/lib/libnvpair/.libs/*.so* ./docker/zfs/lib +cp ../cstor/lib/libzfs/.libs/*.so* ./docker/zfs/lib +cp ../cstor/lib/libzfs_core/.libs/*.so* ./docker/zfs/lib +cp src/.libs/*.so* ./docker/zfs/lib + +sudo docker version +sudo docker build --help + +curl --fail https://raw.githubusercontent.com/openebs/charts/gh-pages/scripts/release/buildscripts/push > ./docker/push +chmod +x ./docker/push + +## Building image for cstor-base +echo "Build image ${REPO_NAME}:ci with BUILD_DATE=${DBUILD_DATE}" +cd docker && \ + sudo docker build -f ${DOCKERFILE_BASE} -t ${REPO_NAME}:ci ${DBUILD_ARGS} . && \ + DIMAGE=${REPO_NAME} ./push && \ + cd .. +if [ $? -ne 0 ]; then + echo "Failed to run push script for ${REPO_NAME}" + exit 1 +fi + +if [ "${ARCH}" = "x86_64" ]; then + REPO_NAME="$IMAGE_ORG/cstor-pool" +elif [ "${ARCH}" = "aarch64" ]; then + REPO_NAME="$IMAGE_ORG/cstor-pool-arm64" +fi + +echo "Build image ${REPO_NAME}:ci with BUILD_DATE=${DBUILD_DATE}" +cd docker && \ + sudo docker build -f ${DOCKERFILE} -t ${REPO_NAME}:ci ${DBUILD_ARGS} . && \ + DIMAGE=${REPO_NAME} ./push && \ + cd .. +if [ $? -ne 0 ]; then + echo "Failed to run push script for ${REPO_NAME}" + exit 1 +fi + +rm -rf ./docker/zfs diff --git a/changelogs/unreleased/70-mittachaitu b/changelogs/unreleased/70-mittachaitu new file mode 100644 index 0000000..c1e09be --- /dev/null +++ b/changelogs/unreleased/70-mittachaitu @@ -0,0 +1 @@ +chore(build): build and push images to repositories from libcstor diff --git a/cmd/Makefile.am b/cmd/Makefile.am new file mode 100644 index 0000000..44d4faa --- /dev/null +++ b/cmd/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = zrepl diff --git a/cmd/zrepl/Makefile.am b/cmd/zrepl/Makefile.am new file mode 100644 index 0000000..28b5b92 --- /dev/null +++ b/cmd/zrepl/Makefile.am @@ -0,0 +1,26 @@ +# -Wnoformat-truncation to get rid of compiler warning for unchecked +# truncating snprintfs on gcc 7.1.1. +AM_CFLAGS = $(FRAME_LARGER_THAN) +# Dynamic symbol table is needed for printing symbol names in stack trace +AM_LDFLAGS = -rdynamic + +AM_CPPFLAGS = -D_GNU_SOURCE + +DEFAULT_INCLUDES = \ + -I$(top_srcdir)/include \ + ${ZFS_SRC} \ + ${SPL_SRC} + +sbin_PROGRAMS = zrepl + +zrepl_SOURCES = \ + zrepl.c + +zrepl_LDADD = \ + $(zfssrc)/../lib/libnvpair/libnvpair.la \ + $(zfssrc)/../lib/libuutil/libuutil.la \ + $(zfssrc)/../lib/libzpool/libzpool.la \ + $(zfssrc)/../lib/libzfs/libzfs.la \ + $(zfssrc)/../lib/libzfs_core/libzfs_core.la + +zrepl_LDADD += -lcstor diff --git a/cmd/zrepl/zrepl.c b/cmd/zrepl/zrepl.c new file mode 100644 index 0000000..e9bab78 --- /dev/null +++ b/cmd/zrepl/zrepl.c @@ -0,0 +1,226 @@ +/* + * Copyright © 2017-2019 The OpenEBS Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ZAP_UPDATE_TIME_INTERVAL 2 + +extern unsigned long zfs_arc_max; +extern unsigned long zfs_arc_min; +extern int zfs_autoimport_disable; +extern int zfs_do_write_coalesce; + +#if DEBUG +inject_error_t inject_error; +#endif + +kthread_t *conn_accpt_thread; +kthread_t *uzfs_timer_thread; +kthread_t *mgmt_conn_thread; + +static void +zrepl_svc_run(void) +{ + mgmt_conn_thread = zk_thread_create(NULL, 0, + uzfs_zvol_mgmt_thread, NULL, 0, NULL, + TS_RUN, 0, PTHREAD_CREATE_DETACHED); + VERIFY3P(mgmt_conn_thread, !=, NULL); + + conn_accpt_thread = zk_thread_create(NULL, 0, + uzfs_zvol_io_conn_acceptor, NULL, 0, NULL, TS_RUN, + 0, PTHREAD_CREATE_DETACHED); + VERIFY3P(conn_accpt_thread, !=, NULL); + + uzfs_timer_thread = zk_thread_create(NULL, 0, + (thread_func_t)uzfs_zvol_timer_thread, NULL, 0, NULL, TS_RUN, + 0, PTHREAD_CREATE_DETACHED); + VERIFY3P(uzfs_timer_thread, !=, NULL); +} + +/* + * Print a stack trace before program exits. + */ +static void +fatal_handler(int sig) +{ + void *array[20]; + size_t size; + + fprintf(stderr, "Fatal signal received: %d\n", sig); + fprintf(stderr, "Stack trace:\n"); + + size = backtrace(array, 20); + backtrace_symbols_fd(array, size, STDERR_FILENO); + + /* + * Hand over the sig for default processing to system to generate + * a coredump + */ + signal(sig, SIG_DFL); + kill(getpid(), sig); +} + +/* + * We would like to do a graceful shutdown here to avoid recovery actions + * when pool is imported next time. However we don't want to call export + * which does a bunch of other things which are not necessary (freeing + * memory resources etc.), since we run in userspace. + * + * mutex_enter(&spa_namespace_lock); + * while ((spa = spa_next(NULL)) != NULL) { + * strlcpy(spaname, spa_name(spa), sizeof (spaname)); + * mutex_exit(&spa_namespace_lock); + * LOG_INFO("Exporting pool %s", spaname); + * spa_export(spaname, NULL, B_TRUE, B_FALSE); + * mutex_enter(&spa_namespace_lock); + * } + * mutex_exit(&spa_namespace_lock); + * + * For now we keep it simple and just exit. + */ +static void +exit_handler(int sig) +{ + LOG_INFO("Caught SIGTERM. Exiting..."); + exit(0); +} + +/* + * Main function for replica. + */ +int +main(int argc, char **argv) +{ + int rc; + char *env; + int fd = open(LOCK_FILE, O_CREAT | O_RDWR, 0644); + if (fd < 0) { + fprintf(stderr, "%s open failed: %s\n", LOCK_FILE, + strerror(errno)); + return (-1); + } + if (flock(fd, LOCK_EX) < 0) { + fprintf(stderr, "flock failed: %s\n", strerror(errno)); + return (-1); + } + + /* Use opt parsing lib if we have more options */ + zrepl_log_level = LOG_LEVEL_INFO; + if (argc == 3 && strcmp(argv[1], "-l") == 0) { + if (strcmp(argv[2], "debug") == 0) + zrepl_log_level = LOG_LEVEL_DEBUG; + else if (strcmp(argv[2], "info") == 0) + zrepl_log_level = LOG_LEVEL_INFO; + else if (strcmp(argv[2], "error") == 0) + zrepl_log_level = LOG_LEVEL_ERR; + else { + fprintf(stderr, "Log level should be one of " + "\"debug\", \"info\" or \"error\"\n"); + return (-1); + } + } + + if (getenv("CONFIG_LOAD_ENABLE") != NULL) { + LOG_INFO("auto importing pools by reading zpool.cache files"); + zfs_autoimport_disable = 0; + } else { + LOG_INFO("disabled auto import (reading of zpool.cache)"); + zfs_autoimport_disable = 1; + } + + zfs_do_write_coalesce = 1; + env = getenv("DISABLE_WRITE_COALESCE"); + if (env != NULL) { + if (strcmp(env, "1") == 0) { + LOG_INFO("Disabling write IOs coalescing"); + zfs_do_write_coalesce = 0; + } + } + + uzfs_write_size = 0; + env = getenv("UZFS_WRITE_SIZE"); + if (env != NULL) { + uzfs_write_size = atoi(env); + LOG_INFO("uzfs write size = %d", uzfs_write_size); + } + + SLIST_INIT(&uzfs_mgmt_conns); + mutex_init(&conn_list_mtx, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&async_tasks_mtx, NULL, MUTEX_DEFAULT, NULL); + + zinfo_create_hook = &zinfo_create_cb; + zinfo_destroy_hook = &zinfo_destroy_cb; + + io_server_port = IO_SERVER_PORT; + rebuild_io_server_port = REBUILD_IO_SERVER_PORT; + + io_receiver = uzfs_zvol_io_receiver; + rebuild_scanner = uzfs_zvol_rebuild_scanner; + dw_replica_fn = uzfs_zvol_rebuild_dw_replica; + + SLIST_INIT(&uzfs_mgmt_conns); + + rc = uzfs_init(); + if (rc != 0) { + LOG_ERR("initialization errored: %d", rc); + return (-1); + } + + /* Ignore SIGPIPE signal */ + signal(SIGPIPE, SIG_IGN); + signal(SIGTERM, exit_handler); + signal(SIGABRT, fatal_handler); + signal(SIGFPE, fatal_handler); + signal(SIGSEGV, fatal_handler); + signal(SIGBUS, fatal_handler); + signal(SIGILL, fatal_handler); + + if (libuzfs_ioctl_init() < 0) { + LOG_ERR("Failed to initialize libuzfs ioctl"); + goto initialize_error; + } + + SLIST_INIT(&uzfs_mgmt_conns); + mutex_init(&conn_list_mtx, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&async_tasks_mtx, NULL, MUTEX_DEFAULT, NULL); + zrepl_svc_run(); + zrepl_monitor_errors(); + +initialize_error: + uzfs_fini(); + return (-1); +} diff --git a/configure.ac b/configure.ac index f68ed42..f4b532d 100644 --- a/configure.ac +++ b/configure.ac @@ -28,9 +28,12 @@ AC_HEADER_STDC CSTOR_AC_DEBUG CSTOR_AC_CONFIG_ZFS +CSTOR_AC_CONFIG_USER_FRAME_LARGER_THAN AC_CONFIG_FILES([ Makefile + cmd/Makefile + cmd/zrepl/Makefile src/Makefile include/Makefile include/sys/Makefile diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..f85d5c0 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,24 @@ +# +# This Dockerfile builds cstor main container running zrepl from base image +# + +FROM openebs/cstor-base:ci + +COPY entrypoint-poolimage.sh /usr/local/bin/ + +RUN chmod +x /usr/local/bin/entrypoint-poolimage.sh + +ARG ARCH +ARG DBUILD_DATE +ARG DBUILD_REPO_URL +ARG DBUILD_SITE_URL + +LABEL org.label-schema.name="cstor" +LABEL org.label-schema.description="OpenEBS cStor" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$DBUILD_DATE +LABEL org.label-schema.vcs-url=$DBUILD_REPO_URL +LABEL org.label-schema.url=$DBUILD_SITE_URL + +ENTRYPOINT entrypoint-poolimage.sh +EXPOSE 7676 diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 new file mode 100644 index 0000000..d5fb56a --- /dev/null +++ b/docker/Dockerfile.arm64 @@ -0,0 +1,24 @@ +# +# This Dockerfile builds cstor main container running zrepl from base image +# + +FROM openebs/cstor-base-arm64:ci + +COPY entrypoint-poolimage.sh /usr/local/bin/ + +RUN chmod +x /usr/local/bin/entrypoint-poolimage.sh + +ARG ARCH +ARG DBUILD_DATE +ARG DBUILD_REPO_URL +ARG DBUILD_SITE_URL + +LABEL org.label-schema.name="cstor" +LABEL org.label-schema.description="OpenEBS cStor" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$DBUILD_DATE +LABEL org.label-schema.vcs-url=$DBUILD_REPO_URL +LABEL org.label-schema.url=$DBUILD_SITE_URL + +ENTRYPOINT entrypoint-poolimage.sh +EXPOSE 7676 diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base new file mode 100644 index 0000000..3755b0e --- /dev/null +++ b/docker/Dockerfile.base @@ -0,0 +1,32 @@ +# +# This Dockerfile builds a recent base image containing cstor binaries and +# libraries. +# + +FROM openebs/cstor-ubuntu:xenial-20190515 + +RUN apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get update && apt-get install -y \ + curl tcpdump dnsutils iputils-ping \ + libaio1 libaio-dev \ + libkqueue-dev libssl1.0.0 rsyslog net-tools gdb apt-utils \ + sed libjemalloc-dev +RUN apt-get -y install apt-file && apt-file update + +COPY zfs/bin/* /usr/local/bin/ +COPY zfs/lib/* /usr/lib/ + +ARG ARCH +ARG DBUILD_DATE +ARG DBUILD_REPO_URL +ARG DBUILD_SITE_URL + +LABEL org.label-schema.name="cstor" +LABEL org.label-schema.description="OpenEBS cStor" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$DBUILD_DATE +LABEL org.label-schema.vcs-url=$DBUILD_REPO_URL +LABEL org.label-schema.url=$DBUILD_SITE_URL + +EXPOSE 7676 diff --git a/docker/Dockerfile.base.arm64 b/docker/Dockerfile.base.arm64 new file mode 100644 index 0000000..33d7a57 --- /dev/null +++ b/docker/Dockerfile.base.arm64 @@ -0,0 +1,32 @@ +# +# This Dockerfile builds a recent base image containing cstor binaries and +# libraries. +# + +FROM openebs/arm64v8-ubuntu:xenial-20200326 + +RUN apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get update && apt-get install -y \ + curl tcpdump dnsutils iputils-ping \ + libaio1 libaio-dev \ + libssl1.0.0 rsyslog net-tools gdb apt-utils \ + sed libjemalloc-dev +RUN apt-get -y install apt-file && apt-file update + +COPY zfs/bin/* /usr/local/bin/ +COPY zfs/lib/* /usr/lib/ + +ARG ARCH +ARG DBUILD_DATE +ARG DBUILD_REPO_URL +ARG DBUILD_SITE_URL + +LABEL org.label-schema.name="cstor" +LABEL org.label-schema.description="OpenEBS cStor" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$DBUILD_DATE +LABEL org.label-schema.vcs-url=$DBUILD_REPO_URL +LABEL org.label-schema.url=$DBUILD_SITE_URL + +EXPOSE 7676 diff --git a/docker/entrypoint-poolimage.sh b/docker/entrypoint-poolimage.sh new file mode 100644 index 0000000..be8ced0 --- /dev/null +++ b/docker/entrypoint-poolimage.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +set -o errexit +trap 'call_exit $LINE_NO' EXIT + +call_exit() +{ +echo "at call_exit.." +echo "exit code:" $? +echo "reference: " $0 +} + + +if [ -z "$LOGLEVEL" ]; then + LOGLEVEL=info +fi + +# Disabling coredumps by default in the shell where zrepl runs +if [ -z "$ENABLE_COREDUMP" ]; then + echo "Disabling dumping core" + ulimit -c 0 +else + echo "Enabling coredumps" + ulimit -c unlimited + ## /var/openebs is mounted as persistent directory on + ## host machine + cd /var/openebs/cstor-pool || exit + mkdir -p core + cd core + +fi +# ulimit being shell specific, ulimit -c in container shows as unlimited + + +echo "sleeping for 2 sec" +sleep 2 +ARCH=$(uname -m) +export LD_PRELOAD=/usr/lib/${ARCH}-linux-gnu/libjemalloc.so +exec /usr/local/bin/zrepl -l $LOGLEVEL diff --git a/m4/headers.m4 b/m4/headers.m4 index a4020e1..43808d5 100644 --- a/m4/headers.m4 +++ b/m4/headers.m4 @@ -8,6 +8,7 @@ AC_DEFUN([CSTOR_AC_CONFIG_ZFS], [ AC_MSG_CHECKING([zfs header file source]) AC_MSG_RESULT([$ZFS_SRC]) AC_SUBST(ZFS_SRC) + AC_SUBST(zfssrc) AC_DEFINE_UNQUOTED([ZFS_SRC], [$ZFS_SRC], [zfs header file source]) diff --git a/m4/user-frame-larger-than.m4 b/m4/user-frame-larger-than.m4 new file mode 100644 index 0000000..3818dd6 --- /dev/null +++ b/m4/user-frame-larger-than.m4 @@ -0,0 +1,26 @@ +dnl # Check if gcc supports -Wframe-larger-than= option. +AC_DEFUN([CSTOR_AC_CONFIG_USER_FRAME_LARGER_THAN], [ + AC_MSG_CHECKING([for -Wframe-larger-than= support]) + + AS_IF([echo "$CFLAGS" | grep O0 >/dev/null], + [ + echo "disabled because -O0" + ], + [ + saved_flags="$CFLAGS" + CFLAGS="$CFLAGS -Wframe-larger-than=1024" + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], + [ + FRAME_LARGER_THAN=-Wframe-larger-than=1024 + AC_MSG_RESULT([yes]) + ], + [ + FRAME_LARGER_THAN= + AC_MSG_RESULT([no]) + ]) + + CFLAGS="$saved_flags" + AC_SUBST([FRAME_LARGER_THAN]) + ]) +]) From a37350b96bf4eceed8bf7b715771d8fdf72d6948 Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Sat, 5 Sep 2020 10:53:02 +0530 Subject: [PATCH 02/10] This commit fixes the compilation issues in Travis Signed-off-by: mittachaitu --- .travis.yml | 40 ++++++++++++++++++++++++++++++---------- Makefile.am | 2 +- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 60c755a..9d88ecc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,11 +27,19 @@ before_install: - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - sudo apt-get update -qq - sudo apt-get install --yes -qq gcc-6 g++-6 - - sudo apt-get install --yes -qq build-essential autoconf libtool gawk alien fakeroot linux-headers-$(uname -r) libaio-dev jq + - sudo apt-get install --yes -qq build-essential autoconf libtool gawk alien fakeroot libaio-dev jq + # linux-header package name is different on arm. + - if [ "$TRAVIS_CPU_ARCH" == "arm64" ]; then + sudo apt-get install --yes -qq linux-headers-generic; + else + sudo apt-get install --yes -qq linux-headers-$(uname -r); + fi - sudo apt-get install --yes -qq zlib1g-dev uuid-dev libattr1-dev libblkid-dev libselinux-dev libudev-dev libssl-dev libjson-c-dev - sudo apt-get install --yes -qq lcov libjemalloc-dev - # packages for tests - - sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio + # packages for tests - only on amd64 + - if [ "$TRAVIS_CPU_ARCH" == "amd64" ]; then + sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio; + fi - sudo apt-get install --yes -qq libgtest-dev cmake # packages for debugging - sudo apt-get install gdb @@ -64,23 +72,32 @@ install: - ./configure - make -j4 - cd .. - # we need cstor code + # we need cstor code for compiling libcstor(uZFS feature) - git clone https://github.com/openebs/cstor.git - cd cstor - if [ ${TRAVIS_BRANCH} == "master" ]; then git checkout develop; else git checkout ${TRAVIS_BRANCH} || git checkout develop; fi - git branch - - sh autogen.sh - - ./configure --with-config=user --enable-debug --enable-uzfs=yes --with-jemalloc --with-fio=$PWD/../fio --with-libcstor=$PWD/../libcstor/include - - make -j4; - - cd .. - # return to libcstor code - - cd libcstor + # Return to libcstor code base + - popd - sh autogen.sh - ./configure --enable-debug --with-zfs-headers=$PWD/../cstor/include --with-spl-headers=$PWD/../cstor/lib/libspl/include - make -j4 - sudo make install - sudo ldconfig + # Return to cstor code + - cd .. + - cd cstor + - sh autogen.sh + - ./configure --with-config=user --enable-debug --enable-uzfs=yes --with-jemalloc --with-fio=$PWD/../fio --with-libcstor=$PWD/../libcstor/include + - make -j4; + # Return to libcstor code to complie zrepl which contains main and to run lint checks + - cd .. + - cd libcstor - make -f ../cstor/Makefile cstyle CSTORDIR=$PWD/../cstor + # Go to zrepl directory to build zrepl related targets + - cd cmd/zrepl + - make + - cd ../../ # back to cstor for running further tests. - cd .. - cd cstor @@ -91,6 +108,9 @@ script: sudo bash ./print_debug_info.sh & sudo bash ../libcstor/tests/cstor/script/test_uzfs.sh -T all || travis_terminate 1; fi + # Go back to libcstor to build images + - cd .. + - cd libcstor - ./build_image.sh || travis_terminate 1; # If this build is running due to travis release tag, then # go ahead and tag the dependent repo. diff --git a/Makefile.am b/Makefile.am index f992334..1f665e3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,4 @@ ACLOCAL_AMFLAGS= -I m4 AUTOMAKE_OPTIONS = foreign -SUBDIRS = src include cmd +SUBDIRS = src include From db29cf2c23d723ef916067755b80b5d5328bf552 Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Tue, 8 Sep 2020 19:13:51 +0530 Subject: [PATCH 03/10] This commit updates the fio version and also fixes the build_image.sh script Signed-off-by: mittachaitu --- .travis.yml | 6 +++--- build_image.sh | 21 ++++++++++++++------- cmd/zrepl/.gitignore | 1 + cmd/zrepl/Makefile.am | 16 ++++++++++++---- tests/cstor/script/test_uzfs.sh | 5 ++++- 5 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 cmd/zrepl/.gitignore diff --git a/.travis.yml b/.travis.yml index 9d88ecc..2fe1aa3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,6 @@ jobs: arch: arm64 env: - RUN_UZFS_TESTS=0 - - RUN_ZTESTS=0 - RELEASE_TAG_DOWNSTREAM=0 before_install: @@ -61,7 +60,7 @@ install: # we need fio repo to build zfs replica fio engine - git clone https://github.com/axboe/fio - cd fio - - git checkout fio-3.7 + - git checkout fio-3.9 - ./configure - make -j4 - cd .. @@ -90,9 +89,10 @@ install: - sh autogen.sh - ./configure --with-config=user --enable-debug --enable-uzfs=yes --with-jemalloc --with-fio=$PWD/../fio --with-libcstor=$PWD/../libcstor/include - make -j4; - # Return to libcstor code to complie zrepl which contains main and to run lint checks + # Return to libcstor code to complie zrepl which contains main process and to run lint checks - cd .. - cd libcstor + - make check_license - make -f ../cstor/Makefile cstyle CSTORDIR=$PWD/../cstor # Go to zrepl directory to build zrepl related targets - cd cmd/zrepl diff --git a/build_image.sh b/build_image.sh index 399477b..52f9ced 100755 --- a/build_image.sh +++ b/build_image.sh @@ -6,6 +6,16 @@ pwd # Determine the arch/os we're building for ARCH=$(uname -m) +# Build libcstor +cd ../libcstor +make clean +sh autogen.sh +./configure --with-zfs-headers=$PWD/../cstor/include --with-spl-headers=$PWD/../cstor/lib/libspl/include +make -j$(nproc) +sudo make install +sudo ldconfig + + # Build cstor cd ../cstor make clean @@ -14,14 +24,11 @@ sh autogen.sh make clean make -j$(nproc) -# Build libcstor -cd ../libcstor +# Build zrepl target and docker files exist in libcstor +cd ../libcstor/cmd/zrepl make clean -sh autogen.sh -./configure --with-zfs-headers=$PWD/../cstor/include --with-spl-headers=$PWD/../cstor/lib/libspl/include -make -j$(nproc) -sudo make install -sudo ldconfig +make +cd ../../ # The images can be pushed to any docker/image registeries # like docker hub, quay. The registries are specified in diff --git a/cmd/zrepl/.gitignore b/cmd/zrepl/.gitignore new file mode 100644 index 0000000..6ba2b05 --- /dev/null +++ b/cmd/zrepl/.gitignore @@ -0,0 +1 @@ +/zrepl diff --git a/cmd/zrepl/Makefile.am b/cmd/zrepl/Makefile.am index 28b5b92..1b5f67b 100644 --- a/cmd/zrepl/Makefile.am +++ b/cmd/zrepl/Makefile.am @@ -4,7 +4,15 @@ AM_CFLAGS = $(FRAME_LARGER_THAN) # Dynamic symbol table is needed for printing symbol names in stack trace AM_LDFLAGS = -rdynamic -AM_CPPFLAGS = -D_GNU_SOURCE +AM_CPPFLAGS = -D_GNU_SOURCE -D__EXTENSIONS__ -D_REENTRANT +AM_CPPFLAGS += -D_POSIX_PTHREAD_SEMANTICS -D_FILE_OFFSET_BITS=64 +AM_CPPFLAGS += -D_LARGEFILE64_SOURCE -DHAVE_LARGE_STACKS=1 +AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\" +AM_CPPFLAGS += -DLIBEXECDIR=\"$(libexecdir)\" +AM_CPPFLAGS += -DRUNSTATEDIR=\"$(runstatedir)\" +AM_CPPFLAGS += -DSBINDIR=\"$(sbindir)\" +AM_CPPFLAGS += -DSYSCONFDIR=\"$(sysconfdir)\" +AM_CPPFLAGS += $(CODE_COVERAGE_CPPFLAGS) $(UZFS_CFLAGS) DEFAULT_INCLUDES = \ -I$(top_srcdir)/include \ @@ -18,9 +26,9 @@ zrepl_SOURCES = \ zrepl_LDADD = \ $(zfssrc)/../lib/libnvpair/libnvpair.la \ - $(zfssrc)/../lib/libuutil/libuutil.la \ - $(zfssrc)/../lib/libzpool/libzpool.la \ + $(zfssrc)/../lib/libuutil/libuutil.la \ + $(zfssrc)/../lib/libzpool/libzpool.la \ $(zfssrc)/../lib/libzfs/libzfs.la \ $(zfssrc)/../lib/libzfs_core/libzfs_core.la -zrepl_LDADD += -lcstor +zrepl_LDADD += -lcstor -ludev -luuid -lz diff --git a/tests/cstor/script/test_uzfs.sh b/tests/cstor/script/test_uzfs.sh index bd07a91..74099a4 100755 --- a/tests/cstor/script/test_uzfs.sh +++ b/tests/cstor/script/test_uzfs.sh @@ -22,7 +22,10 @@ fi ZPOOL="$SRC_PATH/cmd/zpool/zpool" ZFS="$SRC_PATH/cmd/zfs/zfs" ZDB="$SRC_PATH/cmd/zdb/zdb" -ZREPL="$SRC_PATH/cmd/zrepl/zrepl" +## Since zrepl has been moved from cstor to libcstor +## path also needs to be updated +## NOTE: It is assumed that cstor and libcstor will exist in same parent directory +ZREPL="$SRC_PATH/../libcstor/cmd/zrepl/zrepl" GTEST_UZFS="$SRC_PATH/tests/cstor/gtest/test_uzfs" GTEST_ZFS="$SRC_PATH/tests/cstor/gtest/test_zfs" GTEST_ZREPL_PROT="$SRC_PATH/tests/cstor/gtest/test_zrepl_prot" From 067fcb34cd8f185b771fdca5613de744eacaa70a Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Tue, 8 Sep 2020 19:41:20 +0530 Subject: [PATCH 04/10] This commit adds license content to .am, .sh and Docker files Signed-off-by: mittachaitu --- .travis.yml | 2 +- Makefile.am | 24 ++++++++++++++++++++++++ autogen.sh | 14 ++++++++++++++ build_image.sh | 15 +++++++++++++++ cmd/Makefile.am | 14 ++++++++++++++ cmd/zrepl/Makefile.am | 14 ++++++++++++++ docker/Dockerfile | 14 ++++++++++++++ docker/Dockerfile.arm64 | 14 ++++++++++++++ docker/Dockerfile.base | 14 ++++++++++++++ docker/Dockerfile.base.arm64 | 14 ++++++++++++++ docker/entrypoint-poolimage.sh | 14 ++++++++++++++ include/Makefile.am | 14 ++++++++++++++ include/arch/arm/Makefile.am | 14 ++++++++++++++ include/arch/x86/Makefile.am | 14 ++++++++++++++ include/generic/Makefile.am | 14 ++++++++++++++ include/sys/Makefile.am | 14 ++++++++++++++ src/Makefile.am | 14 ++++++++++++++ 17 files changed, 236 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2fe1aa3..90c4b70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,7 +92,7 @@ install: # Return to libcstor code to complie zrepl which contains main process and to run lint checks - cd .. - cd libcstor - - make check_license + - make check-license - make -f ../cstor/Makefile cstyle CSTORDIR=$PWD/../cstor # Go to zrepl directory to build zrepl related targets - cd cmd/zrepl diff --git a/Makefile.am b/Makefile.am index 1f665e3..64494d8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,28 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ACLOCAL_AMFLAGS= -I m4 AUTOMAKE_OPTIONS = foreign SUBDIRS = src include + +check-license: + @echo ">> checking license header" + licRes=$$(for file in $$(find . -type f -regex '.*\.sh\|.*\.c\|.*\.cc\|.*\.am\|.*\.h\|.*Docker.*' ! -path ./config.h) ; do \ + awk 'NR<=5' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \ + done); \ + if [ -n "$${licRes}" ]; then \ + echo "license header checking failed:"; echo "$${licRes}"; \ + exit 1; \ + fi diff --git a/autogen.sh b/autogen.sh index 427394a..f90f017 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,4 +1,18 @@ #!/bin/sh +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + autoreconf -fiv rm -Rf autom4te.cache diff --git a/build_image.sh b/build_image.sh index 52f9ced..0d097f7 100755 --- a/build_image.sh +++ b/build_image.sh @@ -1,4 +1,19 @@ #!/bin/bash + +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -e pwd diff --git a/cmd/Makefile.am b/cmd/Makefile.am index 44d4faa..3cbe2a6 100644 --- a/cmd/Makefile.am +++ b/cmd/Makefile.am @@ -1 +1,15 @@ +# Copyright © 2020 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + SUBDIRS = zrepl diff --git a/cmd/zrepl/Makefile.am b/cmd/zrepl/Makefile.am index 1b5f67b..95ef84a 100644 --- a/cmd/zrepl/Makefile.am +++ b/cmd/zrepl/Makefile.am @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # -Wnoformat-truncation to get rid of compiler warning for unchecked # truncating snprintfs on gcc 7.1.1. AM_CFLAGS = $(FRAME_LARGER_THAN) diff --git a/docker/Dockerfile b/docker/Dockerfile index f85d5c0..6a594ff 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # # This Dockerfile builds cstor main container running zrepl from base image # diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index d5fb56a..5ef2bdf 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # # This Dockerfile builds cstor main container running zrepl from base image # diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 3755b0e..3fc2490 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # # This Dockerfile builds a recent base image containing cstor binaries and # libraries. diff --git a/docker/Dockerfile.base.arm64 b/docker/Dockerfile.base.arm64 index 33d7a57..b50413a 100644 --- a/docker/Dockerfile.base.arm64 +++ b/docker/Dockerfile.base.arm64 @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # # This Dockerfile builds a recent base image containing cstor binaries and # libraries. diff --git a/docker/entrypoint-poolimage.sh b/docker/entrypoint-poolimage.sh index be8ced0..a3e2c63 100644 --- a/docker/entrypoint-poolimage.sh +++ b/docker/entrypoint-poolimage.sh @@ -1,5 +1,19 @@ #!/bin/sh +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -o errexit trap 'call_exit $LINE_NO' EXIT diff --git a/include/Makefile.am b/include/Makefile.am index 7e00f24..6f950f4 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + SUBDIRS = sys generic arch/arm arch/x86 arch/ppc HEADER_H = \ diff --git a/include/arch/arm/Makefile.am b/include/arch/arm/Makefile.am index fb30b04..74a9b8e 100644 --- a/include/arch/arm/Makefile.am +++ b/include/arch/arm/Makefile.am @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + HEADER_H = \ $(top_srcdir)/include/arch/arm/rte_atomic.h \ $(top_srcdir)/include/arch/arm/rte_atomic_32.h \ diff --git a/include/arch/x86/Makefile.am b/include/arch/x86/Makefile.am index dabeb19..25777b4 100644 --- a/include/arch/x86/Makefile.am +++ b/include/arch/x86/Makefile.am @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + HEADER_H = \ $(top_srcdir)/include/arch/x86/rte_atomic_64.h \ $(top_srcdir)/include/arch/x86/rte_atomic_32.h \ diff --git a/include/generic/Makefile.am b/include/generic/Makefile.am index fa97276..de7ce16 100644 --- a/include/generic/Makefile.am +++ b/include/generic/Makefile.am @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + HEADER_H = \ $(top_srcdir)/include/generic/rte_atomic.h \ $(top_srcdir)/include/generic/rte_pause.h diff --git a/include/sys/Makefile.am b/include/sys/Makefile.am index c494326..bbf2c52 100644 --- a/include/sys/Makefile.am +++ b/include/sys/Makefile.am @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + HEADER_H = \ $(top_srcdir)/include/sys/uzfs_zvol.h \ $(top_srcdir)/include/sys/vdev_disk_aio.h diff --git a/src/Makefile.am b/src/Makefile.am index 4f5895e..96e80b6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,3 +1,17 @@ +# Copyright © 2017-2019 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ACLOCAL_AMFLAGS = -I m4 #Enable UZFS support From c19737af16a0cc3b7cfd2e36f637079dd14339bc Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Wed, 9 Sep 2020 12:48:09 +0530 Subject: [PATCH 05/10] This commit incorporate the review comments Signed-off-by: mittachaitu --- .travis.yml | 2 +- build_image.sh | 14 ++------------ tests/cstor/script/test_uzfs.sh | 2 -- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90c4b70..d5780ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ before_install: - sudo apt-get install --yes -qq lcov libjemalloc-dev # packages for tests - only on amd64 - if [ "$TRAVIS_CPU_ARCH" == "amd64" ]; then - sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio; + sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fi - sudo apt-get install --yes -qq libgtest-dev cmake # packages for debugging diff --git a/build_image.sh b/build_image.sh index 0d097f7..916cd98 100755 --- a/build_image.sh +++ b/build_image.sh @@ -22,7 +22,6 @@ pwd ARCH=$(uname -m) # Build libcstor -cd ../libcstor make clean sh autogen.sh ./configure --with-zfs-headers=$PWD/../cstor/include --with-spl-headers=$PWD/../cstor/lib/libspl/include @@ -30,7 +29,6 @@ make -j$(nproc) sudo make install sudo ldconfig - # Build cstor cd ../cstor make clean @@ -39,7 +37,7 @@ sh autogen.sh make clean make -j$(nproc) -# Build zrepl target and docker files exist in libcstor +# Build zrepl binary cd ../libcstor/cmd/zrepl make clean make @@ -71,7 +69,7 @@ DBUILD_DATE=$(date +'%Y-%m-%dT%H:%M:%SZ') # Specify the docker arg for repository url if [ -z "${DBUILD_REPO_URL}" ]; then - DBUILD_REPO_URL="https://github.com/openebs/cstor" + DBUILD_REPO_URL="https://github.com/openebs/libcstor" fi # Specify the docker arg for website url @@ -122,10 +120,6 @@ cd docker && \ sudo docker build -f ${DOCKERFILE_BASE} -t ${REPO_NAME}:ci ${DBUILD_ARGS} . && \ DIMAGE=${REPO_NAME} ./push && \ cd .. -if [ $? -ne 0 ]; then - echo "Failed to run push script for ${REPO_NAME}" - exit 1 -fi if [ "${ARCH}" = "x86_64" ]; then REPO_NAME="$IMAGE_ORG/cstor-pool" @@ -138,9 +132,5 @@ cd docker && \ sudo docker build -f ${DOCKERFILE} -t ${REPO_NAME}:ci ${DBUILD_ARGS} . && \ DIMAGE=${REPO_NAME} ./push && \ cd .. -if [ $? -ne 0 ]; then - echo "Failed to run push script for ${REPO_NAME}" - exit 1 -fi rm -rf ./docker/zfs diff --git a/tests/cstor/script/test_uzfs.sh b/tests/cstor/script/test_uzfs.sh index 74099a4..7d76b3c 100755 --- a/tests/cstor/script/test_uzfs.sh +++ b/tests/cstor/script/test_uzfs.sh @@ -22,8 +22,6 @@ fi ZPOOL="$SRC_PATH/cmd/zpool/zpool" ZFS="$SRC_PATH/cmd/zfs/zfs" ZDB="$SRC_PATH/cmd/zdb/zdb" -## Since zrepl has been moved from cstor to libcstor -## path also needs to be updated ## NOTE: It is assumed that cstor and libcstor will exist in same parent directory ZREPL="$SRC_PATH/../libcstor/cmd/zrepl/zrepl" GTEST_UZFS="$SRC_PATH/tests/cstor/gtest/test_uzfs" From d19a973e04b227c0df6edb27ddca55e18964bc98 Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Wed, 9 Sep 2020 12:55:27 +0530 Subject: [PATCH 06/10] This commit adds semicolumn in Travis Signed-off-by: mittachaitu --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d5780ec..cc6ee50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ before_install: - sudo apt-get install --yes -qq lcov libjemalloc-dev # packages for tests - only on amd64 - if [ "$TRAVIS_CPU_ARCH" == "amd64" ]; then - sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server + sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server; fi - sudo apt-get install --yes -qq libgtest-dev cmake # packages for debugging From 35c939ccb7473e2674a3fe9877c33cd74115740a Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Wed, 9 Sep 2020 13:29:26 +0530 Subject: [PATCH 07/10] This commit adds license file to ppce Makefile.am Signed-off-by: mittachaitu --- include/arch/ppc/Makefile.am | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/include/arch/ppc/Makefile.am b/include/arch/ppc/Makefile.am index 9f0957e..29b123d 100644 --- a/include/arch/ppc/Makefile.am +++ b/include/arch/ppc/Makefile.am @@ -1,3 +1,17 @@ +# Copyright © 2020 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + HEADER_H = \ $(top_srcdir)/include/arch/ppc/rte_atomic.h \ $(top_srcdir)/include/arch/ppc/rte_pause.h From 0e704708359a3f773aa9ee8ebfd91a498d258db6 Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Wed, 9 Sep 2020 16:16:27 +0530 Subject: [PATCH 08/10] This commit updates the .md files and incorporate review comments Signed-off-by: mittachaitu --- .travis.yml | 3 +-- RELEASE.md | 17 ++++++++++++++-- developer-setup.md | 51 ++++++++++++++++++++++++++++++---------------- 3 files changed, 49 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc6ee50..bfa2ed8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -109,8 +109,7 @@ script: sudo bash ../libcstor/tests/cstor/script/test_uzfs.sh -T all || travis_terminate 1; fi # Go back to libcstor to build images - - cd .. - - cd libcstor + - cd ../libcstor - ./build_image.sh || travis_terminate 1; # If this build is running due to travis release tag, then # go ahead and tag the dependent repo. diff --git a/RELEASE.md b/RELEASE.md index 5a09076..4e05cd3 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -18,7 +18,7 @@ Once all the above tests are completed, a main release tagged image is published ## Release Tagging -libcstor is released as a library part of [github/cstor](https://github.com/openebs/cstor) container image with a versioned tag. +libcstor is released as a container image with a versioned tag. Before creating a release, the repo owner needs to create a separate branch from the active branch, which is `master`. Name of the branch should follow the naming convention of `v.1.9.x` if release is for v1.9.0. @@ -26,7 +26,20 @@ Once the release branch is created, changelog from `changelogs/unreleased` needs The format of the release tag is either "Release-Name-RC1" or "Release-Name" depending on whether the tag is a release candidate or a release. (Example: v1.9.0-RC1 is a GitHub release tag for libcstor release build. v1.9.0 is the release tag that is created after the release criteria are satisfied by the libcstor builds.) -Once the release is triggered, Travis build process has to be monitored. Since libcstor is a library and it is published as part of [openebs/cstor](https://github.com/openebs/cstor) images, release process from libcstor repo won't publish any images. +Once the release is triggered, Travis build process has to be monitored. Once Travis build is passed images are pushed to docker hub and quay.io. Images can be verified by going through docker hub and quay.io. Also the images shouldn't have any high level vulnerabilities. + +Images are published at the following location: +For AMD64: +``` +https://quay.io/repository/openebs/cstor-pool?tab=tags +https://hub.docker.com/r/openebs/cstor-pool/tags +``` + +For ARM64: +``` +https://quay.io/repository/openebs/cstor-pool-arm64?tab=tags +https://hub.docker.com/r/openebs/cstor-pool-arm64/tags +``` Once a release is created, update the release description with the changelog mentioned in folder `changelog/v1.9.x`. Once the changelogs are updated in the release, the repo owner needs to create a PR to `master` with the following details: 1. update the changelog from folder `changelog/v1.9.x` to file `libcstor/CHANGELOG-v1.9.md` diff --git a/developer-setup.md b/developer-setup.md index 6fcbf80..2617ac3 100644 --- a/developer-setup.md +++ b/developer-setup.md @@ -27,9 +27,11 @@ ``` * Make sure that you have cloned cstor code in the system. Refer [cstor setup](https://github.com/openebs/cstor/blob/master/developer-setup.md) + **NOTE**: libcstor and cstor must present in the same parent directory. + * Make sure that you have cloned and build fio(branch fio-3.7) code in the linux system. Please use below command for the fio - ``sh + ```sh git clone https://github.com/axboe/fio cd fio git checkout fio-3.7 @@ -66,29 +68,42 @@ git remote -v ### Building and Testing your changes * To build the libcstor library -```sh -sh autogen.sh -./configure --with-config=user --enable-debug --enable-uzfs=yes --with-jemalloc --with-spl-headers=$PWD/../cstor/lib/libspl/include -make -``` + ```sh + sh autogen.sh + ./configure --with-config=user --enable-debug --enable-uzfs=yes --with-jemalloc --with-spl-headers=$PWD/../cstor/lib/libspl/include + make + ``` * To install the library in local machine -```sh -sudo make install -sudo ldconfig -``` + ```sh + sudo make install + sudo ldconfig + ``` * To verify the coding style -```sh -make -f ../cstor/Makefile cstyle CSTORDIR=$PWD/../cstor -``` + ```sh + make -f ../cstor/Makefile cstyle CSTORDIR=$PWD/../cstor + ``` + +* To verify license checks + ```sh + make check-license + ``` + +* To build the zrepl binary(main process of cstor) + ```sh + cd cmd/zrepl + make + cd ../../ + ``` * Test your changes -Integration tests are written in c and c+. Test script is maintained at https://github.com/openebs/libcstor/blob/master/tests/cstor/script/test_uzfs.sh -To run the run the integration tests. -```sh -./tests/cstor/script/test_uzfs.sh -T all -``` + + Integration tests are written in c and c+. Test script is maintained at https://github.com/openebs/libcstor/blob/master/tests/cstor/script/test_uzfs.sh + To run the run the integration tests go to cstor directory and run below command. + ```sh + ./tests/cstor/script/test_uzfs.sh -T all + ``` ## Git Development Workflow From 99994804c28d9df833a8018cabfe098b2d34d96e Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Wed, 9 Sep 2020 16:25:35 +0530 Subject: [PATCH 09/10] This commit updates the licesne content from 2017-2019 to 2020 Signed-off-by: mittachaitu --- Makefile.am | 2 +- autogen.sh | 2 +- build_image.sh | 2 +- cmd/zrepl/Makefile.am | 2 +- cmd/zrepl/zrepl.c | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile.arm64 | 2 +- docker/Dockerfile.base | 2 +- docker/Dockerfile.base.arm64 | 2 +- docker/entrypoint-poolimage.sh | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile.am b/Makefile.am index 64494d8..9f1d584 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,4 @@ -# Copyright © 2017-2019 The OpenEBS Authors +# Copyright © 2020 The OpenEBS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/autogen.sh b/autogen.sh index f90f017..ca481f6 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright © 2017-2019 The OpenEBS Authors +# Copyright © 2019 The OpenEBS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/build_image.sh b/build_image.sh index 916cd98..cc34862 100755 --- a/build_image.sh +++ b/build_image.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright © 2017-2019 The OpenEBS Authors +# Copyright © 2020 The OpenEBS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/cmd/zrepl/Makefile.am b/cmd/zrepl/Makefile.am index 95ef84a..61265a0 100644 --- a/cmd/zrepl/Makefile.am +++ b/cmd/zrepl/Makefile.am @@ -1,4 +1,4 @@ -# Copyright © 2017-2019 The OpenEBS Authors +# Copyright © 2020 The OpenEBS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/cmd/zrepl/zrepl.c b/cmd/zrepl/zrepl.c index e9bab78..8d9dc5c 100644 --- a/cmd/zrepl/zrepl.c +++ b/cmd/zrepl/zrepl.c @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2019 The OpenEBS Authors + * Copyright © 2020 The OpenEBS Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/docker/Dockerfile b/docker/Dockerfile index 6a594ff..4375448 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -# Copyright © 2017-2019 The OpenEBS Authors +# Copyright © 2020 The OpenEBS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 5ef2bdf..d080e03 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,4 +1,4 @@ -# Copyright © 2017-2019 The OpenEBS Authors +# Copyright © 2020 The OpenEBS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 3fc2490..14dbbf0 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -1,4 +1,4 @@ -# Copyright © 2017-2019 The OpenEBS Authors +# Copyright © 2020 The OpenEBS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docker/Dockerfile.base.arm64 b/docker/Dockerfile.base.arm64 index b50413a..39b4f31 100644 --- a/docker/Dockerfile.base.arm64 +++ b/docker/Dockerfile.base.arm64 @@ -1,4 +1,4 @@ -# Copyright © 2017-2019 The OpenEBS Authors +# Copyright © 2020 The OpenEBS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docker/entrypoint-poolimage.sh b/docker/entrypoint-poolimage.sh index a3e2c63..4fe2184 100644 --- a/docker/entrypoint-poolimage.sh +++ b/docker/entrypoint-poolimage.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright © 2017-2019 The OpenEBS Authors +# Copyright © 2020 The OpenEBS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 5e16dcd4bc576f3de31c99e930257b97864ab53a Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Wed, 9 Sep 2020 19:20:26 +0530 Subject: [PATCH 10/10] This commit incorporates the review comments Signed-off-by: mittachaitu --- developer-setup.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/developer-setup.md b/developer-setup.md index 2617ac3..cf25ac9 100644 --- a/developer-setup.md +++ b/developer-setup.md @@ -25,16 +25,16 @@ sudo make -j4 sudo cp *.a /usr/lib ``` -* Make sure that you have cloned cstor code in the system. Refer [cstor setup](https://github.com/openebs/cstor/blob/master/developer-setup.md) +* Make sure that you have cloned cstor code in the system. Refer [cstor setup](https://github.com/openebs/cstor/blob/develop/developer-setup.md) **NOTE**: libcstor and cstor must present in the same parent directory. -* Make sure that you have cloned and build fio(branch fio-3.7) code in the linux system. +* Make sure that you have cloned and build fio(branch fio-3.9) code in the linux system. Please use below command for the fio ```sh git clone https://github.com/axboe/fio cd fio - git checkout fio-3.7 + git checkout fio-3.9 ./configure make -j4 ``` @@ -70,7 +70,7 @@ git remote -v * To build the libcstor library ```sh sh autogen.sh - ./configure --with-config=user --enable-debug --enable-uzfs=yes --with-jemalloc --with-spl-headers=$PWD/../cstor/lib/libspl/include + ./configure --enable-debug --with-zfs-headers=$PWD/../cstor/include --with-spl-headers=$PWD/../cstor/lib/libspl/include make ``` @@ -98,11 +98,11 @@ git remote -v ``` * Test your changes - + Integration tests are written in c and c+. Test script is maintained at https://github.com/openebs/libcstor/blob/master/tests/cstor/script/test_uzfs.sh To run the run the integration tests go to cstor directory and run below command. ```sh - ./tests/cstor/script/test_uzfs.sh -T all + ../libcstor/tests/cstor/script/test_uzfs.sh -T all ``` ## Git Development Workflow