Bump up photon version to 4.0 on release-1.10.0

Makefile

@@ -337,6 +337,9 @@ build_base_docker:
 	fi
 	@for name in chartserver clair clair-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
 		echo $$name ; \
+		if [ $$name == "db" ] ; then \
+			cd $(MAKEFILEPATH_PHOTON)/$$name && $(MAKEFILEPATH_PHOTON)/$$name/rpm_builder.sh && cd - ; \
+		fi; \
 		$(DOCKERBUILD) --pull -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t goharbor/harbor-$$name-base:$(BASEIMAGETAG) . && \
 		$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) goharbor/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) || exit 1 ; \
 	done

make/photon/chartserver/Dockerfile.base

@@ -1,4 +1,4 @@
-FROM photon:2.0
+FROM photon:4.0
 
 RUN tdnf install -y shadow sudo >>/dev/null\
     && tdnf clean all \

make/photon/clair-adapter/Dockerfile.base

@@ -1,7 +1,7 @@
-FROM photon:2.0
+FROM photon:4.0
 
 RUN tdnf install -y sudo >>/dev/null\
     && tdnf clean all \
     && mkdir /clair-adapter/ \
     && groupadd -r -g 10000 clair-adapter \
-    && useradd --no-log-init -m -r -g 10000 -u 10000 clair-adapter
+    && useradd --no-log-init -m -r -g 10000 -u 10000 clair-adapter

make/photon/clair/Dockerfile.base

@@ -1,6 +1,6 @@
-FROM photon:2.0
+FROM photon:4.0
 
 RUN tdnf install -y git shadow sudo rpm xz python3-xml >>/dev/null\
     && tdnf clean all \
     && groupadd -r -g 10000 clair \
-    && useradd --no-log-init -m -g 10000 -u 10000 clair
+    && useradd --no-log-init -m -g 10000 -u 10000 clair

make/photon/core/Dockerfile.base

@@ -1,4 +1,4 @@
-FROM photon:2.0
+FROM photon:4.0
 
 RUN tdnf install sudo tzdata -y >> /dev/null \
     && tdnf clean all \

make/photon/db/Dockerfile

@@ -4,14 +4,16 @@ FROM goharbor/harbor-db-base:${harbor_base_image_version}
 VOLUME /var/lib/postgresql/data
 
 COPY ./make/photon/db/docker-entrypoint.sh /docker-entrypoint.sh
+COPY ./make/photon/db/initdb.sh /initdb.sh
+COPY ./make/photon/db/upgrade.sh /upgrade.sh
 COPY ./make/photon/db/docker-healthcheck.sh /docker-healthcheck.sh
 COPY ./make/photon/db/initial-notaryserver.sql /docker-entrypoint-initdb.d/
 COPY ./make/photon/db/initial-notarysigner.sql /docker-entrypoint-initdb.d/
 COPY ./make/photon/db/initial-registry.sql /docker-entrypoint-initdb.d/
 RUN chown -R postgres:postgres /docker-entrypoint.sh /docker-healthcheck.sh /docker-entrypoint-initdb.d \
     && chmod u+x /docker-entrypoint.sh /docker-healthcheck.sh
 
-ENTRYPOINT ["/docker-entrypoint.sh"]
+ENTRYPOINT ["/docker-entrypoint.sh", "96", "13"]
 HEALTHCHECK CMD ["/docker-healthcheck.sh"]
 
 EXPOSE 5432

make/photon/db/Dockerfile.base

@@ -1,17 +1,22 @@
-FROM photon:2.0
+FROM photon:4.0
 
 ENV PGDATA /var/lib/postgresql/data
 
-RUN tdnf install -y shadow gzip postgresql >> /dev/null\
+COPY ./make/photon/db/postgresql96-libs-9.6.21-1.ph4.x86_64.rpm /pg96/
+COPY ./make/photon/db/postgresql96-9.6.21-1.ph4.x86_64.rpm /pg96/
+
+RUN tdnf install -y /pg96/postgresql96-libs-9.6.21-1.ph4.x86_64.rpm /pg96/postgresql96-9.6.21-1.ph4.x86_64.rpm >> /dev/null \
+    && rm -rf /pg96 \
+    && tdnf install -y shadow gzip postgresql13 findutils bc >> /dev/null \
     && groupadd -r postgres --gid=999 \
     && useradd -m -r -g postgres --uid=999 postgres \
     && mkdir -p /docker-entrypoint-initdb.d \
     && mkdir -p /run/postgresql \
     && chown -R postgres:postgres /run/postgresql \
     && chmod 2777 /run/postgresql \
     && mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" \
-    && sed -i "s|#listen_addresses = 'localhost'.*|listen_addresses = '*'|g" /usr/share/postgresql/postgresql.conf.sample \
-    && sed -i "s|#unix_socket_directories = '/tmp'.*|unix_socket_directories = '/run/postgresql'|g" /usr/share/postgresql/postgresql.conf.sample \
+    && sed -i "s|#listen_addresses = 'localhost'.*|listen_addresses = '*'|g" /usr/pgsql/13/share/postgresql/postgresql.conf.sample \
+    && sed -i "s|#unix_socket_directories = '/tmp'.*|unix_socket_directories = '/run/postgresql'|g" /usr/pgsql/13/share/postgresql/postgresql.conf.sample \
     && tdnf clean all
 
 RUN tdnf erase -y toybox && tdnf install -y util-linux net-tools

make/photon/db/docker-entrypoint.sh

@@ -1,110 +1,68 @@
 #!/bin/bash
 set -e
 
-# usage: file_env VAR [DEFAULT]
-#    ie: file_env 'XYZ_DB_PASSWORD' 'example'
-# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
-#  "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
-file_env() {
-        local var="$1"
-        local fileVar="${var}_FILE"
-        local def="${2:-}"
-        if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
-                echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
-                exit 1
-        fi
-        local val="$def"
-        if [ "${!var:-}" ]; then
-                val="${!var}"
-        elif [ "${!fileVar:-}" ]; then
-                val="$(< "${!fileVar}")"
-        fi
-        export "$var"="$val"
-        unset "$fileVar"
-}
-
-# look specifically for PG_VERSION, as it is expected in the DB dir
-if [ ! -s "$PGDATA/PG_VERSION" ]; then
-        file_env 'POSTGRES_INITDB_ARGS'
-        if [ "$POSTGRES_INITDB_XLOGDIR" ]; then
-                export POSTGRES_INITDB_ARGS="$POSTGRES_INITDB_ARGS --xlogdir $POSTGRES_INITDB_XLOGDIR"
-        fi
-        initdb -D $PGDATA  -U postgres -E UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 $POSTGRES_INITDB_ARGS
-        # check password first so we can output the warning before postgres
-        # messes it up
-        file_env 'POSTGRES_PASSWORD'
-        if [ "$POSTGRES_PASSWORD" ]; then
-                pass="PASSWORD '$POSTGRES_PASSWORD'"
-                authMethod=md5
-        else
-                # The - option suppresses leading tabs but *not* spaces. :)
-                cat >&2 <<-EOF
-                        ****************************************************
-                        WARNING: No password has been set for the database.
-                                        This will allow anyone with access to the
-                                        Postgres port to access your database. In
-                                        Docker's default configuration, this is
-                                        effectively any other container on the same
-                                        system.
-                                        Use "-e POSTGRES_PASSWORD=password" to set
-                                        it in "docker run".
-                        ****************************************************
-EOF
-
-                pass=
-                authMethod=trust
-        fi
+source $PWD/initdb.sh
 
-        {
-                echo
-                echo "host all all all $authMethod"
-        } >> "$PGDATA/pg_hba.conf"
-        echo `whoami`
-        # internal start of server in order to allow set-up using psql-client
-        # does not listen on external TCP/IP and waits until start finishes
-        pg_ctl -D "$PGDATA" -o "-c listen_addresses=''" -w start
+CUR=$PWD
+PG_VERSION_OLD=$1
+PG_VERSION_NEW=$2
 
-        file_env 'POSTGRES_USER' 'postgres'
-        file_env 'POSTGRES_DB' "$POSTGRES_USER"
+PGBINOLD="/usr/local/pg${PG_VERSION_OLD}/bin"
 
-        psql=( psql -v ON_ERROR_STOP=1 )
+PGDATAOLD=${PGDATA}/pg${PG_VERSION_OLD}
+PGDATANEW=${PGDATA}/pg${PG_VERSION_NEW}
+# to handle the PG 9.6 only
+if [ -s $PGDATA/PG_VERSION ]; then
+         PGDATAOLD=$PGDATA
+fi
 
-        if [ "$POSTGRES_DB" != 'postgres' ]; then
-                "${psql[@]}" --username postgres <<-EOSQL
-                        CREATE DATABASE "$POSTGRES_DB" ;
-EOSQL
-                echo
-        fi
+#
+# Init DB: $PGDATA is empty.
+# Upgrade DB: 1, has $PGDATA\PG_VERSION. 2, has pg old version directory with PG_VERSION inside.
+#
+if [ "$(ls -A $PGDATA)" ]; then
+        if [ ! -d $PGDATANEW ]; then
+                if [ ! -d $PGDATAOLD ] || [ ! -s $PGDATAOLD/PG_VERSION ]; then
+                        echo "incorrect data: $PGDATAOLD, make sure $PGDATAOLD is not empty and with PG_VERSION inside."
+                        exit 1
+                fi
 
-        if [ "$POSTGRES_USER" = 'postgres' ]; then
-                op='ALTER'
+                initPG $PGDATANEW false
+                set +e
+                # In some cases, like helm upgrade, the postgresql may not quit cleanly.
+                # Use start & stop to clean the unexpected status. Error:
+                #   There seems to be a postmaster servicing the new cluster.
+                #   Please shutdown that postmaster and try again.
+                #   Failure, exiting
+                $PGBINOLD/pg_ctl -D "$PGDATAOLD" -w -o "-p 5433" start
+                $PGBINOLD/pg_ctl -D "$PGDATAOLD" -m fast -w stop
+                ./$CUR/upgrade.sh --old-bindir $PGBINOLD --old-datadir $PGDATAOLD --new-datadir $PGDATANEW
+                # it needs to clean the $PGDATANEW on upgrade failure
+                if [ $? -ne 0 ]; then
+                        echo "remove the $PGDATANEW after fail to upgrade"
+                        rm -rf $PGDATANEW
+                        exit 1
+                fi
+                set -e
+                echo "remove the $PGDATAOLD after upgrade success."
+                if [ "$PGDATAOLD" = "$PGDATA" ]; then
+                        find $PGDATA/* -prune ! -name pg${PG_VERSION_NEW} -exec rm -rf {} \;
+                else
+                        rm -rf $PGDATAOLD
+                fi
         else
-                op='CREATE'
+                echo "no need to upgrade postgres, launch it."
         fi
-        "${psql[@]}" --username postgres <<-EOSQL
-                $op USER "$POSTGRES_USER" WITH SUPERUSER $pass ;
-EOSQL
-        echo
-
-        psql+=( --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" )
-
-        echo
-        for f in /docker-entrypoint-initdb.d/*; do
-                case "$f" in
-                        *.sh)     echo "$0: running $f"; . "$f" ;;
-                        *.sql)    echo "$0: running $f"; "${psql[@]}" -f "$f"; echo ;;
-                        *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;;
-                        *)        echo "$0: ignoring $f" ;;
-                esac
-                echo
-        done
-
-        PGUSER="${PGUSER:-postgres}" \
-        pg_ctl -D "$PGDATA" -m fast -w stop
+else
+        initPG $PGDATANEW true
+fi
 
-        echo
-        echo 'PostgreSQL init process complete; ready for start up.'
-        echo
+POSTGRES_PARAMETER=''
+file_env 'POSTGRES_MAX_CONNECTIONS' '1024'
+# The max value of 'max_connections' is 262143
+if [ $POSTGRES_MAX_CONNECTIONS -le 0 ] || [ $POSTGRES_MAX_CONNECTIONS -gt 262143 ]; then
+        POSTGRES_MAX_CONNECTIONS=262143
 fi
 
-postgres -D $PGDATA
+POSTGRES_PARAMETER="${POSTGRES_PARAMETER} -c max_connections=${POSTGRES_MAX_CONNECTIONS}"
+exec postgres -D $PGDATANEW $POSTGRES_PARAMETER

make/photon/db/initdb.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+set -e
+
+# usage: file_env VAR [DEFAULT]
+#    ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+#  "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+function file_env() {
+        local var="$1"
+        local fileVar="${var}_FILE"
+        local def="${2:-}"
+        if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+                echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+                exit 1
+        fi
+        local val="$def"
+        if [ "${!var:-}" ]; then
+                val="${!var}"
+        elif [ "${!fileVar:-}" ]; then
+                val="$(< "${!fileVar}")"
+        fi
+        export "$var"="$val"
+        unset "$fileVar"
+}
+
+# usage: initPG $Dir $initSql
+#   Use $Dir to index where to init the postgres db
+#   Use $initSql to indicate whether to execute the sql under docker-entrypoint-initdb.d, default is not.
+function initPG() {
+        file_env 'POSTGRES_INITDB_ARGS'
+        if [ "$POSTGRES_INITDB_XLOGDIR" ]; then
+                export POSTGRES_INITDB_ARGS="$POSTGRES_INITDB_ARGS --xlogdir $POSTGRES_INITDB_XLOGDIR"
+        fi
+        initdb -D $1  -U postgres -E UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 $POSTGRES_INITDB_ARGS
+        # check password first so we can output the warning before postgres
+        # messes it up
+        file_env 'POSTGRES_PASSWORD'
+        if [ "$POSTGRES_PASSWORD" ]; then
+                pass="PASSWORD '$POSTGRES_PASSWORD'"
+                authMethod=md5
+        else
+                # The - option suppresses leading tabs but *not* spaces. :)
+                cat >&2 <<-EOF
+                        ****************************************************
+                        WARNING: No password has been set for the database.
+                                        This will allow anyone with access to the
+                                        Postgres port to access your database. In
+                                        Docker's default configuration, this is
+                                        effectively any other container on the same
+                                        system.
+                                        Use "-e POSTGRES_PASSWORD=password" to set
+                                        it in "docker run".
+                        ****************************************************
+EOF
+
+                pass=
+                authMethod=trust
+        fi
+
+        {
+                echo
+                echo "host all all all $authMethod"
+        } >> "$1/pg_hba.conf"
+        echo `whoami`
+        # internal start of server in order to allow set-up using psql-client
+        # does not listen on external TCP/IP and waits until start finishes
+        pg_ctl -D "$1" -o "-c listen_addresses=''" -w start
+
+        file_env 'POSTGRES_USER' 'postgres'
+        file_env 'POSTGRES_DB' "$POSTGRES_USER"
+
+        psql=( psql -v ON_ERROR_STOP=1 )
+
+        if [ "$POSTGRES_DB" != 'postgres' ]; then
+                "${psql[@]}" --username postgres <<-EOSQL
+                        CREATE DATABASE "$POSTGRES_DB" ;
+EOSQL
+                echo
+        fi
+
+        if [ "$POSTGRES_USER" = 'postgres' ]; then
+                op='ALTER'
+        else
+                op='CREATE'
+        fi
+        "${psql[@]}" --username postgres <<-EOSQL
+                $op USER "$POSTGRES_USER" WITH SUPERUSER $pass ;
+EOSQL
+        echo
+
+        psql+=( --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" )
+
+        echo
+
+        if [ $2 == "true" ]; then
+                for f in /docker-entrypoint-initdb.d/*; do
+                        case "$f" in
+                                *.sh)     echo "$0: running $f"; . "$f" ;;
+                                *.sql)    echo "$0: running $f"; "${psql[@]}" -f "$f"; echo ;;
+                                *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;;
+                                *)        echo "$0: ignoring $f" ;;
+                        esac
+                        echo
+                done
+        fi
+
+        PGUSER="${PGUSER:-postgres}" \
+        pg_ctl -D "$1" -m fast -w stop
+
+        echo
+        echo 'PostgreSQL init process complete; ready for start up.'
+        echo
+
+}