Merge branch 'release/21.03'

* release/21.03:
  Prevent warning when updating LabCA
  Lower frequency of looking for file changes of hostname and rate-limit policies
  Do not wait for boulder startup when setup is not yet complete
  Wait longer for server to be ready before requesting first certificate
  Tweak description string of whitelist mode
  Fix config substitutions for whitelist mode (#15)
  Hold off on starting boulder until setup wizzard is completed
  Make installer more robust
  Make config substitutions more robust
  Stick to release tags even if there are newer commits on master
  Minor tweaks
  Fix query buildup when combining tables

commander

@@ -18,11 +18,11 @@ function wait_server() {
 
     local status=0
     local cnt=0
-    while [ $cnt -lt 20 ] && [ "$status" != "200" ]; do
+    while [ $cnt -lt 40 ] && [ "$status" != "200" ]; do
         status=$(curl -o /dev/null -sSL --head --write-out '%{http_code}\n' $url)
         let cnt=$cnt+1
-        if [ $cnt -lt 10 ] && [ "$status" != "200" ]; then
-            sleep 3
+        if [ "$status" != "200" ]; then
+            sleep 5
         fi
     done
 }
@@ -55,6 +55,7 @@ case $txt in
     chown -R www-data:www-data *
     url=$(grep 'DEFAULT_DIRECTORY_URL =' /home/labca/acme_tiny.py | sed -e 's/.*=[ ]*//' | sed -e 's/\"//g')
     wait_server $url
+    sleep 10
     /home/labca/labca/renew
     ln -sf /home/labca/labca/cron_d /etc/cron.d/labca
     ln -sf /home/labca/labca/logrotate_d /etc/logrotate.d/labca

entrypoint.patch

@@ -2,10 +2,16 @@ diff --git a/test/entrypoint.sh b/test/entrypoint.sh
 index 5ca9929..f18e1d8 100755
 --- a/test/entrypoint.sh
 +++ b/test/entrypoint.sh
-@@ -36,6 +36,12 @@ wait_tcp_port boulder-mysql 3306
+@@ -36,6 +36,18 @@ wait_tcp_port boulder-mysql 3306
  # create the database
  MYSQL_CONTAINER=1 $DIR/create_db.sh
 
++fl=$(pwd)/labca/setup_complete
++while [ ! -f $fl ]; do
++    echo "Waiting for $fl to appear..."
++    sleep 30
++done
++
 +#softhsm2-util --show-slots
 +softhsm2-util --init-token --slot 0 --label "intermediate signing key (rsa)" --pin 1234 --so-pin 5678 | /bin/true
 +[ -e labca/test-ca.p8 ] && softhsm2-util --import labca/test-ca.p8 --id 333333 --force --token "intermediate signing key (rsa)" --pin 1234 --so-pin 5678 --label 'intermediate_key'

gui/acme.go

@@ -297,7 +297,7 @@ func GetOrder(w http.ResponseWriter, r *http.Request, id int) (OrderShow, error)
 		if query != "" {
 			query = query + " UNION "
 		}
-		query = "SELECT id, identifierValue, registrationID, status, expires FROM authz2 WHERE id IN (SELECT authzID FROM orderToAuthz2 WHERE orderID=?)"
+		query = query + "SELECT id, identifierValue, registrationID, status, expires FROM authz2 WHERE id IN (SELECT authzID FROM orderToAuthz2 WHERE orderID=?)"
 	}
 	var rows *sql.Rows
 	if tableExists(db, "authz") && tableExists(db, "authz2") {
@@ -514,7 +514,7 @@ func GetAuth(w http.ResponseWriter, r *http.Request, id string) (AuthShow, error
 		if query != "" {
 			query = query + " UNION "
 		}
-		query = "SELECT id, identifierValue, registrationID, status, expires, validationError, validationRecord FROM authz2 WHERE id IN (SELECT authzID FROM orderToAuthz2 WHERE id=?)"
+		query = query + "SELECT id, identifierValue, registrationID, status, expires, validationError, validationRecord FROM authz2 WHERE id IN (SELECT authzID FROM orderToAuthz2 WHERE id=?)"
 	}
 	if tableExists(db, "authz") && tableExists(db, "authz2") {
 		rows, err = db.Query(query, id, id)

gui/apply-boulder

@@ -16,20 +16,29 @@ sed -i -e "s/\"directoryCAAIdentity\": \".*\"/\"directoryCAAIdentity\": \"$PKI_D
 [ -e ../test/hostname-policy.yaml ] && cp ../test/hostname-policy.yaml ./ || true
 [ -e ../boulder/test/hostname-policy.yaml ] && cp ../boulder/test/hostname-policy.yaml ./ || true
 [ -e hostname-policy.json ] && rm hostname-policy.json || true
-if [ "$PKI_DOMAIN_MODE" == "lockdown" ]; then
+cat hostname-policy.yaml | tr '\n' '\r' | sed -e "s/Lockdown:.*//" | tr '\r' '\n' > hostname-policy.yaml.bak && mv hostname-policy.yaml.bak hostname-policy.yaml
+cat hostname-policy.yaml | tr '\n' '\r' | sed -e "s/Whitelist:.*//" | tr '\r' '\n' > hostname-policy.yaml.bak && mv hostname-policy.yaml.bak hostname-policy.yaml
+if [ "$PKI_DOMAIN_MODE" == "lockdown" ] && [ "$PKI_LOCKDOWN_DOMAINS" != "" ]; then
     echo "Lockdown:" >> hostname-policy.yaml
     echo "  - \"$PKI_LOCKDOWN_DOMAINS\"" >> hostname-policy.yaml
 fi
-if [ "$PKI_DOMAIN_MODE" == "whitelist" ]; then
+if [ "$PKI_DOMAIN_MODE" == "whitelist" ] && [ "$PKI_WHITELIST_DOMAINS" != "" ]; then
     echo "Whitelist:" >> hostname-policy.yaml
-    echo "  - \"$PKI_LOCKDOWN_DOMAINS\"" >> hostname-policy.yaml
+    echo "  - \"$PKI_WHITELIST_DOMAINS\"" >> hostname-policy.yaml
 fi
 if [ "$PKI_DOMAIN_MODE" == "lockdown" ] || [ "$PKI_DOMAIN_MODE" == "whitelist" ]; then
     sed -i -e "s/^\(.*\)\(\"n_subject_common_name_included\"\)/\1\2,\n\1\"e_dnsname_not_valid_tld\"/" config/ca-a.json
     sed -i -e "s/^\(.*\)\(\"n_subject_common_name_included\"\)/\1\2,\n\1\"e_dnsname_not_valid_tld\"/" config/ca-b.json
 
-    sed -i -e "s/\(  registrationOverrides:\)/    $PKI_LOCKDOWN_DOMAINS: 10000\n\1/" rate-limit-policies.yml
-    echo "    $PKI_LOCKDOWN_DOMAINS: 10000" >> rate-limit-policies.yml
+    REPLACEMENT=""
+    if [ "$PKI_DOMAIN_MODE" == "lockdown" ] && [ "$PKI_LOCKDOWN_DOMAINS" != "" ]; then
+        REPLACEMENT="    $PKI_LOCKDOWN_DOMAINS: 10000\n"
+    fi
+    if [ "$PKI_DOMAIN_MODE" == "whitelist" ] && [ "$PKI_WHITELIST_DOMAINS" != "" ]; then
+        REPLACEMENT="    $PKI_WHITELIST_DOMAINS: 10000\n"
+    fi
+    cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s/\(must-staple.le.wtf: 10000\).*\(  registrationOverrides:\)/\1\n$REPLACEMENT\2/" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml
+    cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s/\(certificatesPerFQDNSet:.*must-staple.le.wtf: 10000\).*/\1\n$REPLACEMENT/" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml
 fi
 
 if [ "$PKI_EXTENDED_TIMEOUT" == "1" ]; then
@@ -89,3 +98,5 @@ openssl rsa -in $PKI_ROOT_CERT_BASE.key -pubout > test-root.pubkey.pem 2>/dev/nu
 openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in test-root.key -out test-root.p8
 
 chown -R `ls -l PKI.md | cut -d" " -f 3,4 | sed 's/ /:/g'` .
+
+[ -f setup_complete ] || touch setup_complete

gui/dashboard.go

@@ -3,13 +3,14 @@ package main
 import (
 	"database/sql"
 	"fmt"
-	"github.com/dustin/go-humanize"
 	"log"
 	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/dustin/go-humanize"
 )
 
 // Activity is a message to be shown on the dashboard, with timestamp and css class

gui/setup.sh

@@ -8,7 +8,7 @@ set -e
 if [ ! -e bin/labca ]; then
     go mod download
 
-    go build -o bin/labca main.go acme.go certificate.go dashboard.go
+    go build -o bin/labca
 fi
 
 bin/labca

gui/templates/views/manage.tmpl

@@ -168,7 +168,7 @@
         <span class="error config-error hidden" id="lockdowndomains-error"></span>
         <br/>
 
-        <input type="radio" id="whitelist" name="domain_mode" value="whitelist" {{ if eq .DomainMode "whitelist"}}checked{{ end }}/> Next to all official domains, also allow this domain:<br/>
+        <input type="radio" id="whitelist" name="domain_mode" value="whitelist" {{ if eq .DomainMode "whitelist"}}checked{{ end }}/> Next to all official domains, also allow this domain (whitelist):<br/>
         <input class="form-control non-fluid" type="text" id="whitelist_domains" name="whitelist_domains" value="{{ .WhitelistDomains }}"/>
         <span class="error config-error hidden" id="whitelistdomains-error"></span>
         <br/>

gui/templates/views/setup.tmpl

@@ -31,7 +31,7 @@
     <span class="error">{{ . }}</span><br/>
     {{ end }}
 
-    <input type="radio" id="whitelist" name="domain_mode" value="whitelist" {{ if eq .DomainMode "whitelist"}}checked{{ end }}/> Next to all official domains, also allow this domain:<br/>
+    <input type="radio" id="whitelist" name="domain_mode" value="whitelist" {{ if eq .DomainMode "whitelist"}}checked{{ end }}/> Next to all official domains, also allow this domain (whitelist):<br/>
     <input class="form-control non-fluid" type="text" id="whitelist_domains" name="whitelist_domains" value="{{ .WhitelistDomains }}"/><br/>
     {{ with .Errors.WhitelistDomains }}
     <span class="error">{{ . }}</span><br/>

install

@@ -51,6 +51,7 @@ source "$dn/utils.sh" &>/dev/null || true
 
 cmdlineFqdn=""
 cmdlineBranch=""
+fullCmdline=""
 
 #
 # Helper functions for informing the user and logging to file
@@ -195,6 +196,16 @@ clone_or_pull() {
     fi
 }
 
+# Checkout the latest release tag
+checkout_release() {
+    local branch="$1"
+    if [ "$branch" == "" ] || [ "$branch" == "master" ]; then
+        cd "$cloneDir"
+        TAG=$(git describe --tags $(git rev-list --tags --max-count=1))
+        sudo -u labca -H git reset --hard $TAG &>>$installLog
+    fi
+}
+
 # Restart the script if it was updated itself
 restart_if_updated() {
     local curChecksum="$1"
@@ -206,7 +217,7 @@ restart_if_updated() {
         if [ "$curChecksum" != "$newChecksum" ]; then
             msg_info "Restarting updated version of install script"
             echo
-            exec $cloneDir/install
+            exec $cloneDir/install $fullCmdline
             exit $?
         fi
     fi
@@ -229,6 +240,7 @@ prompt_and_export() {
 
 # Parse the command line options, if any
 parse_cmdline() {
+    fullCmdline="$@"
     local parsed=$(getopt --options=n:,b: --longoptions=name:,fqdn:,branch: --name "$0" -- "$@" 2>>$installLog) || msg_fatal "Could not process commandline parameters"
     eval set -- "$parsed"
     while true; do
@@ -252,19 +264,45 @@ parse_cmdline() {
     done
 }
 
+# Utility method to check if value looks like a host + domain
+has_domain() {
+    local dom="$1"
+
+    if [[ "$dom" =~ ^\..*$ ]]; then
+        false
+    elif [[ "$dom" =~ ^.*\.$ ]]; then
+        false
+    elif [[ "$dom" =~ ^.*\..*$ ]]; then
+        true
+    else
+        false
+    fi
+}
+
 # Determine the remote address of this machine from (in order): commandline parameter,
 # existing configuration or full hostname.
 get_fqdn() {
     local cfgFile="$adminDir/data/config.json"
     local cfgFqdn=$(grep fqdn $cfgFile 2>/dev/null | grep -v LABCA_FQDN | cut -d ":" -f 2- | tr -d " \",")
     LABCA_FQDN=${cfgFqdn:-$(hostname -f)}
 
-    if [ "$cfgFqdn" == "" ]; then
+    while [ "$cfgFqdn" == "" ]; do
         if [ "$cmdlineFqdn" != "" ]; then
             export LABCA_FQDN="$cmdlineFqdn"
         else
             prompt_and_export LABCA_FQDN "$LABCA_FQDN" "FQDN (Fully Qualified Domain Name) for this PKI host (users will use this in their browsers and clients)?"
         fi
+
+        if has_domain $LABCA_FQDN; then
+            cfgFqdn="ok"
+        else
+            msg_err "FQDN must include a hostname AND a domain!"
+            cmdlineFqdn=""
+        fi
+    done
+
+    if ! has_domain $LABCA_FQDN; then
+        msg_fatal "FQDN must include a hostname AND a domain!"
     fi
 
     msg_ok "Determine web address"
@@ -490,6 +528,9 @@ config_boulder() {
     sudo -u labca -H patch -p1 < $cloneDir/ra_ra.patch &>>$installLog
     cp ra/ra.go "$boulderLabCADir/.backup/"
 
+    sudo -u labca -H patch -p1 < $cloneDir/reloader_reloader.patch &>>$installLog
+    cp reloader/reloader.go "$boulderLabCADir/.backup/"
+
     sudo -u labca -H patch -p1 < $cloneDir/mail_mailer.patch &>>$installLog
     cp mail/mailer.go "$boulderLabCADir/.backup/"
 
@@ -584,7 +625,7 @@ config_boulder() {
         export PKI_LOCKDOWN_DOMAINS=$(grep lockdown $adminDir/data/config.json | grep -v domain_mode | sed -e 's/.*:[ ]*//' | sed -e 's/\",//g' | sed -e 's/\"//g')
         export PKI_WHITELIST_DOMAINS=$(grep whitelist $adminDir/data/config.json | grep -v domain_mode | sed -e 's/.*:[ ]*//' | sed -e 's/\",//g' | sed -e 's/\"//g')
 
-        enabled=$(grep "email\": {" config.json -A1 | grep enable | head -1 | perl -p0e 's/.*?:\s+(.*)/\1/' | sed -e 's/\",//g' | sed -e 's/\"//g')
+        enabled=$(grep "email\": {" $adminDir/data/config.json -A1 | grep enable | head -1 | perl -p0e 's/.*?:\s+(.*)/\1/' | sed -e 's/\",//g' | sed -e 's/\"//g')
         if [ "$enabled" == "true," ]; then
             export PKI_EMAIL_SERVER=$(grep server $adminDir/data/config.json | head -1 | perl -p0e 's/.*?:\s+(.*)/\1/' | sed -e 's/\",//g' | sed -e 's/\"//g')
             export PKI_EMAIL_PORT=$(grep port $adminDir/data/config.json | head -1 | perl -p0e 's/.*?:\s+(.*)/\1/' | sed -e 's/\",//g' | sed -e 's/\"//g')
@@ -664,7 +705,7 @@ startup() {
 
     wait_up $PS_MYSQL &>>$installLog
     wait_up $PS_LABCA &>>$installLog
-    wait_up $PS_BOULDER $PS_BOULDER_COUNT &>>$installLog
+    [ -f "$boulderLabCADir/setup_complete" ] && wait_up $PS_BOULDER $PS_BOULDER_COUNT &>>$installLog || /bin/true
 
     msg_ok "$msg"
 }
@@ -709,6 +750,7 @@ main() {
 
     parse_cmdline "$@"
     clone_or_pull "$cloneDir" "$labcaUrl" "$cmdlineBranch"
+    checkout_release "$cmdlineBranch"
     restart_if_updated "$checksum"
 
     get_fqdn

reloader_reloader.patch

@@ -0,0 +1,28 @@
+diff --git a/reloader/reloader.go b/reloader/reloader.go
+index d885af63..ab71babf 100644
+--- a/reloader/reloader.go
++++ b/reloader/reloader.go
+@@ -9,7 +9,7 @@ import (
+
+ // Wrap time.Tick so we can override it in tests.
+ var makeTicker = func() (func(), <-chan time.Time) {
+-	t := time.NewTicker(1 * time.Second)
++	t := time.NewTicker(30 * time.Second)
+ 	return t.Stop, t.C
+ }
+
+@@ -55,8 +55,12 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er
+ 			case <-tickChan:
+ 				currentFileInfo, err := os.Stat(filename)
+ 				if err != nil {
+-					errorCallback(err)
+-					continue
++					time.Sleep(10 * time.Second)
++					currentFileInfo, err = os.Stat(filename)
++					if err != nil {
++						errorCallback(err)
++						continue
++					}
+ 				}
+ 				if !currentFileInfo.ModTime().After(fileInfo.ModTime()) {
+ 					continue