trillian/integration: fix Prometheus config files

Current Prometheus appears to require:
 - absolute paths
 - filenames with .yaml suffix

trillian/integration/ct_functions.sh

@@ -6,6 +6,7 @@ CT_SERVERS=
 CT_CFG=
 CT_LIFECYCLE_CFG=
 CT_COMBINED_CFG=
+PROMETHEUS_CFGDIR=
 
 # ct_prep_test prepares a set of running processes for a CT test.
 # Parameters:
@@ -19,8 +20,9 @@ CT_COMBINED_CFG=
 #  - CT_SERVER_PIDS     : bash array of CT HTTP server pids
 # in addition to the variables populated by log_prep_test.
 # If etcd and Prometheus are configured, it also populates:
-#  - ETCDISCOVER_PID : pid of etcd service watcher
-#  - PROMETHEUS_PID  : pid of local Prometheus server
+#  - ETCDISCOVER_PID   : pid of etcd service watcher
+#  - PROMETHEUS_PID    : pid of local Prometheus server
+#  - PROMETHEUS_CFGDIR : Prometheus configuration directory
 ct_prep_test() {
   # Default to one of everything.
   local rpc_server_count=${1:-1}
@@ -63,13 +65,21 @@ ct_prep_test() {
 
   if [[ -x "${PROMETHEUS_DIR}/prometheus" ]]; then
     if [[ ! -z "${ETCD_OPTS}" ]]; then
+        PROMETHEUS_CFGDIR="$(mktemp -d ${TMPDIR}/ct-prometheus-XXXXXX)"
+        local prom_cfg="${PROMETHEUS_CFGDIR}/config.yaml"
+        local etcdiscovered="${PROMETHEUS_CFGDIR}/trillian.json"
+        sed "s!@ETCDISCOVERED@!${etcdiscovered}!" ${GOPATH}/src/github.com/google/certificate-transparency-go/trillian/integration/prometheus.yml > "${prom_cfg}"
+        echo "Prometheus configuration in ${prom_cfg}:"
+        cat ${prom_cfg}
+
         echo "Building etcdiscover"
         go build github.com/google/trillian/monitoring/prometheus/etcdiscover
-        echo "Launching etcd service monitor"
-        ./etcdiscover ${ETCD_OPTS} --etcd_services=trillian-ctfe-metrics-http,trillian-logserver-http,trillian-logsigner-http -target=./trillian.json --logtostderr &
+
+        echo "Launching etcd service monitor updating ${etcdiscovered}"
+        ./etcdiscover ${ETCD_OPTS} --etcd_services=trillian-ctfe-metrics-http,trillian-logserver-http,trillian-logsigner-http -target=${etcdiscovered} --logtostderr &
         ETCDISCOVER_PID=$!
         echo "Launching Prometheus (default location localhost:9090)"
-        ${PROMETHEUS_DIR}/prometheus --config.file=${GOPATH}/src/github.com/google/certificate-transparency-go/trillian/integration/prometheus.yml \
+        ${PROMETHEUS_DIR}/prometheus --config.file=${prom_cfg} \
                            --web.console.templates=${GOPATH}/src/github.com/google/certificate-transparency-go/trillian/integration/consoles \
                            --web.console.libraries=${GOPATH}/src/github.com/google/certificate-transparency-go/third_party/prometheus/console_libs &
         PROMETHEUS_PID=$!
@@ -105,9 +115,9 @@ ct_provision() {
   CT_COMBINED_CFG=$(mktemp ${TMPDIR}/ct-XXXXXX)
   cat ${CT_CFG} ${CT_LIFECYCLE_CFG} > ${CT_COMBINED_CFG}
 
-  echo "CT Integration Configuration:"
+  echo "CT Integration Configuration in ${CT_CFG}:"
   cat "${CT_CFG}"
-  echo "CT Lifeycle Configuration:"
+  echo "CT Lifeycle Configuration in ${CT_LIFECYCLE_CFG}:"
   cat "${CT_LIFECYCLE_CFG}"
   echo
 }

trillian/integration/ct_hammer_test.sh

@@ -15,7 +15,7 @@ ct_gosmin_config "${CT_SERVER_1}"
 ct_goshawk_config "${CT_SERVER_1}"
 
 # Cleanup for the Trillian components
-TO_DELETE="${TO_DELETE} ${ETCD_DB_DIR}"
+TO_DELETE="${TO_DELETE} ${ETCD_DB_DIR} ${PROMETHEUS_CFGDIR}"
 TO_KILL+=(${LOG_SIGNER_PIDS[@]})
 TO_KILL+=(${RPC_SERVER_PIDS[@]})
 TO_KILL+=(${ETCD_PID})

trillian/integration/ct_integration_test.sh

@@ -12,7 +12,7 @@ HTTP_SERVER_COUNT=${3:-1}
 ct_prep_test "${RPC_SERVER_COUNT}" "${LOG_SIGNER_COUNT}" "${HTTP_SERVER_COUNT}"
 
 # Cleanup for the Trillian components
-TO_DELETE="${TO_DELETE} ${ETCD_DB_DIR}"
+TO_DELETE="${TO_DELETE} ${ETCD_DB_DIR} ${PROMETHEUS_CFGDIR}"
 TO_KILL+=(${LOG_SIGNER_PIDS[@]})
 TO_KILL+=(${RPC_SERVER_PIDS[@]})
 TO_KILL+=(${ETCD_PID})

trillian/integration/prometheus.yml

@@ -6,7 +6,7 @@ global:
 scrape_configs:
   - job_name: 'trillian'  # overwritten by labels in JSON file
     file_sd_configs:
-    - files: ['trillian.json']
+    - files: ['@ETCDISCOVERED@']
   - job_name: 'etcd'
     static_configs:
     - targets: ['localhost:2379']