Merge remote-tracking branch 'origin/release-v1.4.0' into dinsic-rele…

…ase-v1.4.0

* origin/release-v1.4.0: (36 commits)
  Improve logging for a failing test (#709)
  Make 3PID binding tests use /account/3pid/bind ala MSC2290 (#703)
  Use unstable prefix for 3PID unbind API
  Add support for handling email validation challenges (#707)
  do requestToken before adding a 3pid (#706)
  Configure synapse to use the test mail server (#705)
  Implement a mail server to help with 3pid testing (#704)
  federated_rooms_fixture (#701)
  Use Sytest develop for Dendrite's master branch (#700)
  Don't assume atomicity
  Fix flakiness due to create_room_synced (#702)
  Don't require an avatar_url
  Replace HOMESERVER_INFO incantations (#699)
  add some logging for flaky sync test (#698)
  Fix problems when using hash
  Accept access_token instead of id_access_token on the IS, accept Authorization header (#697)
  Use hash and handle variable sized final chunk
  Have the in-built identity server support v2 (#689)
  Add tests for 3PID /unbind API (#691)
  Fix typo
  ...

cpanfile

@@ -23,6 +23,8 @@ requires 'DBI';
 requires 'DBD::Pg';
 requires 'Digest::HMAC_SHA1';
 requires 'Digest::SHA';
+requires 'Email::Address::XS';
+requires 'Email::MIME';
 requires 'File::Basename';
 requires 'File::Path';
 requires 'File::Slurper';

docker/Dockerfile

@@ -46,3 +46,7 @@ RUN cpan XML::Generator
 
 # /logs is where we should expect logs to end up
 RUN mkdir /logs
+
+# Add the bootstrap file.
+ADD docker/bootstrap.sh /bootstrap.sh
+RUN dos2unix /bootstrap.sh

docker/Dockerfile-dendrite

@@ -24,9 +24,4 @@ RUN su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF
 # This is where we expect Dendrite to be binded to from the host
 RUN mkdir -p /src
 
-# The dockerfile context, when ran by the buildscript, will actually be the
-# repo root, not the docker folder
-ADD docker/dendrite_sytest.sh /dendrite_sytest.sh
-RUN dos2unix /dendrite_sytest.sh
-
-CMD ["/usr/bin/bash" "/dendrite_sytest.sh"]
+ENTRYPOINT [ "/bin/bash", "/bootstrap.sh", "dendrite" ]

docker/Dockerfile-synapsepy35

@@ -25,11 +25,6 @@ RUN /venv/bin/pip install -q --no-cache-dir lxml psycopg2 coverage codecov
 # and test
 RUN /venv/bin/pip uninstall -q --no-cache-dir -y matrix-synapse
 
-# The dockerfile context, when ran by the buildscript, will actually be the
-# repo root, not the docker folder
-ADD docker/synapse_sytest.sh /synapse_sytest.sh
-RUN dos2unix /synapse_sytest.sh
-
 ADD docker/pydron.py /pydron.py
 
-CMD ["/usr/bin/bash" "/synapse_sytest.sh"]
+ENTRYPOINT [ "/bin/bash", "/bootstrap.sh", "synapse" ]

docker/bootstrap.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Fetch sytest, and then run the sytest running script.
+
+set -ex
+
+if [ -d "/sytest" ]; then
+    # If the user has mounted in a SyTest checkout, use that.
+    echo "Using local sytests..."
+else
+    if [ -n "BUILDKITE_BRANCH" ]; then
+        branch_name=$BUILDKITE_BRANCH
+    else
+        # Otherwise, try and find out what the branch that the Synapse/Dendrite checkout is using. Fall back to develop if it's not a branch.
+        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
+
+        if [ "$1" == "dendrite" ] && [ branch_name == "master" ]; then
+            # Dendrite uses master as its main branch. If the branch is master, we probably want sytest develop
+            branch_name="develop"
+        fi
+    fi
+
+    # Try and fetch the branch
+    echo "Trying to get same-named sytest branch..."
+    wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
+        # Probably a 404, fall back to develop
+        echo "Using develop instead..."
+        wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
+    }
+
+    mkdir -p /sytest
+    tar -C /sytest --strip-components=1 -xf sytest.tar.gz
+fi
+
+export SYTEST_LIB="/sytest/lib"
+SYTEST_SCRIPT="/sytest/docker/$1_sytest.sh"
+
+# dos2unix files that need to be UNIX line ending
+dos2unix $SYTEST_SCRIPT
+dos2unix /sytest/*.pl
+
+# Run the sytest script
+$SYTEST_SCRIPT "${@:2}"

docker/build.sh

@@ -2,6 +2,6 @@
 
 set -e
 
-cd `dirname $0`
-docker build ../ -f Dockerfile -t matrixdotorg/sytest:dinsic
+cd $(dirname $0)
+docker build --pull ../ -f Dockerfile -t matrixdotorg/sytest:dinsic
 docker build ../ -t matrixdotorg/sytest-synapse:dinsic

docker/dendrite_sytest.sh

@@ -2,35 +2,9 @@
 
 set -ex
 
-# Attempt to find a sytest to use.
-# If /test/run-tests.pl exists, it means that a SyTest checkout has been mounted into the Docker image.
-if [ -e "./run-tests.pl" ]
-then
-    # If the user has mounted in a SyTest checkout, use that. We can tell this by files being in the directory.
-    echo "Using local sytests..."
-else
-    # Otherwise, try and find out what the branch that the Dendrite checkout is
-    # using. If we don't know, assume it's master.
-    branch_name="$(git --git-dir=/src/.git rev-parse --abbrev-ref HEAD 2>/dev/null)" || branch_name="develop"
-
-    # If we're using the master branch of Dendrite, use the develop branch of sytest,
-    # as master is Dendrite's development branch
-    [ "$branch_name" == "master" ] && branch_name="develop"
-
-    # Try and fetch the branch
-    echo "Trying to get same-named sytest branch..."
-    wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
-        # Probably a 404, fall back to develop
-        echo "Using develop instead..."
-        wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
-    }
-
-    tar --strip-components=1 -xf sytest.tar.gz
-
-fi
+cd /sytest
 
 # Make sure all Perl deps are installed -- this is done in the docker build so will only install packages added since the last Docker build
-dos2unix ./install-deps.pl
 ./install-deps.pl
 
 # Start the database
@@ -61,7 +35,7 @@ TEST_STATUS=0
 # Copy out the logs
 mkdir -p /logs
 cp results.tap /logs/results.tap
-rsync --ignore-missing-args -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
+rsync --ignore-missing-args --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
 
 # Write out JUnit for CircleCI
 mkdir -p /logs/sytest

docker/synapse_sytest.sh

@@ -1,60 +1,23 @@
 #!/bin/bash
 #
-# Fetch sytest, and then run the tests for synapse. The entrypoint for the
-# sytest-synapse docker images.
+# Run the sytests.
 
 set -ex
 
-# Attempt to find a sytest to use.
-# If /sytest exists, it means that a SyTest checkout has been mounted into the Docker image.
-if [ -d "/sytest" ]; then
-    # If the user has mounted in a SyTest checkout, use that.
-    echo "Using local sytests..."
-
-    # create ourselves a working directory and dos2unix some scripts therein
-    mkdir -p /work/jenkins
-    for i in install-deps.pl run-tests.pl tap-to-junit-xml.pl jenkins/prep_sytest_for_postgres.sh; do
-        dos2unix -n "/sytest/$i" "/work/$i"
-    done
-    ln -sf /sytest/tests /work
-    ln -sf /sytest/keys /work
-    SYTEST_LIB="/sytest/lib"
-else
-    if [ -n "BUILDKITE_BRANCH" ]
-    then
-        branch_name=$BUILDKITE_BRANCH
-    else
-        # Otherwise, try and find out what the branch that the Synapse checkout is using. Fall back to dinsic if it's not a branch.
-        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="dinsic"
-    fi
-
-    # Try and fetch the branch
-    echo "Trying to get same-named sytest branch..."
-    wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
-        # Probably a 404, fall back to dinsic
-        echo "Using dinsic instead..."
-        wget -q https://github.com/matrix-org/sytest/archive/dinsic.tar.gz -O sytest.tar.gz
-    }
-
-    mkdir -p /work
-    tar -C /work --strip-components=1 -xf sytest.tar.gz
-    SYTEST_LIB="/work/lib"
-fi
-
-cd /work
+cd /sytest
 
 # PostgreSQL setup
-if [ -n "$POSTGRES" ]
-then
+if [ -n "$POSTGRES" ]; then
     export PGUSER=postgres
     export POSTGRES_DB_1=pg1
     export POSTGRES_DB_2=pg2
 
     # Start the database
     su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
 
-    # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse
-    jenkins/prep_sytest_for_postgres.sh
+    # Write out the configuration for a PostgreSQL using Synapse
+    dos2unix docker/prep_sytest_for_postgres.sh
+    docker/prep_sytest_for_postgres.sh
 
     # Make the test databases for the two Synapse servers that will be spun up
     su -c 'psql -c "CREATE DATABASE pg1;"' postgres
@@ -75,7 +38,7 @@ else
     # deps.
     /venv/bin/pip install -q --upgrade --no-cache-dir -e /src
     /venv/bin/pip install -q --upgrade --no-cache-dir \
-        lxml psycopg2 coverage codecov tap.py
+        lxml psycopg2 coverage codecov tap.py coverage_enable_subprocess
 
     # Make sure all Perl deps are installed -- this is done in the docker build
     # so will only install packages added since the last Docker build
@@ -87,7 +50,9 @@ if [ -z "$BLACKLIST" ]; then
 fi
 
 # Run the tests
->&2 echo "+++ Running tests"
+echo >&2 "+++ Running tests"
+
+export COVERAGE_PROCESS_START="/src/.coveragerc"
 
 RUN_TESTS=(
     perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=/src -B "/src/$BLACKLIST" --coverage -O tap --all
@@ -101,41 +66,29 @@ else
     RUN_TESTS+=(-I Synapse)
 fi
 
-"${RUN_TESTS[@]}" "$@" > results.tap || TEST_STATUS=$?
+"${RUN_TESTS[@]}" "$@" >results.tap || TEST_STATUS=$?
 
 if [ $TEST_STATUS -ne 0 ]; then
-    >&2 echo -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
+    echo >&2 -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
 else
-    >&2 echo -e "run-tests \e[32mPASSED\e[0m"
+    echo >&2 -e "run-tests \e[32mPASSED\e[0m"
 fi
 
->&2 echo "--- Copying assets"
+echo >&2 "--- Copying assets"
 
 # Copy out the logs
 mkdir -p /logs
 cp results.tap /logs/results.tap
-rsync --ignore-missing-args  --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
-
-# Upload coverage to codecov and upload files, if running on Buildkite
-if [ -n "$BUILDKITE" ]
-then
-    /venv/bin/coverage combine || true
-    /venv/bin/coverage xml || true
-    /venv/bin/codecov -X gcov -f coverage.xml
-
-    wget -O buildkite.tar.gz https://github.com/buildkite/agent/releases/download/v3.13.0/buildkite-agent-linux-amd64-3.13.0.tar.gz
-    tar xvf buildkite.tar.gz
-    chmod +x ./buildkite-agent
-
-    # Upload the files
-    ./buildkite-agent artifact upload "/logs/**/*.log*"
-    ./buildkite-agent artifact upload "/logs/results.tap"
-
-    if [ $TEST_STATUS -ne 0 ]; then
-        # Annotate, if failure
-        /venv/bin/python /src/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" | ./buildkite-agent annotate --style="error" --context="$BUILDKITE_LABEL"
-    fi
-fi
+rsync --ignore-missing-args --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
+cp /.coverage.* /src || true
+
+cd /src
+export TOP=/src
+/venv/bin/coverage combine
 
+if [ $TEST_STATUS -ne 0 ]; then
+    # Build the annotation
+    /venv/bin/python /src/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" >/logs/annotate.md
+fi
 
 exit $TEST_STATUS