Merge branch 'matrix-org:main' into master

.buildkite/pipeline.yaml

@@ -1,16 +1,39 @@
 steps:
-  - label: ":books: Build spec"
+  - label: ":snake: Build swagger definitions for matrix.org"
     command:
-      - python3 -m venv env
-      - env/bin/pip install -r scripts/requirements.txt
-      - ". env/bin/activate; scripts/generate-matrix-org-assets"
+      # Install the python dependencies necessary to build the spec
+      - python3 -m venv env && . env/bin/activate
+      - pip install -r scripts/requirements.txt
+      # Build the spec
+      - scripts/generate-matrix-org-assets
     artifact_paths:
       - assets.tar.gz
     plugins:
-      - docker#v3.0.1:
-          image: "python:3.6"
+      - docker#v3.7.0:
+          image: python:3.9
 
   - label: "rebuild matrix.org"
     trigger: "matrix-dot-org"
     async: true
-    branches: "master"
+    branches: "main"
+
+  - label: ":books: Build the spec"
+    command:
+      # Install package dependencies
+      - apk add nodejs npm git hugo
+      # Install the node dependencies necessary to build the spec
+      - npm i
+      # Pull all git submodules, required for the hugo theme
+      - git submodule update --init --recursive
+      # Pull current proposal information
+      - npm run get-proposals
+      # Build the spec, will build to './spec'
+      # Set the baseURL as we're deploying to https://spec.matrix.org/unstable
+      - hugo --baseURL "/unstable" -d "spec"
+      # Compress the result and make it available as an artifact
+      - tar -czf spec.tar.gz spec
+    artifact_paths:
+      - spec.tar.gz
+    plugins:
+      - docker#v3.7.0:
+          image: alpine

.circleci/config.yml

@@ -1,13 +1,19 @@
 gendoc: &gendoc
   name: Generate the docs
+  # Note: Node dependencies are required for the hugo build.
+  # Note: We use a custom config file for circleci due to some specifics with hosting the
+  #       site using CircleCI's artifacts platform. See config-circleci.toml for details.
   command: |
-    source /env/bin/activate
-    scripts/gendoc.py
+    apk add nodejs npm git hugo
+    npm i
+    cat config-circleci.toml config.toml > hugo-config.toml
+    hugo --config hugo-config.toml --baseURL "/${CIRCLE_NODE_INDEX}/public"
 
 genswagger: &genswagger
-  name: Generate the swagger
+  name: Validate sources and generate swagger json
   command: |
     source /env/bin/activate
+    scripts/check-swagger-sources.py
     scripts/dump-swagger.py
 
 buildswaggerui: &buildswaggerui
@@ -27,36 +33,21 @@ checkexamples: &checkexamples
   name: Check Event Examples
   command: |
     source /env/bin/activate
-    cd event-schemas
-    ./check_examples.py
-    cd ../api
-    ./check_examples.py
-
-genmatrixassets: &genmatrixassets
-  name: Generate/Verify matrix.org assets
-  command: |
-    source /env/bin/activate
-    ./scripts/generate-matrix-org-assets
+    scripts/check-event-schema-examples.py
 
 validateapi: &validateapi
   name: Validate OpenAPI specifications
   command: |
-    cd api
+    cd scripts
     npm install
-    node validator.js -s "client-server"
+    node validator.js -s "../data/api/client-server"
 
 buildspeculator: &buildspeculator
   name: Build Speculator
   command: |
     cd scripts/speculator
     go build -v
 
-buildcontinuserv: &buildcontinuserv
-  name: Build Continuserv
-  command: |
-    cd scripts/continuserv
-    go build -v
-
 version: 2
 jobs:
   validate-docs:
@@ -71,18 +62,21 @@ jobs:
     steps:
       - checkout
       - run: *checkexamples
-      - run: *genmatrixassets # We don't actually use the assets, but we do want to make sure they build
   build-docs:
     docker:
-      - image: uhoreg/matrix-doc-build
+      - image: alpine
     steps:
+      # Note: We install git in the image so we can pull git submodules. The hugo theme in use
+      # is a git submodule, which has its own submodules, and all need to be loaded.
+      - run: apk add git
       - checkout
+      - run: git submodule update --init --recursive
       - run: *gendoc
       - store_artifacts:
-          path: scripts/gen
+          path: public
       - run:
           name: "Doc build is available at:"
-          command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/gen/index.html"; echo $DOCS_URL
+          command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/public/index.html"; echo $DOCS_URL
   build-swagger:
     docker:
       - image: uhoreg/matrix-doc-build
@@ -104,8 +98,6 @@ jobs:
           name: Install Dependencies
           command: |
             go get -v github.com/hashicorp/golang-lru
-            go get -v gopkg.in/fsnotify/fsnotify.v1
-      - run: *buildcontinuserv
       - run: *buildspeculator
 
 workflows:

.github/ISSUE_TEMPLATE/clarification.md

@@ -0,0 +1,13 @@
+---
+name: Clarity problem
+about: Report an area of the spec that is unclear.
+title: ''
+labels: 'clarification'
+assignees: ''
+
+---
+
+**Link to problem area**:
+
+**Issue**
+What is wrong? How can we improve?

.github/ISSUE_TEMPLATE/config.yaml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+ - name: Matrix Spec Discussion
+   url: "https://matrix.to/#/#matrix-spec:matrix.org"
+   about: Questions about the spec and proposal process can be asked here.
+ - name: Matrix Security Policy
+   url: https://www.matrix.org/security-disclosure-policy/
+   about: Learn more about our security disclosure policy.

.github/ISSUE_TEMPLATE/cosmetic-bug.md

@@ -0,0 +1,13 @@
+---
+name: Cosmetic issue
+about: Report an issue with how the spec looks.
+title: ''
+labels: 'aesthetic'
+assignees: ''
+
+---
+
+**Link to problem area**:
+
+**Issue**
+What is wrong? What can we do to improve?

.github/ISSUE_TEMPLATE/idea.md

@@ -0,0 +1,12 @@
+---
+name: Spec idea
+about: Suggest a future MSC idea.
+title: ''
+labels: 'improvement'
+assignees: ''
+
+---
+
+**Suggestion**
+What would you like to see in Matrix? If your idea is vaguely complete enough, we
+recommend submitting [an MSC](https://matrix.org/docs/spec/proposals) instead.

.github/ISSUE_TEMPLATE/spec-bug.md

@@ -0,0 +1,16 @@
+---
+name: Documentation error
+about: Report an issue with the spec itself (incorrect text).
+title: ''
+labels: 'spec-bug'
+assignees: ''
+
+---
+
+**Link to problem area**:
+
+**Issue**
+What is wrong?
+
+**Expected behaviour**
+How can the issue be fixed? Links to implementations/documents which prove the spec is wrong are appreciated.

.github/workflows/main.yml

@@ -0,0 +1,67 @@
+name: "Spec"
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build-openapi:
+    name: "🐍 Build OpenAPI definitions for matrix.org"
+    runs-on: ubuntu-latest
+    container: "python:3.9"
+    steps:
+      - name: "📥 Source checkout"
+        uses: actions/checkout@v2
+      - name: "📦 Asset creation"
+        run: |
+          python3 -m venv env && . env/bin/activate
+          pip install -r scripts/requirements.txt
+          scripts/generate-matrix-org-assets
+      - name: "📤 Artifact upload"
+        uses: actions/upload-artifact@v2
+        with:
+          name: openapi-artifact
+          path: assets.tar.gz
+
+  build-spec:
+    name: "📖 Build the spec"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "➕ Setup Node"
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: "➕ Setup Hugo"
+        uses: peaceiris/actions-hugo@c03b5dbed22245418539b65eb9a3b1d5fdd9a0a6
+        with:
+          hugo-version: '0.85.0'
+          extended: true
+      - name: "📥 Source checkout"
+        uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: "⚙️ npm"
+        run: |
+          npm i
+          npm run get-proposals
+      - name: "⚙️ hugo"
+        run: hugo --baseURL "/unstable" -d "spec"
+      - name: "📦 Tarball creation"
+        run: tar -czf spec.tar.gz spec
+      - name: "📤 Artifact upload"
+        uses: actions/upload-artifact@v2
+        with:
+          name: spec-artifact
+          path: spec.tar.gz
+
+  rebuild-matrixdotorg:
+    name: "👷 Rebuild matrix.org"
+    needs: build-openapi
+    runs-on: ubuntu-latest
+    if: ${{ false && github.event_name == 'push' && github.ref == 'refs/head/main' }}
+    steps:
+      - name: "🪄 Triggering rebuild of matrix.org"
+        run: |
+          curl -XPOST -u "${{secrets.TRIGGER_MATRIXORG_REBUILD_TOKEN}}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/${{ github.repository_owner }}/matrix.org/actions/workflows/build-matrix.org.yml/dispatches --data '{"ref": "${{ github.ref }}" }'

.gitignore

@@ -1,15 +1,21 @@
 /api/node_modules
 /assets
 /assets.tar.gz
+/data/msc
 /env*
+/node_modules
+/resources
 /scripts/gen
 /scripts/continuserv/continuserv
 /scripts/speculator/speculator
 /scripts/swagger
 /scripts/tmp
 /templating/out
+/hugo-config.toml
+/public
 *.pyc
 *.swp
 _rendered.rst
 /.vscode/
 /.idea/
+/spec/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "themes/docsy"]
+	path = themes/docsy
+	url = https://github.com/matrix-org/docsy.git
+	branch = master

CONTRIBUTING.rst

@@ -29,9 +29,7 @@ some time to complete.
 Changes to the protocol (new endpoints, ideas, etc) need to go through the
 `proposals process <https://matrix.org/docs/spec/proposals>`_. Other changes,
 such as fixing bugs, typos, or clarifying existing behaviour do not need a proposal.
-If you're not sure, visit us at `#matrix-spec:matrix.org`_
-and ask.
-
+If you're not sure, visit us at `#matrix-spec:matrix.org`_ and ask.
 
 Other changes
 ~~~~~~~~~~~~~
@@ -64,12 +62,17 @@ following:
   to fix. On the other hand, introducing new behaviour is best represented by a
   proposal.
 
+* Design or aesthetic changes, such as improving accessibility, colour schemes,
+  etc. Please check in with us at `#matrix-docs:matrix.org`_ with your proposed
+  design change before opening a PR so we can work with you on it.
+
 For such changes, please do just open a `pull request`_. If you're not sure if
 your change is covered by the above, please visit `#matrix-spec:matrix.org` and
 ask.
 
 .. _`pull request`: https://help.github.com/articles/about-pull-requests
 .. _`#matrix-spec:matrix.org`: https://matrix.to/#/#matrix-spec:matrix.org
+.. _`#matrix-docs:matrix.org`: https://matrix.to/#/#matrix-docs:matrix.org
 
 
 Adding to the changelog
@@ -86,8 +89,8 @@ To create a changelog entry, create a file named in the format ``prNumber.type``
 the ``newsfragments`` directory. The ``type`` can be one of the following:
 
 * ``new`` - Used when adding new endpoints. Please have the file contents be the
-  method and route being added, surrounded in RST code tags. For example: ``POST
-  /accounts/whoami``
+  method and route being added, surrounded in markdown code tags. For example: \`POST
+  /accounts/whoami\`.
 
 * ``feature`` - Used when adding backwards-compatible changes to the API.
 
@@ -100,8 +103,7 @@ the ``newsfragments`` directory. The ``type`` can be one of the following:
 
 All news fragments must have a brief summary explaining the change in the
 contents of the file. The summary must end in a full stop to be in line with
-the style guide and and formatting must be done using `Restructured Text
-<http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html>`_.
+the style guide and formatting must be done using Markdown.
 
 Changes that do not change the spec, such as changes to the build script, formatting,
 CSS, etc should not get a news fragment.