From c433e14d76b860cf891f9be4ed27da1293f7373e Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Wed, 31 Jul 2019 01:22:58 +0530
Subject: [PATCH 01/46] Adds group_vars for tor v3 onion services

---
 .../group_vars/securedrop_application_server.yml      | 11 +++++++++++
 .../group_vars/securedrop_monitor_server.yml          |  4 ++++
 2 files changed, 15 insertions(+)

diff --git a/install_files/ansible-base/group_vars/securedrop_application_server.yml b/install_files/ansible-base/group_vars/securedrop_application_server.yml
index 762efd6f87..c6364c3951 100644
--- a/install_files/ansible-base/group_vars/securedrop_application_server.yml
+++ b/install_files/ansible-base/group_vars/securedrop_application_server.yml
@@ -20,6 +20,17 @@ tor_instances:
   - service: journalist
     filename: app-journalist-aths
 
+tor_instances_v3:
+  - "{{ {'service': 'sshv3', 'filename': 'app-sshv3-aths'} if enable_ssh_over_tor else [] }}"
+  - service: sourcev3
+    filename: app-sourcev3-ths
+  - service: journalistv3
+    filename: app-journalistv3-ths
+
+tor_auth_instances_v3:
+  - "{{ 'sshv3' if enable_ssh_over_tor  else [] }}"
+  - "journalistv3"
+
 authd_iprules:
   - chain: OUTPUT
     dest: "{{ monitor_ip  }}"
diff --git a/install_files/ansible-base/group_vars/securedrop_monitor_server.yml b/install_files/ansible-base/group_vars/securedrop_monitor_server.yml
index dca829e96a..854c67e82f 100644
--- a/install_files/ansible-base/group_vars/securedrop_monitor_server.yml
+++ b/install_files/ansible-base/group_vars/securedrop_monitor_server.yml
@@ -14,6 +14,10 @@ local_deb_packages:
 # Configure the tor Onion Services. The Monitor server has only one,
 # for SSH, since no web interfaces.
 tor_instances: "{{ [{ 'service': 'ssh', 'filename': 'mon-ssh-aths'}] if enable_ssh_over_tor else [] }}"
+tor_instances_v3: "{{ [{ 'service': 'ssh', 'filename': 'mon-sshv3-aths'}] if enable_ssh_over_tor else [] }}"
+
+tor_auth_instances_v3:
+  - "{{ 'sshv3' if enable_ssh_over_tor else [] }}"
 
 authd_iprules:
   - chain: INPUT

From 5eaa63ed49040a48a9643a576069ae318889e41e Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Wed, 31 Jul 2019 01:26:48 +0530
Subject: [PATCH 02/46] Adds generation of v3 onion keys if required

We still have to update the requirements files with a future
commit. Requires cryptography >=2.5
---
 admin/securedrop_admin/__init__.py | 61 +++++++++++++++++++++++++++++-
 1 file changed, 60 insertions(+), 1 deletion(-)

diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
index 720b9b28bd..f49af94f2c 100755
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -33,10 +33,14 @@
 import subprocess
 import sys
 import types
+import json
+import base64
 import prompt_toolkit
 from prompt_toolkit.validation import Validator, ValidationError
 import yaml
 from pkg_resources import parse_version
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import x25519
 
 sdlog = logging.getLogger(__name__)
 RELEASE_KEY = '22245C81E3BAEB4138B36061310F561200F4AD77'
@@ -565,18 +569,73 @@ def sdconfig(args):
     SiteConfig(args).load_and_update_config()
     return 0
 
+def generate_new_v3_keys():
+    """This function generate new keys and returns them as tuple.
+
+    :returns: Tuple(public_key, private_key)
+    """
+
+    private_key = x25519.X25519PrivateKey.generate()
+    private_bytes = private_key.private_bytes(
+        encoding=serialization.Encoding.Raw	,
+        format=serialization.PrivateFormat.Raw,
+        encryption_algorithm=serialization.NoEncryption())
+    public_key = private_key.public_key()
+    public_bytes = public_key.public_bytes( 
+        encoding=serialization.Encoding.Raw	,
+        format=serialization.PublicFormat.Raw)
+
+    public = base64.b32encode(public_bytes)[:-4].decode("utf-8")
+    private = base64.b32encode(private_bytes)[:-4].decode("utf-8")
+    return public, private
+
+def save_v3_keys(public_key, private_key, filepath):
+    """
+    Store the keys on the ansible-base directory
+    """
+    with open(filepath, "w") as fobj:
+        json.dump({"public_key": public_key, "private_key": private_key},
+                 fobj)
+
+def get_v3_keys(filepath):
+    """
+    Returns the stored v3 public and private keys as Tuple.
+
+    :returns: Tuple(public_key, private_key)
+    """
+    with open(filepath) as fobj:
+        data = json.load(fobj)
+
+    return data["public_key"], data["private_key"]
+
+def find_or_generate_new_torv3_keys(args):
+    """
+    This method will either read the old keys or generate a new
+    public/private key pair.
+    """
+    secret_key_path = os.path.join(args.ansible_path,
+                                   "tor_v3secret_keys.json")
+    if os.path.exists(secret_key_path):
+        return get_v3_keys(secret_key_path)
+    # No old keys, generate and store them first
+    public_key, private_key = generate_new_v3_keys()
+    save_v3_keys(public_key, private_key, secret_key_path)
+    return public_key, private_key
 
 def install_securedrop(args):
     """Install/Update SecureDrop"""
     SiteConfig(args).load()
 
+    public, private = find_or_generate_new_torv3_keys(args)
     sdlog.info("Now installing SecureDrop on remote servers.")
     sdlog.info("You will be prompted for the sudo password on the "
                "servers.")
     sdlog.info("The sudo password is only necessary during initial "
                "installation.")
     return subprocess.check_call([os.path.join(args.ansible_path,
-                                 'securedrop-prod.yml'), '--ask-become-pass'],
+                                 'securedrop-prod.yml'), '--ask-become-pass',
+                                 '-e', 'publicv3={}'.format(public),
+                                 '-e', 'privatev3={}'.format(private)],
                                  cwd=args.ansible_path)
 
 

From 2c62b811412255818625dab39f740079ef953755 Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Wed, 31 Jul 2019 01:28:31 +0530
Subject: [PATCH 03/46] Adds the initial ansible role update for v3 onion
 addresses

---
 .../tasks/fetch_tor_config.yml                | 35 +++++++++++++
 .../templates/ths_config_v3.j2                |  1 +
 .../tasks/configure_tor_hidden_services.yml   | 50 +++++++++++++++++++
 .../templates/client_auth.j2                  |  1 +
 .../roles/tor-hidden-services/templates/torrc | 20 +++++++-
 5 files changed, 106 insertions(+), 1 deletion(-)
 create mode 100644 install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2

diff --git a/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml b/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml
index 76b278370d..6e3f42017d 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml
+++ b/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml
@@ -29,3 +29,38 @@
   tags:
     - tor
     - admin
+
+
+- name: Wait for all Tor v3 onion services hostname files.
+  wait_for:
+    state: present
+    path: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}/hostname"
+    delay: 5
+  with_items: "{{ tor_instances_v3 }}"
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+
+- name: Collect Tor v3 onion service hostnames.
+  command: cat /var/lib/tor/services/{{ item.service }}/hostname
+  register: tor_hidden_service_hostnamev3_lookup
+  # Read-only task, so don't report changed.
+  changed_when: false
+  with_items: "{{ tor_instances_v3 }}"
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+    - admin
+
+- name: Write Tor v3 onion service hostname files to Admin Workstation.
+  local_action:
+    module: template
+    dest: "{{ role_path }}/../../{{ item.item.filename }}"
+    src: ths_config_v3.j2
+  # Local action, so we don't want elevated privileges
+  become: no
+  with_items: "{{ tor_hidden_service_hostnamev3_lookup.results }}"
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+    - admin
diff --git a/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2 b/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
new file mode 100644
index 0000000000..7da558ddfc
--- /dev/null
+++ b/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
@@ -0,0 +1 @@
+{% if item.item.filename.endswith('-aths') %}HidServAuth {% endif %}{{ item.stdout }}
diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
index e2daed1f26..f970bd7ba1 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
@@ -20,6 +20,18 @@
   tags:
     - tor
 
+- name: Create directories for Tor v3 onion services.
+  file:
+    state: directory
+    dest: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}"
+    owner: "{{ tor_user }}"
+    group: "{{ tor_user }}"
+    mode: "0700"
+  with_items: "{{ tor_instances_v3 }}"
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+
 - name: Copy torrc config file.
   template:
     src: torrc
@@ -43,3 +55,41 @@
     state: started
   tags:
     - tor
+
+# Find the current public key for the v3 onion service
+- name: Register the public key if found in the server
+  command: cat /var/lib/tor/services/publickey
+  register: tor_v3_public_key
+  when: "v3_onion_services|default(False)"
+  ignore_errors: yes
+
+- name: Get the exact key
+  set_fact:
+    tor_publicv3: "{{ tor_v3_public_key.stdout if (tor_v3_public_key.rc == 0) else publicv3 }}"
+
+- name: Create the client auth file on the server
+  template:
+    src: client_auth.j2
+    dest: /var/lib/tor/services/{{ item }}/authorized_clients/client.auth
+    owner: "{{ tor_user }}"
+    group: "{{ tor_user }}"
+    mode: "0700"
+  with_items: "{{ tor_auth_instances_v3 }}"
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+  
+
+- name: Flush handlers to restart Tor.
+  meta: flush_handlers
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+
+- name: Ensure tor is running.
+  service:
+    name: tor
+    state: started
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2
new file mode 100644
index 0000000000..e591ea8837
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2
@@ -0,0 +1 @@
+descriptor:x25519:{{ tor_publicv3 }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/torrc b/install_files/ansible-base/roles/tor-hidden-services/templates/torrc
index 1a711b6692..c1e6a3d63d 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/torrc
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/torrc
@@ -2,7 +2,7 @@ SocksPort 0
 SafeLogging 1
 RunAsDaemon 1
 
-{% if 'securedrop_application_server' in group_names %}
+{% if 'securedrop_application_server' in group_names and v2_onion_services|default(True) %}
 HiddenServiceDir /var/lib/tor/services/source
 HiddenServiceVersion 2
 HiddenServicePort 80 127.0.0.1:80
@@ -22,3 +22,21 @@ HiddenServiceVersion 2
 HiddenServicePort 22 127.0.0.1:22
 HiddenServiceAuthorizeClient stealth admin
 {% endif %}
+
+
+{% if 'securedrop_application_server' in group_names and v3_onion_services|default(False) %}
+HiddenServiceDir /var/lib/tor/services/sourcev3
+HiddenServicePort 80 127.0.0.1:80
+
+{% if securedrop_app_https_on_source_interface|default(False) %}
+HiddenServicePort 443 127.0.0.1:443
+{% endif %}
+
+HiddenServiceDir /var/lib/tor/services/journalistv3
+HiddenServicePort 80 127.0.0.1:8080
+{% endif %}
+
+{% if enable_ssh_over_tor and v3_onion_services|default(False) %}
+HiddenServiceDir /var/lib/tor/services/sshv3
+HiddenServicePort 22 127.0.0.1:22
+{% endif %}

From 3b5eaabec96e069a0869c6261b4f3e10fe46698d Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Wed, 31 Jul 2019 01:29:18 +0530
Subject: [PATCH 04/46] Updates with tor v3 files

---
 .../files/usr.sbin.tor                           | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.tor b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.tor
index 5a2c373f8a..50a75f686a 100644
--- a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.tor
+++ b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.tor
@@ -51,6 +51,22 @@
   /var/lib/tor/services/hostname.tmp rw,
   /var/lib/tor/sevices/private_key rw,
   /var/lib/tor/services/private_key.tmp rw,
+  /var/lib/tor/services/sourcev3/ w,
+  /var/lib/tor/services/sourcev3/hostname rw,
+  /var/lib/tor/services/sourcev3/hs_ed25519_public_key rw,
+  /var/lib/tor/services/sourcev3/hs_ed25519_secret_key rw,
+  /var/lib/tor/services/journalistv3/ w,
+  /var/lib/tor/services/journalistv3/hostname rw,
+  /var/lib/tor/services/journalistv3/hs_ed25519_public_key rw,
+  /var/lib/tor/services/journalistv3/hs_ed25519_secret_key rw,
+  /var/lib/tor/services/journalistv3/authorized_keys/ w,
+  /var/lib/tor/services/journalistv3/authorized_keys/client.auth r,
+  /var/lib/tor/services/sshv3/ w,
+  /var/lib/tor/services/sshv3/hostname rw,
+  /var/lib/tor/services/sshv3/hs_ed25519_public_key rw,
+  /var/lib/tor/services/sshv3/hs_ed25519_secret_key rw,
+  /var/lib/tor/services/sshv3/authorized_keys/ w,
+  /var/lib/tor/services/sshv3/authorized_keys/client.auth r,
   /var/lib/tor/lock rwk,
   /var/lib/tor/state rw,
   /var/lib/tor/state.tmp rw,

From 0921ca696efc85f36395bf60c743b9cbe0605f24 Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Thu, 1 Aug 2019 22:29:38 +0530
Subject: [PATCH 05/46] Marks only v2 service based tasks

---
 .../roles/restrict-direct-access/tasks/fetch_tor_config.yml    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml b/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml
index 6e3f42017d..cefb7017c3 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml
+++ b/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml
@@ -5,6 +5,7 @@
     path: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}/hostname"
     delay: 5
   with_items: "{{ tor_instances }}"
+  when: "v2_onion_services|default(True)"
   tags:
     - tor
 
@@ -14,6 +15,7 @@
   # Read-only task, so don't report changed.
   changed_when: false
   with_items: "{{ tor_instances }}"
+  when: "v2_onion_services|default(True)"
   tags:
     - tor
     - admin
@@ -26,6 +28,7 @@
   # Local action, so we don't want elevated privileges
   become: no
   with_items: "{{ tor_hidden_service_hostname_lookup.results }}"
+  when: "v2_onion_services|default(True)"
   tags:
     - tor
     - admin

From 5769b5ff6e173d8ba84084af68e56fa90414f7ac Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Thu, 1 Aug 2019 22:30:58 +0530
Subject: [PATCH 06/46] Adds proper v3 services for source,journalist and ssh

It also fetches the public and private key from the server if already
available and saves as `tor_v3secret_keys.json`.
---
 .../tasks/configure_tor_hidden_services.yml   | 55 +++++++++++++++++--
 .../templates/save_private_key.j2             |  1 +
 .../templates/save_public_key.j2              |  1 +
 .../templates/tor_v3secret_keys.j2            |  1 +
 4 files changed, 54 insertions(+), 4 deletions(-)
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/save_private_key.j2
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/save_public_key.j2
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3secret_keys.j2

diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
index f970bd7ba1..434a84eff0 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
@@ -58,19 +58,19 @@
 
 # Find the current public key for the v3 onion service
 - name: Register the public key if found in the server
-  command: cat /var/lib/tor/services/publickey
+  command: "cat {{ tor_hidden_services_parent_dir }}/publickey"
   register: tor_v3_public_key
   when: "v3_onion_services|default(False)"
   ignore_errors: yes
 
-- name: Get the exact key
+- name: Get the exact public key
   set_fact:
     tor_publicv3: "{{ tor_v3_public_key.stdout if (tor_v3_public_key.rc == 0) else publicv3 }}"
 
 - name: Create the client auth file on the server
   template:
     src: client_auth.j2
-    dest: /var/lib/tor/services/{{ item }}/authorized_clients/client.auth
+    dest: "{{ tor_hidden_services_parent_dir }}/{{ item }}/authorized_clients/client.auth"
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
     mode: "0700"
@@ -92,4 +92,51 @@
     state: started
   when: "v3_onion_services|default(False)"
   tags:
-    - tor
\ No newline at end of file
+    - tor
+
+# Find the private key in the server if available
+- name: Register the private key if found in the server
+  command: "cat {{ tor_hidden_services_parent_dir }}/privatekey"
+  register: tor_v3_private_key
+  when: "v3_onion_services|default(False)"
+  ignore_errors: yes
+
+- name: Get the exact private key
+  set_fact:
+    tor_privatev3: "{{ tor_v3_private_key.stdout if (tor_v3_private_key.rc == 0) else privatev3 }}"
+
+
+# Save the public key in the server
+- name: Save the public key on the server
+  template:
+    src: save_public_key.j2
+    dest: "{{ tor_hidden_services_parent_dir }}/publickey"
+    owner: "{{ tor_user }}"
+    group: "{{ tor_user }}"
+    mode: "0700"
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+
+- name: Save the private key on the server
+  template:
+    src: save_private_key.j2
+    dest: "{{ tor_hidden_services_parent_dir }}/privatekey"
+    owner: "{{ tor_user }}"
+    group: "{{ tor_user }}"
+    mode: "0700"
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+
+- name: Save Tor Onion v3 public and private key locally.
+  local_action:
+    module: template
+    dest: "{{ role_path }}/../../tor_v3secret_keys.json"
+    src: tor_v3secret_keys.j2
+  # Local action, so we don't want elevated privileges
+  become: no
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+    - admin
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/save_private_key.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/save_private_key.j2
new file mode 100644
index 0000000000..6fd8a10a82
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/save_private_key.j2
@@ -0,0 +1 @@
+{{ tor_privatev3 }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/save_public_key.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/save_public_key.j2
new file mode 100644
index 0000000000..c6b92aa55f
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/save_public_key.j2
@@ -0,0 +1 @@
+{{ tor_publicv3 }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3secret_keys.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3secret_keys.j2
new file mode 100644
index 0000000000..87ded5ea25
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3secret_keys.j2
@@ -0,0 +1 @@
+{"public_key": "{{ tor_publicv3 }}", "private_key": "{{ tor_privatev3 }}"}
\ No newline at end of file

From 8b3ff1b233560827beee6c3519a7c7acdc43cccb Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Thu, 8 Aug 2019 00:01:41 +0530
Subject: [PATCH 07/46] Creates 3 keys for the install command

---
 admin/securedrop_admin/__init__.py            | 30 +++++++++----------
 .../{tor_v3secret_keys.j2 => tor_v3_keys.j2}  |  0
 2 files changed, 14 insertions(+), 16 deletions(-)
 rename install_files/ansible-base/roles/tor-hidden-services/templates/{tor_v3secret_keys.j2 => tor_v3_keys.j2} (100%)

diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
index f49af94f2c..916c0fdb5b 100755
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -589,14 +589,6 @@ def generate_new_v3_keys():
     private = base64.b32encode(private_bytes)[:-4].decode("utf-8")
     return public, private
 
-def save_v3_keys(public_key, private_key, filepath):
-    """
-    Store the keys on the ansible-base directory
-    """
-    with open(filepath, "w") as fobj:
-        json.dump({"public_key": public_key, "private_key": private_key},
-                 fobj)
-
 def get_v3_keys(filepath):
     """
     Returns the stored v3 public and private keys as Tuple.
@@ -606,7 +598,7 @@ def get_v3_keys(filepath):
     with open(filepath) as fobj:
         data = json.load(fobj)
 
-    return data["public_key"], data["private_key"]
+    return data
 
 def find_or_generate_new_torv3_keys(args):
     """
@@ -614,28 +606,34 @@ def find_or_generate_new_torv3_keys(args):
     public/private key pair.
     """
     secret_key_path = os.path.join(args.ansible_path,
-                                   "tor_v3secret_keys.json")
+                                   "tor_v3_keys.json")
     if os.path.exists(secret_key_path):
         return get_v3_keys(secret_key_path)
     # No old keys, generate and store them first
     public_key, private_key = generate_new_v3_keys()
-    save_v3_keys(public_key, private_key, secret_key_path)
-    return public_key, private_key
+    # For app ssh service
+    app_ssh_key, app_ssh_private_key = generate_new_v3_keys()
+    # For mon ssh service
+    mon_ssh_key, mon_ssh_private_key = generate_new_v3_keys()
+    data = {"public_key": public_key, "private_key": private_key,
+            "app_ssh_key": app_ssh_key, "app_ssh_private_key": app_ssh_private_key,
+            "mon_ssh_key": mon_ssh_key, "mon_ssh_private_key": mon_ssh_private_key,}
+    with open(secret_key_path, 'w') as fobj:
+        json.dump(data, fobj)
+
 
 def install_securedrop(args):
     """Install/Update SecureDrop"""
     SiteConfig(args).load()
 
-    public, private = find_or_generate_new_torv3_keys(args)
+    find_or_generate_new_torv3_keys(args)
     sdlog.info("Now installing SecureDrop on remote servers.")
     sdlog.info("You will be prompted for the sudo password on the "
                "servers.")
     sdlog.info("The sudo password is only necessary during initial "
                "installation.")
     return subprocess.check_call([os.path.join(args.ansible_path,
-                                 'securedrop-prod.yml'), '--ask-become-pass',
-                                 '-e', 'publicv3={}'.format(public),
-                                 '-e', 'privatev3={}'.format(private)],
+                                 'securedrop-prod.yml'), '--ask-become-pass'],
                                  cwd=args.ansible_path)
 
 
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3secret_keys.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3_keys.j2
similarity index 100%
rename from install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3secret_keys.j2
rename to install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3_keys.j2

From d7856a8d854ab64b799452a3ddb1a837918a7a88 Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Thu, 8 Aug 2019 00:04:52 +0530
Subject: [PATCH 08/46] Gernarates 3 different keysets and saves them
 accordingly

---
 .../tasks/configure_tor_hidden_services.yml   | 118 +++++++++++++-----
 .../templates/client_auth.j2                  |   2 +-
 .../templates/client_auth_app_ssh.j2          |   1 +
 .../templates/client_auth_mon_ssh.j2          |   1 +
 .../templates/journalist_keys.j2              |   1 +
 .../templates/save_keys_app.j2                |   1 +
 .../templates/save_keys_mon.j2                |   1 +
 .../templates/save_private_key.j2             |   1 -
 .../templates/save_public_key.j2              |   1 -
 .../templates/tor_v3_keys.j2                  |   2 +-
 10 files changed, 91 insertions(+), 38 deletions(-)
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_app_ssh.j2
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_mon_ssh.j2
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/journalist_keys.j2
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_app.j2
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_mon.j2
 delete mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/save_private_key.j2
 delete mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/save_public_key.j2

diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
index 434a84eff0..7eaea02c6f 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
@@ -56,29 +56,74 @@
   tags:
     - tor
 
-# Find the current public key for the v3 onion service
-- name: Register the public key if found in the server
-  command: "cat {{ tor_hidden_services_parent_dir }}/publickey"
+- name: Get the v3 keys locally from the Tails admin system
+  command: "cat {{ role_path }}/../../tor_v3_keys.json"
+  register: localdata
+  delegate_to: 127.0.0.1
+  # Local action, so we don't want elevated privileges
+  become: no
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+    - admin
+
+
+- name: Register the v3 keys if found in the server
+  command: "cat {{ tor_hidden_services_parent_dir }}/tor_v3_keys.json"
   register: tor_v3_public_key
   when: "v3_onion_services|default(False)"
   ignore_errors: yes
 
-- name: Get the exact public key
+- name: Get the keys for app
   set_fact:
-    tor_publicv3: "{{ tor_v3_public_key.stdout if (tor_v3_public_key.rc == 0) else publicv3 }}"
+    tor_v3_app: "{{ tor_v3_public_key.stdout | from_json if (tor_v3_public_key.rc == 0) else localdata.stdout | from_json }}"
+  when:  inventory_hostname == 'app'
 
-- name: Create the client auth file on the server
+- name: Get the keys for mon
+  set_fact:
+    tor_v3_mon: "{{ tor_v3_public_key.stdout | from_json if (tor_v3_public_key.rc == 0) else localdata.stdout | from_json }}"
+  when: inventory_hostname == 'mon'
+
+- name: Create the client auth file for the app server for Journalist interface
   template:
     src: client_auth.j2
-    dest: "{{ tor_hidden_services_parent_dir }}/{{ item }}/authorized_clients/client.auth"
+    dest: "{{ tor_hidden_services_parent_dir }}/journalistv3/authorized_clients/client.auth"
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
-    mode: "0700"
-  with_items: "{{ tor_auth_instances_v3 }}"
-  when: "v3_onion_services|default(False)"
+    mode: "0600"
+  when:
+    - "v3_onion_services|default(False)"
+    - inventory_hostname == 'app'
+  tags:
+    - tor
+
+- name: Create the client auth file for the app server for ssh interface
+  template:
+    src: client_auth_app_ssh.j2
+    dest: "{{ tor_hidden_services_parent_dir }}/sshv3/authorized_clients/client.auth"
+    owner: "{{ tor_user }}"
+    group: "{{ tor_user }}"
+    mode: "0600"
+  when:
+    - "v3_onion_services|default(False)"
+    - inventory_hostname == 'app'
+    - enable_ssh_over_tor
   tags:
     - tor
   
+- name: Create the client auth file for the mon server for ssh interface
+  template:
+    src: client_auth_mon_ssh.j2
+    dest: "{{ tor_hidden_services_parent_dir }}/sshv3/authorized_clients/client.auth"
+    owner: "{{ tor_user }}"
+    group: "{{ tor_user }}"
+    mode: "0600"
+  when:
+    - "v3_onion_services|default(False)"
+    - inventory_hostname == 'mon'
+    - enable_ssh_over_tor
+  tags:
+    - tor
 
 - name: Flush handlers to restart Tor.
   meta: flush_handlers
@@ -94,49 +139,54 @@
   tags:
     - tor
 
-# Find the private key in the server if available
-- name: Register the private key if found in the server
-  command: "cat {{ tor_hidden_services_parent_dir }}/privatekey"
-  register: tor_v3_private_key
-  when: "v3_onion_services|default(False)"
-  ignore_errors: yes
-
-- name: Get the exact private key
-  set_fact:
-    tor_privatev3: "{{ tor_v3_private_key.stdout if (tor_v3_private_key.rc == 0) else privatev3 }}"
 
-
-# Save the public key in the server
-- name: Save the public key on the server
+- name: Save the v3 keys on the app server
   template:
-    src: save_public_key.j2
-    dest: "{{ tor_hidden_services_parent_dir }}/publickey"
+    src: save_keys_app.j2
+    dest: "{{ tor_hidden_services_parent_dir }}/tor_v3_keys.json"
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
-    mode: "0700"
-  when: "v3_onion_services|default(False)"
+    mode: "0600"
+  when:
+    - "v3_onion_services|default(False)"
+    - inventory_hostname == 'app'
   tags:
     - tor
 
-- name: Save the private key on the server
+- name: Save the v3 keys on the mon server
   template:
-    src: save_private_key.j2
-    dest: "{{ tor_hidden_services_parent_dir }}/privatekey"
+    src: save_keys_mon.j2
+    dest: "{{ tor_hidden_services_parent_dir }}/tor_v3_keys.json"
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
-    mode: "0700"
-  when: "v3_onion_services|default(False)"
+    mode: "0600"
+  when:
+    - "v3_onion_services|default(False)"
+    - inventory_hostname == 'mon'
   tags:
     - tor
 
 - name: Save Tor Onion v3 public and private key locally.
   local_action:
     module: template
-    dest: "{{ role_path }}/../../tor_v3secret_keys.json"
-    src: tor_v3secret_keys.j2
+    dest: "{{ role_path }}/../../tor_v3_keys.json"
+    src: tor_v3_keys.j2
   # Local action, so we don't want elevated privileges
   become: no
   when: "v3_onion_services|default(False)"
   tags:
     - tor
     - admin
+
+- name: Save Tor Onion v3 journalist public and private keys locally.
+  local_action:
+    module: template
+    dest: "{{ role_path }}/../../journalist_keys.json"
+    src: journalist_keys.j2
+  # Local action, so we don't want elevated privileges
+  become: no
+  when: "v3_onion_services|default(False)"
+  tags:
+    - tor
+    - admin
+
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2
index e591ea8837..58af48a9e0 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2
@@ -1 +1 @@
-descriptor:x25519:{{ tor_publicv3 }}
\ No newline at end of file
+descriptor:x25519:{{ tor_v3_app["public_key"] }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_app_ssh.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_app_ssh.j2
new file mode 100644
index 0000000000..5f53c5d306
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_app_ssh.j2
@@ -0,0 +1 @@
+descriptor:x25519:{{ tor_v3_app["app_ssh_key"] }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_mon_ssh.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_mon_ssh.j2
new file mode 100644
index 0000000000..0bd0231cb7
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_mon_ssh.j2
@@ -0,0 +1 @@
+descriptor:x25519:{{ tor_v3_mon["mon_ssh_key"] }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/journalist_keys.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/journalist_keys.j2
new file mode 100644
index 0000000000..4bb622d459
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/journalist_keys.j2
@@ -0,0 +1 @@
+{"public_key": "{{ hostvars["app"]["tor_v3_app"]["public_key"] }}", "private_key": "{{ hostvars["app"]["tor_v3_app"]["private_key"] }}"}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_app.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_app.j2
new file mode 100644
index 0000000000..2eb70d2a75
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_app.j2
@@ -0,0 +1 @@
+{"public_key": "{{ tor_v3_app["public_key"] }}", "private_key": "{{ tor_v3_app["private_key"] }}", "app_ssh_private_key": "{{ tor_v3_app["app_ssh_private_key"] }}", "app_ssh_key": "{{ tor_v3_app["app_ssh_key"] }}"}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_mon.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_mon.j2
new file mode 100644
index 0000000000..d38d17206c
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_mon.j2
@@ -0,0 +1 @@
+{"mon_ssh_private_key": "{{ tor_v3_mon["mon_ssh_private_key"] }}", "mon_ssh_key": "{{ tor_v3_mon["mon_ssh_key"] }}"}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/save_private_key.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/save_private_key.j2
deleted file mode 100644
index 6fd8a10a82..0000000000
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/save_private_key.j2
+++ /dev/null
@@ -1 +0,0 @@
-{{ tor_privatev3 }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/save_public_key.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/save_public_key.j2
deleted file mode 100644
index c6b92aa55f..0000000000
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/save_public_key.j2
+++ /dev/null
@@ -1 +0,0 @@
-{{ tor_publicv3 }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3_keys.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3_keys.j2
index 87ded5ea25..7f76463de8 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3_keys.j2
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3_keys.j2
@@ -1 +1 @@
-{"public_key": "{{ tor_publicv3 }}", "private_key": "{{ tor_privatev3 }}"}
\ No newline at end of file
+{"public_key": "{{ hostvars["app"]["tor_v3_app"]["public_key"] }}", "private_key": "{{ hostvars["app"]["tor_v3_app"]["private_key"] }}", "app_ssh_private_key": "{{ hostvars["app"]["tor_v3_app"]["app_ssh_private_key"] }}", "app_ssh_key": "{{ hostvars["app"]["tor_v3_app"]["app_ssh_key"] }}", "mon_ssh_private_key": "{{ hostvars["mon"]["tor_v3_mon"]["mon_ssh_private_key"] }}", "mon_ssh_key": "{{ hostvars["mon"]["tor_v3_mon"]["mon_ssh_key"] }}"}
\ No newline at end of file

From 2f5f93410306e711dcc4f76219811cfb16e769e8 Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Thu, 8 Aug 2019 00:11:22 +0530
Subject: [PATCH 09/46] Updates cryptography to 2.7 for Tor key pair generation

---
 admin/requirements-ansible.in |  2 +-
 admin/requirements.in         |  1 +
 admin/requirements.txt        | 41 +++++++++++++++--------------------
 3 files changed, 19 insertions(+), 25 deletions(-)

diff --git a/admin/requirements-ansible.in b/admin/requirements-ansible.in
index f7a963fe08..c0667262cf 100644
--- a/admin/requirements-ansible.in
+++ b/admin/requirements-ansible.in
@@ -1,3 +1,3 @@
 ansible>2.6<2.7
-cryptography>=2.3
+cryptography>=2.5
 netaddr
diff --git a/admin/requirements.in b/admin/requirements.in
index de6bf04f6e..a93748ab7a 100644
--- a/admin/requirements.in
+++ b/admin/requirements.in
@@ -1,2 +1,3 @@
 prompt_toolkit
 pyyaml
+cryptography==2.7
diff --git a/admin/requirements.txt b/admin/requirements.txt
index c6f4324bfc..36d933bcd0 100644
--- a/admin/requirements.txt
+++ b/admin/requirements.txt
@@ -70,36 +70,29 @@ cffi==1.11.4 \
     --hash=sha256:f4719d0bafc5f0a67b2ec432086d40f653840698d41fa6e9afa679403dea9d78 \
     --hash=sha256:f4992cd7b4c867f453d44c213ee29e8fd484cf81cfece4b6e836d0982b6fa1cf \
     # via bcrypt, cryptography, pynacl
-cryptography==2.3 \
-    --hash=sha256:21af753934f2f6d1a10fe8f4c0a64315af209ef6adeaee63ca349797d747d687 \
-    --hash=sha256:27bb401a20a838d6d0ea380f08c6ead3ccd8c9d8a0232dc9adcc0e4994576a66 \
-    --hash=sha256:29720c4253263cff9aea64585adbbe85013ba647f6e98367efff9db2d7193ded \
-    --hash=sha256:2a35b7570d8f247889784010aac8b384fd2e4a47b33e15c4a60b45a7c1944120 \
-    --hash=sha256:42c531a6a354407f42ee07fda5c2c0dc822cf6d52744949c182f2b295fbd4183 \
-    --hash=sha256:5eb86f03f9c4f0ac2336ac5431271072ddf7ecc76b338e26366732cfac58aa19 \
-    --hash=sha256:67f7f57eae8dede577f3f7775957f5bec93edd6bdb6ce597bb5b28e1bdf3d4fb \
-    --hash=sha256:6ec84edcbc966ae460560a51a90046503ff0b5b66157a9efc61515c68059f6c8 \
-    --hash=sha256:7ba834564daef87557e7fcd35c3c3183a4147b0b3a57314e53317360b9b201b3 \
-    --hash=sha256:7d7f084cbe1fdb82be5a0545062b59b1ad3637bc5a48612ac2eb428ff31b31ea \
-    --hash=sha256:82409f5150e529d699e5c33fa8fd85e965104db03bc564f5f4b6a9199e591f7c \
-    --hash=sha256:87d092a7c2a44e5f7414ab02fb4145723ebba411425e1a99773531dd4c0e9b8d \
-    --hash=sha256:8c56ef989342e42b9fcaba7c74b446f0cc9bed546dd00034fa7ad66fc00307ef \
-    --hash=sha256:9449f5d4d7c516a6118fa9210c4a00f34384cb1d2028672100ee0c6cce49d7f6 \
-    --hash=sha256:bc2301170986ad82d9349a91eb8884e0e191209c45f5541b16aa7c0cfb135978 \
-    --hash=sha256:c132bab45d4bd0fff1d3fe294d92b0a6eb8404e93337b3127bdec9f21de117e6 \
-    --hash=sha256:c3d945b7b577f07a477700f618f46cbc287af3a9222cd73035c6ef527ef2c363 \
-    --hash=sha256:cee18beb4c807b5c0b178f4fa2fae03cef9d51821a358c6890f8b23465b7e5d2 \
-    --hash=sha256:d01dfc5c2b3495184f683574e03c70022674ca9a7be88589c5aba130d835ea90
+cryptography==2.7 \
+    --hash=sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c \
+    --hash=sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643 \
+    --hash=sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216 \
+    --hash=sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799 \
+    --hash=sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a \
+    --hash=sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9 \
+    --hash=sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc \
+    --hash=sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8 \
+    --hash=sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53 \
+    --hash=sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1 \
+    --hash=sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609 \
+    --hash=sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292 \
+    --hash=sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e \
+    --hash=sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6 \
+    --hash=sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed \
+    --hash=sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d
 enum34==1.1.6 \
     --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \
     --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \
     --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
     --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 \
     # via cryptography
-idna==2.6 \
-    --hash=sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f \
-    --hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4 \
-    # via cryptography
 ipaddress==1.0.19 \
     --hash=sha256:200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81 \
     # via cryptography

From 332c6fc30c32303e060a4f39188a4d91aaaca54c Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Thu, 8 Aug 2019 00:21:19 +0530
Subject: [PATCH 10/46] Fixes code formatting for flake8 and yaml

---
 admin/securedrop_admin/__init__.py                  | 13 +++++++++----
 .../tasks/configure_tor_hidden_services.yml         |  5 ++---
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
index 916c0fdb5b..389fea26ea 100755
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -569,6 +569,7 @@ def sdconfig(args):
     SiteConfig(args).load_and_update_config()
     return 0
 
+
 def generate_new_v3_keys():
     """This function generate new keys and returns them as tuple.
 
@@ -581,14 +582,15 @@ def generate_new_v3_keys():
         format=serialization.PrivateFormat.Raw,
         encryption_algorithm=serialization.NoEncryption())
     public_key = private_key.public_key()
-    public_bytes = public_key.public_bytes( 
-        encoding=serialization.Encoding.Raw	,
+    public_bytes = public_key.public_bytes(
+        encoding=serialization.Encoding.Raw,
         format=serialization.PublicFormat.Raw)
 
     public = base64.b32encode(public_bytes)[:-4].decode("utf-8")
     private = base64.b32encode(private_bytes)[:-4].decode("utf-8")
     return public, private
 
+
 def get_v3_keys(filepath):
     """
     Returns the stored v3 public and private keys as Tuple.
@@ -600,6 +602,7 @@ def get_v3_keys(filepath):
 
     return data
 
+
 def find_or_generate_new_torv3_keys(args):
     """
     This method will either read the old keys or generate a new
@@ -616,8 +619,10 @@ def find_or_generate_new_torv3_keys(args):
     # For mon ssh service
     mon_ssh_key, mon_ssh_private_key = generate_new_v3_keys()
     data = {"public_key": public_key, "private_key": private_key,
-            "app_ssh_key": app_ssh_key, "app_ssh_private_key": app_ssh_private_key,
-            "mon_ssh_key": mon_ssh_key, "mon_ssh_private_key": mon_ssh_private_key,}
+            "app_ssh_key": app_ssh_key,
+            "app_ssh_private_key": app_ssh_private_key,
+            "mon_ssh_key": mon_ssh_key,
+            "mon_ssh_private_key": mon_ssh_private_key}
     with open(secret_key_path, 'w') as fobj:
         json.dump(data, fobj)
 
diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
index 7eaea02c6f..cc2ee01791 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
@@ -77,7 +77,7 @@
 - name: Get the keys for app
   set_fact:
     tor_v3_app: "{{ tor_v3_public_key.stdout | from_json if (tor_v3_public_key.rc == 0) else localdata.stdout | from_json }}"
-  when:  inventory_hostname == 'app'
+  when: inventory_hostname == 'app'
 
 - name: Get the keys for mon
   set_fact:
@@ -110,7 +110,7 @@
     - enable_ssh_over_tor
   tags:
     - tor
-  
+
 - name: Create the client auth file for the mon server for ssh interface
   template:
     src: client_auth_mon_ssh.j2
@@ -189,4 +189,3 @@
   tags:
     - tor
     - admin
-

From 50c67d618374d4494e0bacd850eca725974ed7a3 Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Thu, 8 Aug 2019 14:59:29 -0700
Subject: [PATCH 11/46] Uses more explicit var names in v3 keygen

The per-service vars weren't intuitively explained in the JSON file.
Updated the var names so that their use in the corresponding Ansible
logic is more readable, and less error-prone.

We should move the v3 keygen logic out of securedrop-admin and into
Ansible, since otherwise it will not work in developer environments, or
CI.
---
 admin/securedrop_admin/__init__.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
index 389fea26ea..c714617903 100755
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -613,18 +613,21 @@ def find_or_generate_new_torv3_keys(args):
     if os.path.exists(secret_key_path):
         return get_v3_keys(secret_key_path)
     # No old keys, generate and store them first
-    public_key, private_key = generate_new_v3_keys()
+    app_journalist_public_key, app_journalist_private_key = generate_new_v3_keys()
     # For app ssh service
-    app_ssh_key, app_ssh_private_key = generate_new_v3_keys()
+    app_ssh_public_key, app_ssh_private_key = generate_new_v3_keys()
     # For mon ssh service
-    mon_ssh_key, mon_ssh_private_key = generate_new_v3_keys()
-    data = {"public_key": public_key, "private_key": private_key,
-            "app_ssh_key": app_ssh_key,
+    mon_ssh_public_key, mon_ssh_private_key = generate_new_v3_keys()
+    tor_v3_service_info = {
+            "app_journalist_public_key": app_journalist_public_key,
+            "app_journalist_private_key": app_journalist_private_key,
+            "app_ssh_public_key": app_ssh_public_key,
             "app_ssh_private_key": app_ssh_private_key,
-            "mon_ssh_key": mon_ssh_key,
-            "mon_ssh_private_key": mon_ssh_private_key}
+            "mon_ssh_public_key": mon_ssh_public_key,
+            "mon_ssh_private_key": mon_ssh_private_key,
+    }
     with open(secret_key_path, 'w') as fobj:
-        json.dump(data, fobj)
+        json.dump(tor_v3_service_info, fobj, indent=4)
 
 
 def install_securedrop(args):

From 09749f9f59d4c667ab5383ce6d936f84a7bca59c Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Thu, 8 Aug 2019 11:35:12 -0700
Subject: [PATCH 12/46] Terser v3 key munging tasks

Tightens up the logic wrangling Tor v3 Onion keypairs, by reusing logic
for tasks where possible. A number of group_vars already provide insight
into which services should be configured, on v2 or v3, and whether
client auth should be used. Let's reference those vars to apply to the
correct services.

Removes a handful of just-introduced templates from the tor config role,
since the logic for writing the remote files is now at the task level.
---
 .../tasks/configure_tor_hidden_services.yml   | 139 ++++--------------
 .../templates/client_auth.j2                  |   1 -
 .../templates/client_auth_app_ssh.j2          |   1 -
 .../templates/client_auth_mon_ssh.j2          |   1 -
 .../templates/journalist_keys.j2              |   1 -
 .../templates/save_keys_app.j2                |   1 -
 .../templates/save_keys_mon.j2                |   1 -
 .../templates/tor_v3_keys.j2                  |   1 -
 8 files changed, 31 insertions(+), 115 deletions(-)
 delete mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2
 delete mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_app_ssh.j2
 delete mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_mon_ssh.j2
 delete mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/journalist_keys.j2
 delete mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_app.j2
 delete mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_mon.j2
 delete mode 100644 install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3_keys.j2

diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
index cc2ee01791..966083487b 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
@@ -16,19 +16,24 @@
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
     mode: "0700"
-  with_items: "{{ tor_instances }}"
+  with_flattened:
+    - "{{ tor_instances if v2_onion_services else [] }}"
+    - "{{ tor_instances_v3 if v3_onion_services else [] }}"
   tags:
     - tor
 
-- name: Create directories for Tor v3 onion services.
+- name: Create directories for Tor v3 authenticated onion services.
   file:
     state: directory
-    dest: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}"
+    dest: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}/authorized_clients"
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
     mode: "0700"
-  with_items: "{{ tor_instances_v3 }}"
-  when: "v3_onion_services|default(False)"
+  with_items: "{{ tor_auth_instances_v3 }}"
+  when:
+    - v3_onion_services|default(False)
+    # Source Interface is always public, don't configure client auth
+    - "'source' not in item.service"
   tags:
     - tor
 
@@ -44,22 +49,10 @@
   tags:
     - tor
 
-- name: Flush handlers to restart Tor.
-  meta: flush_handlers
-  tags:
-    - tor
-
-- name: Ensure tor is running.
-  service:
-    name: tor
-    state: started
-  tags:
-    - tor
-
 - name: Get the v3 keys locally from the Tails admin system
-  command: "cat {{ role_path }}/../../tor_v3_keys.json"
-  register: localdata
-  delegate_to: 127.0.0.1
+  set_fact:
+    v3_local_key_info: "{{ lookup('file', role_path+'/../../tor_v3_keys.json')|from_json }}"
+  delegate_to: localhost
   # Local action, so we don't want elevated privileges
   become: no
   when: "v3_onion_services|default(False)"
@@ -67,60 +60,41 @@
     - tor
     - admin
 
-
-- name: Register the v3 keys if found in the server
-  command: "cat {{ tor_hidden_services_parent_dir }}/tor_v3_keys.json"
-  register: tor_v3_public_key
-  when: "v3_onion_services|default(False)"
-  ignore_errors: yes
-
-- name: Get the keys for app
+- name: Look up SSH v3 pubkey info.
   set_fact:
-    tor_v3_app: "{{ tor_v3_public_key.stdout | from_json if (tor_v3_public_key.rc == 0) else localdata.stdout | from_json }}"
-  when: inventory_hostname == 'app'
-
-- name: Get the keys for mon
-  set_fact:
-    tor_v3_mon: "{{ tor_v3_public_key.stdout | from_json if (tor_v3_public_key.rc == 0) else localdata.stdout | from_json }}"
-  when: inventory_hostname == 'mon'
-
-- name: Create the client auth file for the app server for Journalist interface
-  template:
-    src: client_auth.j2
-    dest: "{{ tor_hidden_services_parent_dir }}/journalistv3/authorized_clients/client.auth"
-    owner: "{{ tor_user }}"
-    group: "{{ tor_user }}"
-    mode: "0600"
+    tor_v3_ssh_pubkey: "{{ v3_local_key_info.app_ssh_public_key if 'securedrop_application_server' in group_names else v3_local_key_info.mon_ssh_public_key }}"
   when:
-    - "v3_onion_services|default(False)"
-    - inventory_hostname == 'app'
+    - v3_onion_services|default(False)
+    - "'sshv3' in tor_auth_instances_v3"
+    - enable_ssh_over_tor
   tags:
     - tor
 
-- name: Create the client auth file for the app server for ssh interface
-  template:
-    src: client_auth_app_ssh.j2
-    dest: "{{ tor_hidden_services_parent_dir }}/sshv3/authorized_clients/client.auth"
+- name: Create the client auth file for the app server for Journalist interface
+  copy:
+    dest: "{{ tor_hidden_services_parent_dir }}/journalistv3/authorized_clients/client.auth"
+    content: |
+      descriptor:x25519:{{ v3_local_key_info.app_journalist_public_key }}
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
     mode: "0600"
   when:
-    - "v3_onion_services|default(False)"
-    - inventory_hostname == 'app'
-    - enable_ssh_over_tor
+    - v3_onion_services|default(False)
+    - "'journalistv3' in tor_auth_instances_v3"
   tags:
     - tor
 
-- name: Create the client auth file for the mon server for ssh interface
-  template:
-    src: client_auth_mon_ssh.j2
+- name: Create the client auth file for the app server for ssh interface
+  copy:
     dest: "{{ tor_hidden_services_parent_dir }}/sshv3/authorized_clients/client.auth"
+    content: |
+      descriptor:x25519:{{ tor_v3_ssh_pubkey }}
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
     mode: "0600"
   when:
-    - "v3_onion_services|default(False)"
-    - inventory_hostname == 'mon'
+    - v3_onion_services|default(False)
+    - "'sshv3' in tor_auth_instances_v3"
     - enable_ssh_over_tor
   tags:
     - tor
@@ -138,54 +112,3 @@
   when: "v3_onion_services|default(False)"
   tags:
     - tor
-
-
-- name: Save the v3 keys on the app server
-  template:
-    src: save_keys_app.j2
-    dest: "{{ tor_hidden_services_parent_dir }}/tor_v3_keys.json"
-    owner: "{{ tor_user }}"
-    group: "{{ tor_user }}"
-    mode: "0600"
-  when:
-    - "v3_onion_services|default(False)"
-    - inventory_hostname == 'app'
-  tags:
-    - tor
-
-- name: Save the v3 keys on the mon server
-  template:
-    src: save_keys_mon.j2
-    dest: "{{ tor_hidden_services_parent_dir }}/tor_v3_keys.json"
-    owner: "{{ tor_user }}"
-    group: "{{ tor_user }}"
-    mode: "0600"
-  when:
-    - "v3_onion_services|default(False)"
-    - inventory_hostname == 'mon'
-  tags:
-    - tor
-
-- name: Save Tor Onion v3 public and private key locally.
-  local_action:
-    module: template
-    dest: "{{ role_path }}/../../tor_v3_keys.json"
-    src: tor_v3_keys.j2
-  # Local action, so we don't want elevated privileges
-  become: no
-  when: "v3_onion_services|default(False)"
-  tags:
-    - tor
-    - admin
-
-- name: Save Tor Onion v3 journalist public and private keys locally.
-  local_action:
-    module: template
-    dest: "{{ role_path }}/../../journalist_keys.json"
-    src: journalist_keys.j2
-  # Local action, so we don't want elevated privileges
-  become: no
-  when: "v3_onion_services|default(False)"
-  tags:
-    - tor
-    - admin
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2
deleted file mode 100644
index 58af48a9e0..0000000000
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth.j2
+++ /dev/null
@@ -1 +0,0 @@
-descriptor:x25519:{{ tor_v3_app["public_key"] }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_app_ssh.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_app_ssh.j2
deleted file mode 100644
index 5f53c5d306..0000000000
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_app_ssh.j2
+++ /dev/null
@@ -1 +0,0 @@
-descriptor:x25519:{{ tor_v3_app["app_ssh_key"] }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_mon_ssh.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_mon_ssh.j2
deleted file mode 100644
index 0bd0231cb7..0000000000
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/client_auth_mon_ssh.j2
+++ /dev/null
@@ -1 +0,0 @@
-descriptor:x25519:{{ tor_v3_mon["mon_ssh_key"] }}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/journalist_keys.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/journalist_keys.j2
deleted file mode 100644
index 4bb622d459..0000000000
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/journalist_keys.j2
+++ /dev/null
@@ -1 +0,0 @@
-{"public_key": "{{ hostvars["app"]["tor_v3_app"]["public_key"] }}", "private_key": "{{ hostvars["app"]["tor_v3_app"]["private_key"] }}"}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_app.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_app.j2
deleted file mode 100644
index 2eb70d2a75..0000000000
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_app.j2
+++ /dev/null
@@ -1 +0,0 @@
-{"public_key": "{{ tor_v3_app["public_key"] }}", "private_key": "{{ tor_v3_app["private_key"] }}", "app_ssh_private_key": "{{ tor_v3_app["app_ssh_private_key"] }}", "app_ssh_key": "{{ tor_v3_app["app_ssh_key"] }}"}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_mon.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_mon.j2
deleted file mode 100644
index d38d17206c..0000000000
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/save_keys_mon.j2
+++ /dev/null
@@ -1 +0,0 @@
-{"mon_ssh_private_key": "{{ tor_v3_mon["mon_ssh_private_key"] }}", "mon_ssh_key": "{{ tor_v3_mon["mon_ssh_key"] }}"}
\ No newline at end of file
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3_keys.j2 b/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3_keys.j2
deleted file mode 100644
index 7f76463de8..0000000000
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/tor_v3_keys.j2
+++ /dev/null
@@ -1 +0,0 @@
-{"public_key": "{{ hostvars["app"]["tor_v3_app"]["public_key"] }}", "private_key": "{{ hostvars["app"]["tor_v3_app"]["private_key"] }}", "app_ssh_private_key": "{{ hostvars["app"]["tor_v3_app"]["app_ssh_private_key"] }}", "app_ssh_key": "{{ hostvars["app"]["tor_v3_app"]["app_ssh_key"] }}", "mon_ssh_private_key": "{{ hostvars["mon"]["tor_v3_mon"]["mon_ssh_private_key"] }}", "mon_ssh_key": "{{ hostvars["mon"]["tor_v3_mon"]["mon_ssh_key"] }}"}
\ No newline at end of file

From 140207d1ae3914ffe65206032b871980e405af77 Mon Sep 17 00:00:00 2001
From: Kushal Das <kushal@freedom.press>
Date: Mon, 12 Aug 2019 23:13:45 +0530
Subject: [PATCH 13/46] Adds generate-tor-v3-keypairs script

Also adds comment to explain why we need cryptography >= 2.5

This is to generate the key pairs for Tor v3 onion authenticated
services.
---
 admin/requirements-ansible.in                 |  4 ++
 .../files/generate-tor-v3-keypairs            | 52 +++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/files/generate-tor-v3-keypairs

diff --git a/admin/requirements-ansible.in b/admin/requirements-ansible.in
index c0667262cf..b51d9dd629 100644
--- a/admin/requirements-ansible.in
+++ b/admin/requirements-ansible.in
@@ -1,3 +1,7 @@
 ansible>2.6<2.7
+# We need cryptography equal or higher than 2.5 to generate
+# v3 authentication key pairs, the `private_bytes` method was introduced
+# in 2.5 release.
+# https://cryptography.io/en/latest/hazmat/primitives/asymmetric/x25519/?highlight=x25519#cryptography.hazmat.primitives.asymmetric.x25519.X25519PrivateKey.private_bytes
 cryptography>=2.5
 netaddr
diff --git a/install_files/ansible-base/roles/tor-hidden-services/files/generate-tor-v3-keypairs b/install_files/ansible-base/roles/tor-hidden-services/files/generate-tor-v3-keypairs
new file mode 100644
index 0000000000..dd69e2db57
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/files/generate-tor-v3-keypairs
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+import base64
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import x25519
+
+
+def generate_x25519_keypair():
+    """This function generate new keys and returns them as tuple.
+
+    :returns: Tuple(public_key, private_key)
+    """
+
+    private_key = x25519.X25519PrivateKey.generate()
+    private_bytes = private_key.private_bytes(
+        encoding=serialization.Encoding.Raw,
+        format=serialization.PrivateFormat.Raw,
+        encryption_algorithm=serialization.NoEncryption())
+    public_key = private_key.public_key()
+    public_bytes = public_key.public_bytes(
+        encoding=serialization.Encoding.Raw,
+        format=serialization.PublicFormat.Raw)
+
+    public = base64.b32encode(public_bytes)[:-4].decode("utf-8")
+    private = base64.b32encode(private_bytes)[:-4].decode("utf-8")
+    return public, private
+
+
+def generate_new_tor_v3_keypairs():
+    """
+    This method will either read the old keys or generate a new
+    public/private key pair.
+    """
+    # No old keys, generate and store them first
+    app_journalist_public_key, app_journalist_private_key = generate_x25519_keypair()
+    # For app ssh service
+    app_ssh_public_key, app_ssh_private_key = generate_x25519_keypair()
+    # For mon ssh service
+    mon_ssh_public_key, mon_ssh_private_key = generate_x25519_keypair()
+    tor_v3_service_info = {
+            "app_journalist_public_key": app_journalist_public_key,
+            "app_journalist_private_key": app_journalist_private_key,
+            "app_ssh_public_key": app_ssh_public_key,
+            "app_ssh_private_key": app_ssh_private_key,
+            "mon_ssh_public_key": mon_ssh_public_key,
+            "mon_ssh_private_key": mon_ssh_private_key,
+    }
+    # Send results to stdout
+    print(json.dumps(tor_v3_service_info))
+
+
+if __name__ == "__main__":
+    generate_new_tor_v3_keypairs()
\ No newline at end of file

From 9c14e02a23b1d86078e942b82323e85a136af8f2 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Wed, 14 Aug 2019 17:33:29 -0400
Subject: [PATCH 14/46] v3 onions: remove additional standalone script

---
 .../files/generate-tor-v3-keypairs            | 52 -------------------
 1 file changed, 52 deletions(-)
 delete mode 100644 install_files/ansible-base/roles/tor-hidden-services/files/generate-tor-v3-keypairs

diff --git a/install_files/ansible-base/roles/tor-hidden-services/files/generate-tor-v3-keypairs b/install_files/ansible-base/roles/tor-hidden-services/files/generate-tor-v3-keypairs
deleted file mode 100644
index dd69e2db57..0000000000
--- a/install_files/ansible-base/roles/tor-hidden-services/files/generate-tor-v3-keypairs
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env python3
-import base64
-from cryptography.hazmat.primitives import serialization
-from cryptography.hazmat.primitives.asymmetric import x25519
-
-
-def generate_x25519_keypair():
-    """This function generate new keys and returns them as tuple.
-
-    :returns: Tuple(public_key, private_key)
-    """
-
-    private_key = x25519.X25519PrivateKey.generate()
-    private_bytes = private_key.private_bytes(
-        encoding=serialization.Encoding.Raw,
-        format=serialization.PrivateFormat.Raw,
-        encryption_algorithm=serialization.NoEncryption())
-    public_key = private_key.public_key()
-    public_bytes = public_key.public_bytes(
-        encoding=serialization.Encoding.Raw,
-        format=serialization.PublicFormat.Raw)
-
-    public = base64.b32encode(public_bytes)[:-4].decode("utf-8")
-    private = base64.b32encode(private_bytes)[:-4].decode("utf-8")
-    return public, private
-
-
-def generate_new_tor_v3_keypairs():
-    """
-    This method will either read the old keys or generate a new
-    public/private key pair.
-    """
-    # No old keys, generate and store them first
-    app_journalist_public_key, app_journalist_private_key = generate_x25519_keypair()
-    # For app ssh service
-    app_ssh_public_key, app_ssh_private_key = generate_x25519_keypair()
-    # For mon ssh service
-    mon_ssh_public_key, mon_ssh_private_key = generate_x25519_keypair()
-    tor_v3_service_info = {
-            "app_journalist_public_key": app_journalist_public_key,
-            "app_journalist_private_key": app_journalist_private_key,
-            "app_ssh_public_key": app_ssh_public_key,
-            "app_ssh_private_key": app_ssh_private_key,
-            "mon_ssh_public_key": mon_ssh_public_key,
-            "mon_ssh_private_key": mon_ssh_private_key,
-    }
-    # Send results to stdout
-    print(json.dumps(tor_v3_service_info))
-
-
-if __name__ == "__main__":
-    generate_new_tor_v3_keypairs()
\ No newline at end of file

From 847f5e613cc664a160d5df08d6deba4e01811e9f Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Wed, 14 Aug 2019 18:40:01 -0400
Subject: [PATCH 15/46] v3 onions: remove magic number for removing base32
 padding characters

---
 admin/securedrop_admin/__init__.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
index c714617903..a706eddbe0 100755
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -571,7 +571,8 @@ def sdconfig(args):
 
 
 def generate_new_v3_keys():
-    """This function generate new keys and returns them as tuple.
+    """This function generate new keys for Tor v3 onion
+    services and returns them as as tuple.
 
     :returns: Tuple(public_key, private_key)
     """
@@ -586,8 +587,9 @@ def generate_new_v3_keys():
         encoding=serialization.Encoding.Raw,
         format=serialization.PublicFormat.Raw)
 
-    public = base64.b32encode(public_bytes)[:-4].decode("utf-8")
-    private = base64.b32encode(private_bytes)[:-4].decode("utf-8")
+    # Base32 encode and remove base32 padding characters (`=`)
+    public = base64.b32encode(public_bytes).replace('=', '').decode("utf-8")
+    private = base64.b32encode(private_bytes).replace('=', '').decode("utf-8")
     return public, private
 
 
@@ -598,9 +600,9 @@ def get_v3_keys(filepath):
     :returns: Tuple(public_key, private_key)
     """
     with open(filepath) as fobj:
-        data = json.load(fobj)
+        v3_onion_service_keys = json.load(fobj)
 
-    return data
+    return v3_onion_service_keys
 
 
 def find_or_generate_new_torv3_keys(args):
@@ -613,7 +615,8 @@ def find_or_generate_new_torv3_keys(args):
     if os.path.exists(secret_key_path):
         return get_v3_keys(secret_key_path)
     # No old keys, generate and store them first
-    app_journalist_public_key, app_journalist_private_key = generate_new_v3_keys()
+    app_journalist_public_key, \
+        app_journalist_private_key = generate_new_v3_keys()
     # For app ssh service
     app_ssh_public_key, app_ssh_private_key = generate_new_v3_keys()
     # For mon ssh service

From 9c323d36b7a72b8fe93a7304ae0e9c22e0798236 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Wed, 14 Aug 2019 18:40:42 -0400
Subject: [PATCH 16/46] make: pin pip version (fix bug 4667 due to
 pip/pip-tools divergence)

---
 admin/Dockerfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/admin/Dockerfile b/admin/Dockerfile
index 934a77cf1f..58c3213cc1 100644
--- a/admin/Dockerfile
+++ b/admin/Dockerfile
@@ -17,4 +17,7 @@ ENV VIRTUAL_ENV /opt/.venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 COPY requirements-dev.txt .
 RUN pip install --require-hashes -r requirements-dev.txt
+# Now also pin pip due to https://github.com/jazzband/pip-tools/issues/853
+RUN pip install pip==19.1
+
 RUN chown -R $USER_NAME /opt

From 440847c982f7f04817cbae7313c130286ec33244 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Wed, 14 Aug 2019 18:44:36 -0400
Subject: [PATCH 17/46] v3 onions: use cryptography>=2.7

Strictly only cryptography>=2.5 is required (when the private_bytes
method on the X25519 private key object was added (to load raw 25519 private
keys), but since we are bumping the version of cryptography we'll
go to the most recent version
---
 admin/requirements-ansible.in                            | 6 +-----
 admin/requirements.in                                    | 1 -
 securedrop/requirements/python2/develop-requirements.txt | 2 +-
 securedrop/requirements/python3/develop-requirements.txt | 4 ++--
 4 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/admin/requirements-ansible.in b/admin/requirements-ansible.in
index b51d9dd629..09201692c6 100644
--- a/admin/requirements-ansible.in
+++ b/admin/requirements-ansible.in
@@ -1,7 +1,3 @@
 ansible>2.6<2.7
-# We need cryptography equal or higher than 2.5 to generate
-# v3 authentication key pairs, the `private_bytes` method was introduced
-# in 2.5 release.
-# https://cryptography.io/en/latest/hazmat/primitives/asymmetric/x25519/?highlight=x25519#cryptography.hazmat.primitives.asymmetric.x25519.X25519PrivateKey.private_bytes
-cryptography>=2.5
+cryptography>=2.7
 netaddr
diff --git a/admin/requirements.in b/admin/requirements.in
index a93748ab7a..de6bf04f6e 100644
--- a/admin/requirements.in
+++ b/admin/requirements.in
@@ -1,3 +1,2 @@
 prompt_toolkit
 pyyaml
-cryptography==2.7
diff --git a/securedrop/requirements/python2/develop-requirements.txt b/securedrop/requirements/python2/develop-requirements.txt
index 6a8fc9e1ed..3ddbc1905d 100644
--- a/securedrop/requirements/python2/develop-requirements.txt
+++ b/securedrop/requirements/python2/develop-requirements.txt
@@ -32,7 +32,7 @@ click==6.7                # via click-completion, cookiecutter, git-url-parse, m
 colorama==0.3.9           # via molecule, python-gilt
 configparser==3.7.4       # via entrypoints, flake8, pylint
 cookiecutter==1.6.0       # via molecule
-cryptography==2.3.1
+cryptography==2.7
 dnspython==1.15.0
 docker-py==1.10.6
 docker-pycreds==0.2.1     # via docker-py
diff --git a/securedrop/requirements/python3/develop-requirements.txt b/securedrop/requirements/python3/develop-requirements.txt
index 9471850c66..cbf3b2e6ab 100644
--- a/securedrop/requirements/python3/develop-requirements.txt
+++ b/securedrop/requirements/python3/develop-requirements.txt
@@ -28,7 +28,7 @@ click-completion==0.3.1   # via molecule
 click==6.7                # via click-completion, cookiecutter, git-url-parse, molecule, pip-tools, python-gilt, safety
 colorama==0.3.9           # via molecule, python-gilt
 cookiecutter==1.6.0       # via molecule
-cryptography==2.3.1
+cryptography==2.7
 dnspython==1.15.0
 docker-py==1.10.6
 docker-pycreds==0.2.1     # via docker-py
@@ -44,7 +44,7 @@ git-url-parse==1.0.2      # via python-gilt
 gitdb2==2.0.3             # via gitpython
 gitpython==2.1.8          # via bandit
 html-linter==0.4.0
-idna==2.5                 # via cryptography, molecule, requests
+idna==2.5                 # via molecule, requests
 imagesize==0.7.1          # via sphinx
 isort==4.2.15             # via pylint
 jinja2-time==0.2.0        # via cookiecutter

From eb5b30e709b1d62205c130197e374f5fca914c32 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Wed, 14 Aug 2019 18:58:03 -0400
Subject: [PATCH 18/46] v3 onions: move onion services default values to group
 vars

---
 .../ansible-base/group_vars/all/securedrop         |  5 +++++
 .../tasks/fetch_tor_config.yml                     | 12 ++++++------
 .../tasks/configure_tor_hidden_services.yml        | 14 +++++++-------
 .../roles/tor-hidden-services/templates/torrc      |  6 +++---
 4 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/install_files/ansible-base/group_vars/all/securedrop b/install_files/ansible-base/group_vars/all/securedrop
index b797cbab9b..7b3aff0fcf 100644
--- a/install_files/ansible-base/group_vars/all/securedrop
+++ b/install_files/ansible-base/group_vars/all/securedrop
@@ -46,6 +46,11 @@ appserver_dependencies:
 # Enable Tor over SSH by default
 enable_ssh_over_tor: true
 
+# v2 Tor onion services are on / v3 Tor onion services are off by default for backwards
+# compatibility. Note that new install after 1.0 will have v3 enabled by sdconfig which
+# will override these variables.
+v2_onion_services: true
+v3_onion_services: false
 
 # If file is present on system at the end of ansible run
 # force a reboot. Needed because of the de-coupled nature of
diff --git a/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml b/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml
index cefb7017c3..499de3650a 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml
+++ b/install_files/ansible-base/roles/restrict-direct-access/tasks/fetch_tor_config.yml
@@ -5,7 +5,7 @@
     path: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}/hostname"
     delay: 5
   with_items: "{{ tor_instances }}"
-  when: "v2_onion_services|default(True)"
+  when: "v2_onion_services"
   tags:
     - tor
 
@@ -15,7 +15,7 @@
   # Read-only task, so don't report changed.
   changed_when: false
   with_items: "{{ tor_instances }}"
-  when: "v2_onion_services|default(True)"
+  when: "v2_onion_services"
   tags:
     - tor
     - admin
@@ -28,7 +28,7 @@
   # Local action, so we don't want elevated privileges
   become: no
   with_items: "{{ tor_hidden_service_hostname_lookup.results }}"
-  when: "v2_onion_services|default(True)"
+  when: "v2_onion_services"
   tags:
     - tor
     - admin
@@ -40,7 +40,7 @@
     path: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}/hostname"
     delay: 5
   with_items: "{{ tor_instances_v3 }}"
-  when: "v3_onion_services|default(False)"
+  when: "v3_onion_services"
   tags:
     - tor
 
@@ -50,7 +50,7 @@
   # Read-only task, so don't report changed.
   changed_when: false
   with_items: "{{ tor_instances_v3 }}"
-  when: "v3_onion_services|default(False)"
+  when: "v3_onion_services"
   tags:
     - tor
     - admin
@@ -63,7 +63,7 @@
   # Local action, so we don't want elevated privileges
   become: no
   with_items: "{{ tor_hidden_service_hostnamev3_lookup.results }}"
-  when: "v3_onion_services|default(False)"
+  when: "v3_onion_services"
   tags:
     - tor
     - admin
diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
index 966083487b..c8115f860e 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
@@ -31,7 +31,7 @@
     mode: "0700"
   with_items: "{{ tor_auth_instances_v3 }}"
   when:
-    - v3_onion_services|default(False)
+    - v3_onion_services
     # Source Interface is always public, don't configure client auth
     - "'source' not in item.service"
   tags:
@@ -55,7 +55,7 @@
   delegate_to: localhost
   # Local action, so we don't want elevated privileges
   become: no
-  when: "v3_onion_services|default(False)"
+  when: "v3_onion_services"
   tags:
     - tor
     - admin
@@ -64,7 +64,7 @@
   set_fact:
     tor_v3_ssh_pubkey: "{{ v3_local_key_info.app_ssh_public_key if 'securedrop_application_server' in group_names else v3_local_key_info.mon_ssh_public_key }}"
   when:
-    - v3_onion_services|default(False)
+    - v3_onion_services
     - "'sshv3' in tor_auth_instances_v3"
     - enable_ssh_over_tor
   tags:
@@ -79,7 +79,7 @@
     group: "{{ tor_user }}"
     mode: "0600"
   when:
-    - v3_onion_services|default(False)
+    - v3_onion_services
     - "'journalistv3' in tor_auth_instances_v3"
   tags:
     - tor
@@ -93,7 +93,7 @@
     group: "{{ tor_user }}"
     mode: "0600"
   when:
-    - v3_onion_services|default(False)
+    - v3_onion_services
     - "'sshv3' in tor_auth_instances_v3"
     - enable_ssh_over_tor
   tags:
@@ -101,7 +101,7 @@
 
 - name: Flush handlers to restart Tor.
   meta: flush_handlers
-  when: "v3_onion_services|default(False)"
+  when: "v3_onion_services"
   tags:
     - tor
 
@@ -109,6 +109,6 @@
   service:
     name: tor
     state: started
-  when: "v3_onion_services|default(False)"
+  when: "v3_onion_services"
   tags:
     - tor
diff --git a/install_files/ansible-base/roles/tor-hidden-services/templates/torrc b/install_files/ansible-base/roles/tor-hidden-services/templates/torrc
index c1e6a3d63d..d7464fda80 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/templates/torrc
+++ b/install_files/ansible-base/roles/tor-hidden-services/templates/torrc
@@ -2,7 +2,7 @@ SocksPort 0
 SafeLogging 1
 RunAsDaemon 1
 
-{% if 'securedrop_application_server' in group_names and v2_onion_services|default(True) %}
+{% if 'securedrop_application_server' in group_names and v2_onion_services %}
 HiddenServiceDir /var/lib/tor/services/source
 HiddenServiceVersion 2
 HiddenServicePort 80 127.0.0.1:80
@@ -24,7 +24,7 @@ HiddenServiceAuthorizeClient stealth admin
 {% endif %}
 
 
-{% if 'securedrop_application_server' in group_names and v3_onion_services|default(False) %}
+{% if 'securedrop_application_server' in group_names and v3_onion_services %}
 HiddenServiceDir /var/lib/tor/services/sourcev3
 HiddenServicePort 80 127.0.0.1:80
 
@@ -36,7 +36,7 @@ HiddenServiceDir /var/lib/tor/services/journalistv3
 HiddenServicePort 80 127.0.0.1:8080
 {% endif %}
 
-{% if enable_ssh_over_tor and v3_onion_services|default(False) %}
+{% if enable_ssh_over_tor and v3_onion_services %}
 HiddenServiceDir /var/lib/tor/services/sshv3
 HiddenServicePort 22 127.0.0.1:22
 {% endif %}

From f8225460eff053323cd03f85c84537c5a7e30c29 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Wed, 14 Aug 2019 19:00:39 -0400
Subject: [PATCH 19/46] v3 onions: set v3_onion_services to True for staging

---
 install_files/ansible-base/group_vars/staging.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/install_files/ansible-base/group_vars/staging.yml b/install_files/ansible-base/group_vars/staging.yml
index ea8d764d3c..d13ac7d012 100644
--- a/install_files/ansible-base/group_vars/staging.yml
+++ b/install_files/ansible-base/group_vars/staging.yml
@@ -50,6 +50,9 @@ postfix_enable_service: no
 # Otherwise, all SSH connections would be forced over Tor.
 enable_ssh_over_tor: false
 
+# v3 onion services should be available in staging for testing.
+v3_onion_services: true
+
 ### Use for backup restores ###
 # If the `backup_zip` variable is defined ansible will copy the defined file to
 # the app server and run the 0.3_collect.py script to unzip and restore those

From f4cad4918ff645f2274786e1bdc1e2888dde5a8b Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Wed, 14 Aug 2019 23:12:22 -0400
Subject: [PATCH 20/46] v3 onions: loop over all Tor services since source is
 excluded

tor_auth_instances_v3 has no service key, hence why this task was
failing
---
 .../tor-hidden-services/tasks/configure_tor_hidden_services.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
index c8115f860e..1313d14642 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
@@ -29,7 +29,7 @@
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
     mode: "0700"
-  with_items: "{{ tor_auth_instances_v3 }}"
+  with_items: "{{ tor_instances_v3 }}"
   when:
     - v3_onion_services
     # Source Interface is always public, don't configure client auth

From 04667112e1c3177aa9d9b0588d06e103740a6a78 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Wed, 14 Aug 2019 23:43:54 -0400
Subject: [PATCH 21/46] v3 onions: gitignore v3 JSON file

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index 66107224f8..cbec3a02ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,9 @@ wheelhouse
 # ignore the instance information JSON file to prevent commit of private info
 securedrop/tests/functional/instance_information.json
 
+# ignore v3 onion JSON file
+install_files/ansible-base/tor_v3_keys.json
+
 # ignore the ATHS/THS hostname file ansible places
 app-ssh-aths
 app-document-aths # leave this here for historic reasons

From b545bc956a32a020ac58b6d1ec4cee0d44416119 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Wed, 14 Aug 2019 23:59:09 -0400
Subject: [PATCH 22/46] v3 onions: add subcommand, run in ansible task

---
 admin/securedrop_admin/__init__.py                | 14 +++++++++++---
 .../roles/tor-hidden-services/defaults/main.yml   |  1 +
 .../tasks/configure_tor_hidden_services.yml       | 15 +++++++++++++++
 3 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
index a706eddbe0..f1a1c4855d 100755
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -607,13 +607,14 @@ def get_v3_keys(filepath):
 
 def find_or_generate_new_torv3_keys(args):
     """
-    This method will either read the old keys or generate a new
-    public/private key pair.
+    This method will either read v3 Tor onion service keys if found or generate
+    a new public/private keypair.
     """
     secret_key_path = os.path.join(args.ansible_path,
                                    "tor_v3_keys.json")
     if os.path.exists(secret_key_path):
-        return get_v3_keys(secret_key_path)
+        print('Tor v3 onion service keys already exist in: {}'.format(secret_key_path))
+        sys.exit(0)
     # No old keys, generate and store them first
     app_journalist_public_key, \
         app_journalist_private_key = generate_new_v3_keys()
@@ -631,6 +632,8 @@ def find_or_generate_new_torv3_keys(args):
     }
     with open(secret_key_path, 'w') as fobj:
         json.dump(tor_v3_service_info, fobj, indent=4)
+    print('Tor v3 onion service keys generated and stored in: {}'.format(secret_key_path))
+    sys.exit(0)
 
 
 def install_securedrop(args):
@@ -895,6 +898,11 @@ class ArgParseFormatterCombo(argparse.ArgumentDefaultsHelpFormatter,
                                               help=run_tails_config.__doc__)
     parse_tailsconfig.set_defaults(func=run_tails_config)
 
+    parse_generate_tor_keys = subparsers.add_parser(
+        'generate_v3_keys',
+        help=find_or_generate_new_torv3_keys.__doc__)
+    parse_generate_tor_keys.set_defaults(func=find_or_generate_new_torv3_keys)
+
     parse_backup = subparsers.add_parser('backup',
                                          help=backup_securedrop.__doc__)
     parse_backup.set_defaults(func=backup_securedrop)
diff --git a/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml b/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml
index 6e86c95ddf..132b257e5a 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml
@@ -2,3 +2,4 @@
 tor_hidden_services_parent_dir: /var/lib/tor/services
 tor_user: debian-tor
 enable_ssh_over_tor: true
+sd_root_dir: "{{ lookup('pipe','git rev-parse --show-toplevel') }}"
diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
index 1313d14642..d7e25b5bca 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
@@ -49,6 +49,21 @@
   tags:
     - tor
 
+# TODO: While cryptography is available in the development requirements, this task does
+# requires the install of admin requirements in whatever virtualenv is activated for running
+# staging (this seems reasonable to me).
+- name: Generate Onion v3 keys if required the Tails admin system
+  command: "python {{ sd_root_dir }}/admin/securedrop_admin/__init__.py --root {{ sd_root_dir }} generate_v3_keys"
+  delegate_to: localhost
+  # Local action, so we don't want elevated privileges
+  become: no
+  when: "v3_onion_services"
+  register: onion_v3_generation
+  changed_when: "'onion service keys generated' in onion_v3_generation.stdout"
+  tags:
+    - tor
+    - admin
+
 - name: Get the v3 keys locally from the Tails admin system
   set_fact:
     v3_local_key_info: "{{ lookup('file', role_path+'/../../tor_v3_keys.json')|from_json }}"

From dd787ce4e29448c729932b0f1a162ed4b0c2c181 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Thu, 15 Aug 2019 00:08:42 -0400
Subject: [PATCH 23/46] v3 onions: suppress output containing private
 information using no_log

---
 .../tor-hidden-services/tasks/configure_tor_hidden_services.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
index d7e25b5bca..7d39b1a376 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
@@ -70,6 +70,8 @@
   delegate_to: localhost
   # Local action, so we don't want elevated privileges
   become: no
+  # Suppress output since it contains Tor keys
+  no_log: true
   when: "v3_onion_services"
   tags:
     - tor

From 6c1dd65c1f19f9c5c7fda4cb611559a24ac64cf1 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Thu, 15 Aug 2019 11:22:32 -0400
Subject: [PATCH 24/46] v3 onions: register restart tor handler after final v3
 auth tasks

---
 .../tasks/configure_tor_hidden_services.yml                   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
index 7d39b1a376..fc3cabd8d0 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/configure_tor_hidden_services.yml
@@ -95,6 +95,8 @@
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
     mode: "0600"
+  notify:
+    - restart tor
   when:
     - v3_onion_services
     - "'journalistv3' in tor_auth_instances_v3"
@@ -109,6 +111,8 @@
     owner: "{{ tor_user }}"
     group: "{{ tor_user }}"
     mode: "0600"
+  notify:
+    - restart tor
   when:
     - v3_onion_services
     - "'sshv3' in tor_auth_instances_v3"

From 2dc859ba5c954c080cf5cf7175258efdec8e8567 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Thu, 15 Aug 2019 11:49:01 -0400
Subject: [PATCH 25/46] v3 onions: testinfra - tor_services duplicated from
 app-staging

app-staging.yml is what is used in the test_tor_hidden_services tests,
the variables removed in this diff seem to be just unnecessary
duplicated/confusing
---
 molecule/testinfra/staging/vars/staging.yml | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/molecule/testinfra/staging/vars/staging.yml b/molecule/testinfra/staging/vars/staging.yml
index 2049389d6a..fc3c085aed 100644
--- a/molecule/testinfra/staging/vars/staging.yml
+++ b/molecule/testinfra/staging/vars/staging.yml
@@ -51,19 +51,6 @@ app_directories:
   - /var/lib/securedrop/keys
   - /var/lib/securedrop/tmp
 
-tor_services:
-  - name: source
-    ports:
-      - "80"
-    authenticated: no
-
-  - name: journalist
-    ports:
-      - "80"
-      - "8080"
-    authenticated: yes
-    client: journalist
-
 # Staging permits presence of "source-error.log".
 allowed_apache_logfiles:
   - /var/log/apache2/access.log

From 50b21d16568129dac79762f57f1381014c13a54c Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Thu, 15 Aug 2019 11:51:22 -0400
Subject: [PATCH 26/46] v3 onions: testinfra - add v3 interfaces to app-staging
 sdvars

---
 molecule/testinfra/staging/vars/app-staging.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/molecule/testinfra/staging/vars/app-staging.yml b/molecule/testinfra/staging/vars/app-staging.yml
index ee47e2fd83..a3b3ec1c0a 100644
--- a/molecule/testinfra/staging/vars/app-staging.yml
+++ b/molecule/testinfra/staging/vars/app-staging.yml
@@ -56,6 +56,7 @@ tor_services:
     ports:
       - "80"
     authenticated: no
+    version: 2
 
   - name: journalist
     ports:
@@ -63,6 +64,19 @@ tor_services:
       - "8080"
     authenticated: yes
     client: journalist
+    version: 2
+
+  - name: journalistv3
+    ports:
+      - "80"
+    authenticated: yes
+    version: 3
+
+  - name: sourcev3
+    ports:
+      - "80"
+    authenticated: yes
+    version: 3
 
 # Staging permits presence of "source-error.log".
 allowed_apache_logfiles:

From 9154e9d755c5f2ea3e3319fbc0b664cd5f9ef322 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Thu, 15 Aug 2019 12:45:55 -0400
Subject: [PATCH 27/46] v3 onions: testinfra updates for
 test_tor_hidden_services

---
 .../staging/app/test_tor_hidden_services.py   | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/molecule/testinfra/staging/app/test_tor_hidden_services.py b/molecule/testinfra/staging/app/test_tor_hidden_services.py
index 18d21c2299..0e98198ee4 100644
--- a/molecule/testinfra/staging/app/test_tor_hidden_services.py
+++ b/molecule/testinfra/staging/app/test_tor_hidden_services.py
@@ -29,6 +29,7 @@ def test_tor_service_hostnames(host, tor_service):
     # Declare regex only for THS; we'll build regex for ATHS only if
     # necessary, since we won't have the required values otherwise.
     ths_hostname_regex = r"[a-z0-9]{16}\.onion"
+    ths_hostname_regex_v3 = r"[a-z0-9]{56}\.onion"
 
     with host.sudo():
         f = host.file("/var/lib/tor/services/{}/hostname".format(
@@ -41,15 +42,24 @@ def test_tor_service_hostnames(host, tor_service):
         # All hostnames should contain at *least* the hostname.
         assert re.search(ths_hostname_regex, f.content_string)
 
-        if tor_service['authenticated']:
+        if tor_service['authenticated'] and tor_service['version'] == 2:
             # HidServAuth regex is approximately [a-zA-Z0-9/+], but validating
             # the entire entry is sane, and we don't need to nitpick the
             # charset.
             aths_hostname_regex = ths_hostname_regex + " .{22} # client: " + \
                                   tor_service['client']
             assert re.search("^{}$".format(aths_hostname_regex), f.content_string)
-        else:
+        elif tor_service['authenticated'] and tor_service['version'] == 3:
+            # For authenticated version 3 onion services, the authorized_client
+            # directory will exist and contain a file called client.auth.
+            client_auth = host.file(
+                "/var/lib/tor/services/{}/authorized_clients/client.auth".format(
+                    tor_service['name']))
+            assert client_auth.is_file
+        elif tor_service['version'] == 2:
             assert re.search("^{}$".format(ths_hostname_regex), f.content_string)
+        else:
+            assert re.search("^{}$".format(ths_hostname_regex_v3), f.content_string)
 
 
 @pytest.mark.parametrize('tor_service', sdvars.tor_services)
@@ -61,7 +71,7 @@ def test_tor_services_config(host, tor_service):
       * HiddenServiceDir
       * HiddenServicePort
 
-    Only authenticated Onion Services must also include:
+    Only v2 authenticated Onion Services must also include:
 
       * HiddenServiceAuthorizeClient
 
@@ -81,7 +91,8 @@ def test_tor_services_config(host, tor_service):
 
     # Ensure that service is hardcoded to v2, for compatibility
     # with newer versions of Tor, which default to v3.
-    version_string = "HiddenServiceVersion 2"
+    if tor_service['version'] == 2:
+        version_string = "HiddenServiceVersion 2"
 
     port_regex = "HiddenServicePort {} 127.0.0.1:{}".format(
             remote_port, local_port)
@@ -91,7 +102,7 @@ def test_tor_services_config(host, tor_service):
 
     service_regex = "\n".join([dir_regex, version_string, port_regex])
 
-    if tor_service['authenticated']:
+    if tor_service['authenticated'] and tor_service['version'] == 2:
         auth_regex = "HiddenServiceAuthorizeClient stealth {}".format(
                 tor_service['client'])
         assert f.contains("^{}$".format(auth_regex))

From 96497fb9742e1b36aa7289143d2dd0728b01fd6d Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Thu, 15 Aug 2019 12:51:49 -0400
Subject: [PATCH 28/46] v3 onions: add admin requirements to develop
 requirements

---
 Makefile                                                 | 2 ++
 securedrop/requirements/python3/develop-requirements.txt | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index fcc3b2cecd..a070fee555 100644
--- a/Makefile
+++ b/Makefile
@@ -37,6 +37,7 @@ update-python3-requirements:  ## Update Python 3 requirements with pip-compile.
 	@$(DEVSHELL) pip-compile \
 		--output-file requirements/python3/develop-requirements.txt \
 		../admin/requirements-ansible.in \
+		../admin/requirements.in \
 		requirements/python3/develop-requirements.in
 	@$(DEVSHELL) pip-compile \
 		--output-file requirements/python3/test-requirements.txt \
@@ -51,6 +52,7 @@ update-python2-requirements:  ## Update Python 2 requirements with pip-compile.
 	@PYTHON_VERSION=2 $(DEVSHELL) pip-compile \
 		--output-file requirements/python2/develop-requirements.txt \
 		../admin/requirements-ansible.in \
+		../admin/requirements.in \
 		requirements/python2/develop-requirements.in
 	@PYTHON_VERSION=2 $(DEVSHELL) pip-compile \
 		--output-file requirements/python2/test-requirements.txt \
diff --git a/securedrop/requirements/python3/develop-requirements.txt b/securedrop/requirements/python3/develop-requirements.txt
index cbf3b2e6ab..ac111022a7 100644
--- a/securedrop/requirements/python3/develop-requirements.txt
+++ b/securedrop/requirements/python3/develop-requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=requirements/python3/develop-requirements.txt ../admin/requirements-ansible.in requirements/python3/develop-requirements.in
+#    pip-compile --output-file=requirements/python3/develop-requirements.txt ../admin/requirements-ansible.in ../admin/requirements.in requirements/python3/develop-requirements.in
 #
 alabaster==0.7.10         # via sphinx
 ansible-lint==4.1.0       # via molecule
@@ -68,6 +68,7 @@ pexpect==4.6.0            # via molecule
 pip-tools==4.0.0
 port_for==0.3.1           # via sphinx-autobuild
 poyo==0.4.1               # via cookiecutter
+prompt-toolkit==2.0.9
 psutil==5.4.6             # via molecule
 ptyprocess==0.5.2         # via pexpect
 py==1.4.34                # via pytest
@@ -92,7 +93,7 @@ ruamel.yaml==0.15.97      # via ansible-lint
 s3transfer==0.1.12        # via boto3
 safety==1.8.4
 sh==1.12.14               # via molecule, python-gilt
-six==1.11.0               # via ansible-lint, astroid, bandit, bcrypt, click-completion, cryptography, docker-py, docker-pycreds, dparse, fasteners, git-url-parse, livereload, molecule, packaging, pip-tools, pylint, pynacl, python-dateutil, sphinx, stevedore, testinfra, websocket-client
+six==1.11.0               # via ansible-lint, astroid, bandit, bcrypt, click-completion, cryptography, docker-py, docker-pycreds, dparse, fasteners, git-url-parse, livereload, molecule, packaging, pip-tools, prompt-toolkit, pylint, pynacl, python-dateutil, sphinx, stevedore, testinfra, websocket-client
 smmap2==2.0.3             # via gitdb2
 snowballstemmer==1.2.1    # via sphinx
 sphinx-autobuild==0.7.1
@@ -108,6 +109,7 @@ tree-format==0.1.2        # via molecule
 typed-ast==1.3.5          # via mypy
 urllib3==1.25.3
 watchdog==0.8.3           # via sphinx-autobuild
+wcwidth==0.1.7            # via prompt-toolkit
 websocket-client==0.44.0  # via docker-py
 whichcraft==0.4.1         # via cookiecutter
 wrapt==1.10.11            # via astroid

From a866a226aa1150062ec57e5dd4ac3bc854a2aa0e Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Thu, 15 Aug 2019 14:16:38 -0400
Subject: [PATCH 29/46] makefile: update-pip-requirements should also update
 python 2 requirements

---
 Makefile                                                 | 2 +-
 securedrop/requirements/python2/develop-requirements.txt | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index a070fee555..cad651e579 100644
--- a/Makefile
+++ b/Makefile
@@ -62,7 +62,7 @@ update-python2-requirements:  ## Update Python 2 requirements with pip-compile.
 		requirements/python2/securedrop-app-code-requirements.in
 
 .PHONY: update-pip-requirements
-update-pip-requirements: update-admin-pip-requirements update-python3-requirements ## Update all requirements with pip-compile.
+update-pip-requirements: update-admin-pip-requirements update-python2-requirements update-python3-requirements ## Update all requirements with pip-compile.
 
 
 #################
diff --git a/securedrop/requirements/python2/develop-requirements.txt b/securedrop/requirements/python2/develop-requirements.txt
index 3ddbc1905d..9830af978b 100644
--- a/securedrop/requirements/python2/develop-requirements.txt
+++ b/securedrop/requirements/python2/develop-requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=requirements/python2/develop-requirements.txt ../admin/requirements-ansible.in requirements/python2/develop-requirements.in
+#    pip-compile --output-file=requirements/python2/develop-requirements.txt ../admin/requirements-ansible.in ../admin/requirements.in requirements/python2/develop-requirements.in
 #
 alabaster==0.7.10         # via sphinx
 ansible-lint==4.1.0       # via molecule
@@ -51,7 +51,7 @@ git-url-parse==1.0.2      # via python-gilt
 gitdb2==2.0.3             # via gitpython
 gitpython==2.1.8          # via bandit
 html-linter==0.4.0
-idna==2.5                 # via cryptography, molecule, requests
+idna==2.5                 # via molecule, requests
 imagesize==0.7.1          # via sphinx
 ipaddress==1.0.22         # via cryptography, docker-py
 isort==4.2.15             # via pylint
@@ -74,6 +74,7 @@ pexpect==4.6.0            # via molecule
 pip-tools==4.0.0
 port-for==0.3.1           # via sphinx-autobuild
 poyo==0.4.1               # via cookiecutter
+prompt-toolkit==2.0.9
 psutil==5.4.6             # via molecule
 ptyprocess==0.5.2         # via pexpect
 py==1.4.34                # via pytest
@@ -100,7 +101,7 @@ s3transfer==0.1.12        # via boto3
 safety==1.8.4
 sh==1.12.14               # via molecule, python-gilt
 singledispatch==3.4.0.3   # via astroid, pylint, tornado
-six==1.11.0               # via ansible-lint, astroid, bandit, bcrypt, click-completion, cryptography, docker-py, docker-pycreds, dparse, fasteners, git-url-parse, livereload, molecule, packaging, pip-tools, pylint, pynacl, python-dateutil, singledispatch, sphinx, stevedore, testinfra, websocket-client
+six==1.11.0               # via ansible-lint, astroid, bandit, bcrypt, click-completion, cryptography, docker-py, docker-pycreds, dparse, fasteners, git-url-parse, livereload, molecule, packaging, pip-tools, prompt-toolkit, pylint, pynacl, python-dateutil, singledispatch, sphinx, stevedore, testinfra, websocket-client
 smmap2==2.0.3             # via gitdb2
 snowballstemmer==1.2.1    # via sphinx
 sphinx-autobuild==0.7.1
@@ -116,6 +117,7 @@ tree-format==0.1.2        # via molecule
 typing==3.6.6             # via flake8, sphinx
 urllib3==1.25.3
 watchdog==0.8.3           # via sphinx-autobuild
+wcwidth==0.1.7            # via prompt-toolkit
 websocket-client==0.44.0  # via docker-py
 whichcraft==0.4.1         # via cookiecutter
 wrapt==1.10.11            # via astroid

From c94d3dab94b8ec61213c807901feabfba6c7b83f Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Thu, 15 Aug 2019 11:11:59 -0700
Subject: [PATCH 30/46] Supports Python 2 & 3 in v3 keygen logic

The securedrop-admin virtualenv is still py2, but the rest of the
project, including developer env, uses py3. Let's modify just the keygen
logic for v3 URLs to work under both Python 2 & Python 3.
---
 admin/securedrop_admin/__init__.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
index f1a1c4855d..d19325bb8f 100755
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -588,8 +588,15 @@ def generate_new_v3_keys():
         format=serialization.PublicFormat.Raw)
 
     # Base32 encode and remove base32 padding characters (`=`)
-    public = base64.b32encode(public_bytes).replace('=', '').decode("utf-8")
-    private = base64.b32encode(private_bytes).replace('=', '').decode("utf-8")
+    # Using try/except blocks for Python 2/3 support.
+    try:
+        public = base64.b32encode(public_bytes).replace('=', '').decode("utf-8")
+    except TypeError:
+        public = base64.b32encode(public_bytes).replace(b'=', b'').decode("utf-8")
+    try:
+        private = base64.b32encode(private_bytes).replace('=', '').decode("utf-8")
+    except TypeError:
+        private = base64.b32encode(private_bytes).replace(b'=', b'').decode("utf-8")
     return public, private
 
 

From c11af151d6401b7ef551a3221b0b5540a839663b Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Thu, 15 Aug 2019 14:24:55 -0700
Subject: [PATCH 31/46] Makes v3 keygen play nicely with install action

Two changes here. First, we remove the sys.exit() calls inside the v3
keygen logic. Those worked well enough when the script is invoked
directly via the subcommand, but causes the `./securedrop-admin install`
exit to quit before calling out to Ansible. We replace the sys.exit
calls with `return 0`, since the parent function expects an exit code
to be returned, and passes this to the shell.

Second, we stop calling the keygen logic automatically as part of the
install action. If anything, it'd be more appropriate to configure those
values as part of `sdconfig`, but since the generation depends on the
site config already being evaluted (i.e. v3_onion_services=True), we
shouldn't automatically generate keys before inspecting the config.
---
 admin/securedrop_admin/__init__.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
index d19325bb8f..f3cca8ba4b 100755
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -621,7 +621,7 @@ def find_or_generate_new_torv3_keys(args):
                                    "tor_v3_keys.json")
     if os.path.exists(secret_key_path):
         print('Tor v3 onion service keys already exist in: {}'.format(secret_key_path))
-        sys.exit(0)
+        return 0
     # No old keys, generate and store them first
     app_journalist_public_key, \
         app_journalist_private_key = generate_new_v3_keys()
@@ -640,14 +640,13 @@ def find_or_generate_new_torv3_keys(args):
     with open(secret_key_path, 'w') as fobj:
         json.dump(tor_v3_service_info, fobj, indent=4)
     print('Tor v3 onion service keys generated and stored in: {}'.format(secret_key_path))
-    sys.exit(0)
+    return 0
 
 
 def install_securedrop(args):
     """Install/Update SecureDrop"""
     SiteConfig(args).load()
 
-    find_or_generate_new_torv3_keys(args)
     sdlog.info("Now installing SecureDrop on remote servers.")
     sdlog.info("You will be prompted for the sudo password on the "
                "servers.")

From 0399bdcf9ec8f0e34f7d8f1dbc6888f822e7ccbe Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Thu, 15 Aug 2019 18:08:36 -0400
Subject: [PATCH 32/46] v3 onions: fix testinfra tor tests

---
 .../staging/app/test_tor_hidden_services.py   |  7 ++++-
 .../testinfra/staging/vars/app-staging.yml    |  2 +-
 molecule/testinfra/staging/vars/staging.yml   | 27 +++++++++++++++++++
 3 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/molecule/testinfra/staging/app/test_tor_hidden_services.py b/molecule/testinfra/staging/app/test_tor_hidden_services.py
index 0e98198ee4..319cdd1f25 100644
--- a/molecule/testinfra/staging/app/test_tor_hidden_services.py
+++ b/molecule/testinfra/staging/app/test_tor_hidden_services.py
@@ -93,6 +93,8 @@ def test_tor_services_config(host, tor_service):
     # with newer versions of Tor, which default to v3.
     if tor_service['version'] == 2:
         version_string = "HiddenServiceVersion 2"
+    else:
+        version_string = ""
 
     port_regex = "HiddenServicePort {} 127.0.0.1:{}".format(
             remote_port, local_port)
@@ -100,7 +102,10 @@ def test_tor_services_config(host, tor_service):
     assert f.contains("^{}$".format(dir_regex))
     assert f.contains("^{}$".format(port_regex))
 
-    service_regex = "\n".join([dir_regex, version_string, port_regex])
+    if version_string:
+        service_regex = "\n".join([dir_regex, version_string, port_regex])
+    else:
+        service_regex = "\n".join([dir_regex, port_regex])
 
     if tor_service['authenticated'] and tor_service['version'] == 2:
         auth_regex = "HiddenServiceAuthorizeClient stealth {}".format(
diff --git a/molecule/testinfra/staging/vars/app-staging.yml b/molecule/testinfra/staging/vars/app-staging.yml
index a3b3ec1c0a..1477456136 100644
--- a/molecule/testinfra/staging/vars/app-staging.yml
+++ b/molecule/testinfra/staging/vars/app-staging.yml
@@ -75,7 +75,7 @@ tor_services:
   - name: sourcev3
     ports:
       - "80"
-    authenticated: yes
+    authenticated: no
     version: 3
 
 # Staging permits presence of "source-error.log".
diff --git a/molecule/testinfra/staging/vars/staging.yml b/molecule/testinfra/staging/vars/staging.yml
index fc3c085aed..f18c27d8ff 100644
--- a/molecule/testinfra/staging/vars/staging.yml
+++ b/molecule/testinfra/staging/vars/staging.yml
@@ -51,6 +51,33 @@ app_directories:
   - /var/lib/securedrop/keys
   - /var/lib/securedrop/tmp
 
+tor_services:
+  - name: source
+    ports:
+      - "80"
+    authenticated: no
+    version: 2
+
+  - name: journalist
+    ports:
+      - "80"
+      - "8080"
+    authenticated: yes
+    client: journalist
+    version: 2
+
+  - name: journalistv3
+    ports:
+      - "80"
+    authenticated: yes
+    version: 3
+
+  - name: sourcev3
+    ports:
+      - "80"
+    authenticated: no
+    version: 3
+
 # Staging permits presence of "source-error.log".
 allowed_apache_logfiles:
   - /var/log/apache2/access.log

From 78f83faac8dd6a51bdd2ccf89eff7f82627ae94e Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Thu, 15 Aug 2019 15:13:18 -0700
Subject: [PATCH 33/46] Fixes tor v3 ssh service for Monitor Server

The service name is different for v2/v3 Onion services; the v3 config
was missing the "v3" suffix on the end of the service name, causing the
install to fail when writing.
---
 .../ansible-base/group_vars/securedrop_monitor_server.yml       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install_files/ansible-base/group_vars/securedrop_monitor_server.yml b/install_files/ansible-base/group_vars/securedrop_monitor_server.yml
index 854c67e82f..ef32a0cb94 100644
--- a/install_files/ansible-base/group_vars/securedrop_monitor_server.yml
+++ b/install_files/ansible-base/group_vars/securedrop_monitor_server.yml
@@ -14,7 +14,7 @@ local_deb_packages:
 # Configure the tor Onion Services. The Monitor server has only one,
 # for SSH, since no web interfaces.
 tor_instances: "{{ [{ 'service': 'ssh', 'filename': 'mon-ssh-aths'}] if enable_ssh_over_tor else [] }}"
-tor_instances_v3: "{{ [{ 'service': 'ssh', 'filename': 'mon-sshv3-aths'}] if enable_ssh_over_tor else [] }}"
+tor_instances_v3: "{{ [{ 'service': 'sshv3', 'filename': 'mon-sshv3-aths'}] if enable_ssh_over_tor else [] }}"
 
 tor_auth_instances_v3:
   - "{{ 'sshv3' if enable_ssh_over_tor else [] }}"

From 816fdd8e2b1be43d14d22cedb75b1e1b2044ff62 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Thu, 15 Aug 2019 18:43:52 -0400
Subject: [PATCH 34/46] v3 onion: fix linting issues, add test for
 generate_new_v3_keys

---
 admin/securedrop_admin/__init__.py   | 18 ++++++++++++------
 admin/tests/test_securedrop-admin.py |  9 +++++++++
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
index f3cca8ba4b..ff2dcffbfe 100755
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -590,13 +590,17 @@ def generate_new_v3_keys():
     # Base32 encode and remove base32 padding characters (`=`)
     # Using try/except blocks for Python 2/3 support.
     try:
-        public = base64.b32encode(public_bytes).replace('=', '').decode("utf-8")
+        public = base64.b32encode(public_bytes).replace('=', '') \
+                       .decode("utf-8")
     except TypeError:
-        public = base64.b32encode(public_bytes).replace(b'=', b'').decode("utf-8")
+        public = base64.b32encode(public_bytes).replace(b'=', b'') \
+                       .decode("utf-8")
     try:
-        private = base64.b32encode(private_bytes).replace('=', '').decode("utf-8")
+        private = base64.b32encode(private_bytes).replace('=', '') \
+                        .decode("utf-8")
     except TypeError:
-        private = base64.b32encode(private_bytes).replace(b'=', b'').decode("utf-8")
+        private = base64.b32encode(private_bytes).replace(b'=', b'') \
+                        .decode("utf-8")
     return public, private
 
 
@@ -620,7 +624,8 @@ def find_or_generate_new_torv3_keys(args):
     secret_key_path = os.path.join(args.ansible_path,
                                    "tor_v3_keys.json")
     if os.path.exists(secret_key_path):
-        print('Tor v3 onion service keys already exist in: {}'.format(secret_key_path))
+        print('Tor v3 onion service keys already exist in: {}'.format(
+            secret_key_path))
         return 0
     # No old keys, generate and store them first
     app_journalist_public_key, \
@@ -639,7 +644,8 @@ def find_or_generate_new_torv3_keys(args):
     }
     with open(secret_key_path, 'w') as fobj:
         json.dump(tor_v3_service_info, fobj, indent=4)
-    print('Tor v3 onion service keys generated and stored in: {}'.format(secret_key_path))
+    print('Tor v3 onion service keys generated and stored in: {}'.format(
+        secret_key_path))
     return 0
 
 
diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py
index 31b34c1d6a..e5126f19ab 100644
--- a/admin/tests/test_securedrop-admin.py
+++ b/admin/tests/test_securedrop-admin.py
@@ -1008,3 +1008,12 @@ def test_load(self, caplog):
         with pytest.raises(yaml.YAMLError) as e:
             site_config.load()
         assert 'issue processing' in caplog.text
+
+
+def test_generate_new_v3_keys():
+    public, private = securedrop_admin.generate_new_v3_keys()
+
+    for key in [public, private]:
+        # base32 padding characters should be removed
+        assert '=' not in key
+        assert len(key) == 52

From 0248ef03665e32da178a7cead32c44812b133e3b Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Thu, 15 Aug 2019 16:28:27 -0700
Subject: [PATCH 35/46] Formats v3 client auth files in restrict-direct-access

Implementing suggestion by @zenmonkeykstop, to format the files directly
on the Admin Workstation in the format required for ingestion by the
`tailsconfig` logic in securedrop-admin.

Modified the filename for the journalistv3 service, so that the existing
"if filename ends with '-aths'" logic will work, as with the SSH
services.

h/t to @redshiftzero for an assist on the Jinja.
---
 .../group_vars/securedrop_application_server.yml         | 2 +-
 .../roles/restrict-direct-access/defaults/main.yml       | 9 +++++++++
 .../restrict-direct-access/templates/ths_config_v3.j2    | 2 +-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/install_files/ansible-base/group_vars/securedrop_application_server.yml b/install_files/ansible-base/group_vars/securedrop_application_server.yml
index c6364c3951..97c10893b6 100644
--- a/install_files/ansible-base/group_vars/securedrop_application_server.yml
+++ b/install_files/ansible-base/group_vars/securedrop_application_server.yml
@@ -25,7 +25,7 @@ tor_instances_v3:
   - service: sourcev3
     filename: app-sourcev3-ths
   - service: journalistv3
-    filename: app-journalistv3-ths
+    filename: app-journalistv3-aths
 
 tor_auth_instances_v3:
   - "{{ 'sshv3' if enable_ssh_over_tor  else [] }}"
diff --git a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
index 8028884f60..44e2adb821 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
+++ b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
@@ -21,3 +21,12 @@ admin_net_int:
   Darwin:
     cmd: "/sbin/route -n get "
     rgx: "(?<=interface: )\\w+"
+
+# Lookup table for querying keypair info from the local JSON
+# file on the Admin Workstation, required for configuring client
+# auth on Tor v3 Onion URLs. See the tor_v3_keys.json file for
+# reference on structure.
+tor_v3_service_map:
+  app-journalistv3-aths: "app_journalist_private_key"
+  app-sshv3-aths: "app_ssh_private_key"
+  mon-sshv3-aths: "mon_ssh_private_key"
diff --git a/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2 b/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
index 7da558ddfc..b0f2a69d1d 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
+++ b/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
@@ -1 +1 @@
-{% if item.item.filename.endswith('-aths') %}HidServAuth {% endif %}{{ item.stdout }}
+{% if item.item.filename.endswith('-aths') %}{{ item.stdout|truncate(56, True, '') }}:descriptor:x25519:{{ (lookup('file', role_path+'/../../tor_v3_keys.json')|from_json)[tor_v3_service_map[item.item.filename]] }}{% else %}{{ item.stdout }}{% endif %}

From 67e09249a740afbab65504aefdbbabf5ccbd591f Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Fri, 16 Aug 2019 12:49:31 -0400
Subject: [PATCH 36/46] v3 onions: gitignore new v3 *ths files

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index cbec3a02ce..c2b7cb791c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,10 @@ app-document-aths # leave this here for historic reasons
 app-journalist-aths
 app-source-ths
 mon-ssh-aths
+app-journalistv3-aths
+app-sourcev3-ths
+app-sshv3-aths
+mon-sshv3-aths
 *.key
 *.csr
 *.pem

From 728ef386961dcf2d41781e629dcad27942c99cd1 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Fri, 16 Aug 2019 10:39:15 -0400
Subject: [PATCH 37/46] ci: bump staging job timeout to 25 minutes

this seems to be timing out a bunch lately in CI, giving it
another 5 minutes
---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 90cafc9aa2..bf8c00b2aa 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -328,7 +328,7 @@ jobs:
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs)")
             if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             make ci-go
-          no_output_timeout: 20m
+          no_output_timeout: 25m
 
       - run:
           name: Ensure environment torn down

From fd9da6f50f88733082e59c7811e3e5128f37ad5e Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Fri, 16 Aug 2019 11:25:20 -0400
Subject: [PATCH 38/46] v3 onion: remove get_v3_keys, add remaining unit test
 coverage

---
 admin/securedrop_admin/__init__.py   | 12 -------
 admin/tests/test_securedrop-admin.py | 47 ++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 12 deletions(-)

diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
index ff2dcffbfe..7f068707f9 100755
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -604,18 +604,6 @@ def generate_new_v3_keys():
     return public, private
 
 
-def get_v3_keys(filepath):
-    """
-    Returns the stored v3 public and private keys as Tuple.
-
-    :returns: Tuple(public_key, private_key)
-    """
-    with open(filepath) as fobj:
-        v3_onion_service_keys = json.load(fobj)
-
-    return v3_onion_service_keys
-
-
 def find_or_generate_new_torv3_keys(args):
     """
     This method will either read v3 Tor onion service keys if found or generate
diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py
index e5126f19ab..2300701222 100644
--- a/admin/tests/test_securedrop-admin.py
+++ b/admin/tests/test_securedrop-admin.py
@@ -22,6 +22,7 @@
 import argparse
 from flaky import flaky
 from os.path import dirname, join, basename, exists
+import json
 import mock
 from prompt_toolkit.validation import ValidationError
 import pytest
@@ -1017,3 +1018,49 @@ def test_generate_new_v3_keys():
         # base32 padding characters should be removed
         assert '=' not in key
         assert len(key) == 52
+
+
+def test_find_or_generate_new_torv3_keys_first_run(tmpdir, capsys):
+    args = argparse.Namespace(ansible_path=str(tmpdir))
+
+    return_code = securedrop_admin.find_or_generate_new_torv3_keys(args)
+
+    captured = capsys.readouterr()
+    assert 'Tor v3 onion service keys generated' in captured.out
+    assert return_code == 0
+
+    secret_key_path = os.path.join(args.ansible_path,
+                                   "tor_v3_keys.json")
+
+    with open(secret_key_path) as f:
+        v3_onion_service_keys = json.load(f)
+
+    expected_keys = ['app_journalist_public_key',
+                     'app_journalist_private_key',
+                     'app_ssh_public_key',
+                     'app_ssh_private_key',
+                     'mon_ssh_public_key',
+                     'mon_ssh_private_key']
+    for key in expected_keys:
+        assert key in v3_onion_service_keys.keys()
+
+
+def test_find_or_generate_new_torv3_keys_subsequent_run(tmpdir, capsys):
+    args = argparse.Namespace(ansible_path=str(tmpdir))
+
+    secret_key_path = os.path.join(args.ansible_path,
+                                   "tor_v3_keys.json")
+    old_keys = {'foo': 'bar'}
+    with open(secret_key_path, 'w') as f:
+        json.dump(old_keys, f)
+
+    return_code = securedrop_admin.find_or_generate_new_torv3_keys(args)
+
+    captured = capsys.readouterr()
+    assert 'Tor v3 onion service keys already exist' in captured.out
+    assert return_code == 0
+
+    with open(secret_key_path) as f:
+        v3_onion_service_keys = json.load(f)
+
+    assert v3_onion_service_keys == old_keys

From 261bba6b5894245f53524c19929bc9210e36acdd Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Fri, 16 Aug 2019 12:41:16 -0400
Subject: [PATCH 39/46] v3 onions: update syscheck configuration for new
 service directories

---
 .../var/ossec/etc/ossec.conf                      | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf b/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf
index 9b14f4a818..35778f56be 100644
--- a/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf
+++ b/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf
@@ -14,16 +14,29 @@
     <directories realtime="yes" check_all="yes" report_changes="yes">/var/www</directories>
     <directories realtime="yes" check_all="yes" report_changes="yes">/var/lib/securedrop</directories>
     <directories realtime="yes" check_all="yes" report_changes="yes">/var/lib/tor/services/source/hostname</directories>
+    <directories realtime="yes" check_all="yes" report_changes="yes">/var/lib/tor/services/sourcev3/hostname</directories>
     <directories realtime="yes" check_all="yes" report_changes="yes">/var/lib/tor/services/journalist/hostname</directories>
+    <directories realtime="yes" check_all="yes" report_changes="yes">/var/lib/tor/services/journalistv3/hostname</directories>
     <directories realtime="yes" check_all="yes" report_changes="yes">/var/lib/tor/services/ssh/hostname</directories>
+    <directories realtime="yes" check_all="yes" report_changes="yes">/var/lib/tor/services/sshv3/hostname</directories>
     <directories realtime="yes" check_all="yes" report_changes="yes">/var/lib/tor/lock</directories>
     <directories realtime="yes" check_all="yes" report_changes="yes">/boot</directories>
 
     <!-- Files/directories to ignore -->
-    <ignore>/var/lib/tor/services/source/private_keys</ignore>
+
+    <!-- note for /var/lib/tor/services these rules are included as defense in depth since /var/lib/tor/services is not monitored -->
+    <ignore>/var/lib/tor/services/source/private_key</ignore>
     <ignore>/var/lib/tor/services/journalist/client_keys</ignore>
+    <ignore>/var/lib/tor/services/journalist/private_key</ignore>
+    <ignore>/var/lib/tor/services/ssh/private_key</ignore>
     <ignore>/var/lib/tor/services/ssh/client_keys</ignore>
 
+    <ignore>/var/lib/tor/services/journalistv3/authorized_clients</ignore>
+    <ignore>/var/lib/tor/services/journalistv3/hs_ed25519_secret_key</ignore>
+    <ignore>/var/lib/tor/services/sourcev3/hs_ed25519_secret_key</ignore>
+    <ignore>/var/lib/tor/services/sshv3/authorized_clients</ignore>
+    <ignore>/var/lib/tor/services/sshv3/hs_ed25519_secret_key</ignore>
+
     <ignore>/var/lib/securedrop/keys/random_seed</ignore>
     <ignore>/var/lib/securedrop/keys/pubring.gpg</ignore>
     <ignore>/var/lib/securedrop/keys/secring.gpg</ignore>

From 769a05f1cb4fc9ea37773c29526866c57b18bd83 Mon Sep 17 00:00:00 2001
From: redshiftzero <jen@freedom.press>
Date: Fri, 16 Aug 2019 13:33:09 -0400
Subject: [PATCH 40/46] v3 onions: move v2/v3 default vars to role-level
 defaults

h/t to @conorsch for pointing this out
---
 install_files/ansible-base/group_vars/all/securedrop       | 6 ------
 .../roles/restrict-direct-access/defaults/main.yml         | 7 ++++++-
 .../roles/tor-hidden-services/defaults/main.yml            | 6 ++++++
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/install_files/ansible-base/group_vars/all/securedrop b/install_files/ansible-base/group_vars/all/securedrop
index 7b3aff0fcf..0fbe7663a6 100644
--- a/install_files/ansible-base/group_vars/all/securedrop
+++ b/install_files/ansible-base/group_vars/all/securedrop
@@ -46,12 +46,6 @@ appserver_dependencies:
 # Enable Tor over SSH by default
 enable_ssh_over_tor: true
 
-# v2 Tor onion services are on / v3 Tor onion services are off by default for backwards
-# compatibility. Note that new install after 1.0 will have v3 enabled by sdconfig which
-# will override these variables.
-v2_onion_services: true
-v3_onion_services: false
-
 # If file is present on system at the end of ansible run
 # force a reboot. Needed because of the de-coupled nature of
 # the many roles of the current prod playbook
diff --git a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
index 44e2adb821..b107a347e3 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
+++ b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
@@ -11,7 +11,6 @@ ssh_listening_address: "{{ '127.0.0.1' if enable_ssh_over_tor else '0.0.0.0' }}"
 # the hostname files can be fetched back to the Admin Workstation.
 tor_hidden_services_parent_dir: /var/lib/tor/services
 
-
 # Platform specific command for inferring the local interface utilized
 # to route to a specific IP host
 admin_net_int:
@@ -30,3 +29,9 @@ tor_v3_service_map:
   app-journalistv3-aths: "app_journalist_private_key"
   app-sshv3-aths: "app_ssh_private_key"
   mon-sshv3-aths: "mon_ssh_private_key"
+
+# v2 Tor onion services are on / v3 Tor onion services are off by default for backwards
+# compatibility. Note that new install after 1.0 will have v3 enabled by sdconfig which
+# will override these variables.
+v2_onion_services: true
+v3_onion_services: false
diff --git a/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml b/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml
index 132b257e5a..47359a2292 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/defaults/main.yml
@@ -3,3 +3,9 @@ tor_hidden_services_parent_dir: /var/lib/tor/services
 tor_user: debian-tor
 enable_ssh_over_tor: true
 sd_root_dir: "{{ lookup('pipe','git rev-parse --show-toplevel') }}"
+
+# v2 Tor onion services are on / v3 Tor onion services are off by default for backwards
+# compatibility. Note that new install after 1.0 will have v3 enabled by sdconfig which
+# will override these variables.
+v2_onion_services: true
+v3_onion_services: false

From 6057fc8867a6668854dfbff6a6940296430e40bb Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Fri, 16 Aug 2019 10:38:08 -0700
Subject: [PATCH 41/46] Updates v3 client-auth templates with final filename

Let's name the v3 client auth info with a compatible filename for use
with the tor services on the Admin Workstation. Simplifies the
tailsconfig logic to move the files, rather than rename them according
to a map.
---
 .gitignore                                     |  8 +++++---
 .../securedrop_application_server.yml          |  4 ++--
 .../group_vars/securedrop_monitor_server.yml   |  2 +-
 .../restrict-direct-access/defaults/main.yml   | 18 +++++++++---------
 .../templates/ths_config_v3.j2                 |  2 +-
 5 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/.gitignore b/.gitignore
index c2b7cb791c..3d1317a243 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,15 +19,17 @@ securedrop/tests/functional/instance_information.json
 install_files/ansible-base/tor_v3_keys.json
 
 # ignore the ATHS/THS hostname file ansible places
+# Tor v2
 app-ssh-aths
 app-document-aths # leave this here for historic reasons
 app-journalist-aths
 app-source-ths
 mon-ssh-aths
-app-journalistv3-aths
+# Tor v3
+app-journalist.client_auth
 app-sourcev3-ths
-app-sshv3-aths
-mon-sshv3-aths
+app-ssh.client_auth
+mon-ssh.client_auth
 *.key
 *.csr
 *.pem
diff --git a/install_files/ansible-base/group_vars/securedrop_application_server.yml b/install_files/ansible-base/group_vars/securedrop_application_server.yml
index 97c10893b6..dff43b82df 100644
--- a/install_files/ansible-base/group_vars/securedrop_application_server.yml
+++ b/install_files/ansible-base/group_vars/securedrop_application_server.yml
@@ -21,11 +21,11 @@ tor_instances:
     filename: app-journalist-aths
 
 tor_instances_v3:
-  - "{{ {'service': 'sshv3', 'filename': 'app-sshv3-aths'} if enable_ssh_over_tor else [] }}"
+  - "{{ {'service': 'sshv3', 'filename': 'app-ssh.client_auth'} if enable_ssh_over_tor else [] }}"
   - service: sourcev3
     filename: app-sourcev3-ths
   - service: journalistv3
-    filename: app-journalistv3-aths
+    filename: app-journalist.auth_private
 
 tor_auth_instances_v3:
   - "{{ 'sshv3' if enable_ssh_over_tor  else [] }}"
diff --git a/install_files/ansible-base/group_vars/securedrop_monitor_server.yml b/install_files/ansible-base/group_vars/securedrop_monitor_server.yml
index ef32a0cb94..b646d02af2 100644
--- a/install_files/ansible-base/group_vars/securedrop_monitor_server.yml
+++ b/install_files/ansible-base/group_vars/securedrop_monitor_server.yml
@@ -14,7 +14,7 @@ local_deb_packages:
 # Configure the tor Onion Services. The Monitor server has only one,
 # for SSH, since no web interfaces.
 tor_instances: "{{ [{ 'service': 'ssh', 'filename': 'mon-ssh-aths'}] if enable_ssh_over_tor else [] }}"
-tor_instances_v3: "{{ [{ 'service': 'sshv3', 'filename': 'mon-sshv3-aths'}] if enable_ssh_over_tor else [] }}"
+tor_instances_v3: "{{ [{ 'service': 'sshv3', 'filename': 'mon-ssh.auth_private'}] if enable_ssh_over_tor else [] }}"
 
 tor_auth_instances_v3:
   - "{{ 'sshv3' if enable_ssh_over_tor else [] }}"
diff --git a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
index b107a347e3..e1ead2680a 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
+++ b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
@@ -21,17 +21,17 @@ admin_net_int:
     cmd: "/sbin/route -n get "
     rgx: "(?<=interface: )\\w+"
 
-# Lookup table for querying keypair info from the local JSON
-# file on the Admin Workstation, required for configuring client
-# auth on Tor v3 Onion URLs. See the tor_v3_keys.json file for
-# reference on structure.
-tor_v3_service_map:
-  app-journalistv3-aths: "app_journalist_private_key"
-  app-sshv3-aths: "app_ssh_private_key"
-  mon-sshv3-aths: "mon_ssh_private_key"
-
 # v2 Tor onion services are on / v3 Tor onion services are off by default for backwards
 # compatibility. Note that new install after 1.0 will have v3 enabled by sdconfig which
 # will override these variables.
 v2_onion_services: true
 v3_onion_services: false
+
+# Lookup table for querying keypair info from the local JSON
+# file on the Admin Workstation, required for configuring client
+# auth on Tor v3 Onion URLs. See the tor_v3_keys.json file for
+# reference on structure.
+tor_v3_service_map:
+  app-journalist.client_auth: "app_journalist_private_key"
+  app-ssh.client_auth: "app_ssh_private_key"
+  mon-ssh.client_auth: "mon_ssh_private_key"
diff --git a/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2 b/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
index b0f2a69d1d..6b9576bb53 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
+++ b/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
@@ -1 +1 @@
-{% if item.item.filename.endswith('-aths') %}{{ item.stdout|truncate(56, True, '') }}:descriptor:x25519:{{ (lookup('file', role_path+'/../../tor_v3_keys.json')|from_json)[tor_v3_service_map[item.item.filename]] }}{% else %}{{ item.stdout }}{% endif %}
+{% if item.item.filename.endswith('.client_auth') %}{{ item.stdout|truncate(56, True, '') }}:descriptor:x25519:{{ (lookup('file', role_path+'/../../tor_v3_keys.json')|from_json)[tor_v3_service_map[item.item.filename]] }}{% else %}{{ item.stdout }}{% endif %}

From fc00d79f3040b5e80cc9b95cdf901aa84cfb7ba6 Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Thu, 15 Aug 2019 16:49:21 -0700
Subject: [PATCH 42/46] Don't clobber v3 client auth info

If an instance has multiple Administrators, and one chooses to enable
v3, we should detect that scenario and fail early to avoid destroying
the previously configured client auth info.
---
 .../check_tor_service_config_for_admins.yml   | 37 +++++++++++++++++++
 .../roles/tor-hidden-services/tasks/main.yml  |  2 +
 2 files changed, 39 insertions(+)
 create mode 100644 install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml

diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml
new file mode 100644
index 0000000000..19d5ef2bdd
--- /dev/null
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml
@@ -0,0 +1,37 @@
+---
+# The migration from Tor v2 -> v3 services requires Admins to opt-in
+# via `securedrop-admin sdconfig`, then rerun the install action to enable
+# v3 services. If a *different* Admin Workstation is subsequently used,
+# without the corresponding sdconfig changes, we halt the play, otherwise
+# the v3 client auth config will be clobbered, locking out the first Admin.
+
+- name: Check whether v3 services exist on server
+  stat:
+    path: "{{ tor_hidden_services_parent_dir }}/{{ item.service }}"
+  register: _v3_services_existence_check_result
+  with_items: "{{ tor_instances_v3 }}"
+
+  # Returns a list of booleans; should be ["True"] if v3 is enabled in sdconfig,
+  # otherwise ["False"]
+- name: Store info about existing v3 service state
+  set_fact:
+    _v3_services_state_info: "{{ _v3_services_state_info|extract('exists')|unique }}"
+
+- name: Debug the v3 service state lookup
+  debug:
+    var: _v3_services_state_info
+
+  # Fail if v3 service configs exist, but v3_onion_services is not enabled.
+  # If v3 service configs do not exist, but v3_onion_services is enabled,
+  # we're configuring the v3 services for the first time, so don't fail.
+- name: Confirm service state matches declared config
+  assert:
+    that: >
+      (True in _v3_services_state_info and v3_onion_services) or
+      (True not in _v3_services_state_info and not v3_onion_services)
+    fail_msg: >
+      ERROR. The 'sdconfig' settings do not specify v3 Onion Services,
+      but v3 Onion Services were found on the server. If your SecureDrop
+      instance has multiple Administrators, contact the other Administrators
+      to request the Tor v3 Onion Service config information. You must copy
+      the '*ths' files to this workstation, then re-run the install action.
diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/main.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/main.yml
index 3e24b6a0ff..baa1012094 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/main.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/main.yml
@@ -1,4 +1,6 @@
 ---
 - include: install_tor.yml
 
+- include: check_tor_service_config_for_admins.yml
+
 - include: configure_tor_hidden_services.yml

From a834177fd46367b165f148827be7f48696978b1c Mon Sep 17 00:00:00 2001
From: Kevin O Gorman <kevin.ogorman@gmail.com>
Date: Fri, 16 Aug 2019 17:38:19 -0400
Subject: [PATCH 43/46] Updated v3 auth filenames to use .auth_private
 extension

---
 .gitignore                                                  | 6 +++---
 .../group_vars/securedrop_application_server.yml            | 2 +-
 .../roles/restrict-direct-access/defaults/main.yml          | 6 +++---
 .../roles/restrict-direct-access/templates/ths_config_v3.j2 | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.gitignore b/.gitignore
index 3d1317a243..1daefbeab8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,10 +26,10 @@ app-journalist-aths
 app-source-ths
 mon-ssh-aths
 # Tor v3
-app-journalist.client_auth
+app-journalist.auth_private
 app-sourcev3-ths
-app-ssh.client_auth
-mon-ssh.client_auth
+app-ssh.auth_private
+mon-ssh.auth_private
 *.key
 *.csr
 *.pem
diff --git a/install_files/ansible-base/group_vars/securedrop_application_server.yml b/install_files/ansible-base/group_vars/securedrop_application_server.yml
index dff43b82df..3874a95c1a 100644
--- a/install_files/ansible-base/group_vars/securedrop_application_server.yml
+++ b/install_files/ansible-base/group_vars/securedrop_application_server.yml
@@ -21,7 +21,7 @@ tor_instances:
     filename: app-journalist-aths
 
 tor_instances_v3:
-  - "{{ {'service': 'sshv3', 'filename': 'app-ssh.client_auth'} if enable_ssh_over_tor else [] }}"
+  - "{{ {'service': 'sshv3', 'filename': 'app-ssh.auth_private'} if enable_ssh_over_tor else [] }}"
   - service: sourcev3
     filename: app-sourcev3-ths
   - service: journalistv3
diff --git a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
index e1ead2680a..49ef6e62f8 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
+++ b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
@@ -32,6 +32,6 @@ v3_onion_services: false
 # auth on Tor v3 Onion URLs. See the tor_v3_keys.json file for
 # reference on structure.
 tor_v3_service_map:
-  app-journalist.client_auth: "app_journalist_private_key"
-  app-ssh.client_auth: "app_ssh_private_key"
-  mon-ssh.client_auth: "mon_ssh_private_key"
+  app-journalist.auth_private: "app_journalist_private_key"
+  app-ssh.auth_private: "app_ssh_private_key"
+  mon-ssh.auth_private: "mon_ssh_private_key"
diff --git a/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2 b/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
index 6b9576bb53..9a535d462c 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
+++ b/install_files/ansible-base/roles/restrict-direct-access/templates/ths_config_v3.j2
@@ -1 +1 @@
-{% if item.item.filename.endswith('.client_auth') %}{{ item.stdout|truncate(56, True, '') }}:descriptor:x25519:{{ (lookup('file', role_path+'/../../tor_v3_keys.json')|from_json)[tor_v3_service_map[item.item.filename]] }}{% else %}{{ item.stdout }}{% endif %}
+{% if item.item.filename.endswith('.auth_private') %}{{ item.stdout|truncate(56, True, '') }}:descriptor:x25519:{{ (lookup('file', role_path+'/../../tor_v3_keys.json')|from_json)[tor_v3_service_map[item.item.filename]] }}{% else %}{{ item.stdout }}{% endif %}

From e0412d74b65b3345b3ea6149c84f4502167f669a Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Fri, 16 Aug 2019 15:03:53 -0700
Subject: [PATCH 44/46] Updates v3 fail-fast logic for admins

We're aiming to fail early if an Admin runs the install action
against a server with v3 configured, but without having run
sdconfig to enable v3 (indicates multiple Admins).

Updated the monstrous filter line to pull out the booleans, then inspect
that list in an assert statement. Seems to be working as expected, will
test with multiple configuration paths prior to squashing.

Better support for "staging" scenario, which has v3_onion_services=true,
but enable_ssh_over_tor=false, so mon-staging was failing the check.
Updated to accommodate.

If no onions exist, don't fail: we're about to set them up!
---
 .../check_tor_service_config_for_admins.yml   | 25 +++++++++++--------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml
index 19d5ef2bdd..bb574567b5 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml
@@ -11,15 +11,11 @@
   register: _v3_services_existence_check_result
   with_items: "{{ tor_instances_v3 }}"
 
-  # Returns a list of booleans; should be ["True"] if v3 is enabled in sdconfig,
-  # otherwise ["False"]
+  # Returns a list of booleans, one boolean for each v3 service, denoting whether
+  # a config currently exists for that service.
 - name: Store info about existing v3 service state
   set_fact:
-    _v3_services_state_info: "{{ _v3_services_state_info|extract('exists')|unique }}"
-
-- name: Debug the v3 service state lookup
-  debug:
-    var: _v3_services_state_info
+    _v3_services_state_info: "{{ _v3_services_existence_check_result.results|map(attribute='stat')|map(attribute='exists')|list }}"
 
   # Fail if v3 service configs exist, but v3_onion_services is not enabled.
   # If v3 service configs do not exist, but v3_onion_services is enabled,
@@ -27,11 +23,18 @@
 - name: Confirm service state matches declared config
   assert:
     that: >
-      (True in _v3_services_state_info and v3_onion_services) or
-      (True not in _v3_services_state_info and not v3_onion_services)
-    fail_msg: >
+      (true in _v3_services_state_info and v3_onion_services) or
+      (true not in _v3_services_state_info and not v3_onion_services)
+    msg: >
       ERROR. The 'sdconfig' settings do not specify v3 Onion Services,
       but v3 Onion Services were found on the server. If your SecureDrop
       instance has multiple Administrators, contact the other Administrators
       to request the Tor v3 Onion Service config information. You must copy
-      the '*ths' files to this workstation, then re-run the install action.
+      the 'tor_v3_keys.json' and '*.auth_private' files to this workstation,
+      then re-run the install action.
+  when:
+    # In staging, Monitor Server will have 0 (SSH-over-Tor disabled)
+    - tor_instances_v3|length > 0
+    # Only run if we're connected over Tor (i.e. enabling v3 after v2).
+    # If we're not connected over Tor, this is a first-run.
+    - (ansible_host|default(ansible_ssh_host)).endswith('.onion')

From 5637a818bf75855df286e2eabb800387759c7bdb Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Tue, 20 Aug 2019 17:41:56 -0700
Subject: [PATCH 45/46] Updates fail-fast logic for v2 -> v3 transition

Checks that BOTH of the following conditions are NOT true:

  * v3_onion_services=false
  * v3 services exist on remote host

If both are true, that's likely the result of a misconfiguration.
This logic handles the prod use case of v2 enabled, also enabling
v3. It hasn't yet been tested with ssh-over-LAN, we'll likely need to
modify the restrict-direct-access role to accommodate.
---
 .../tasks/check_tor_service_config_for_admins.yml              | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml b/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml
index bb574567b5..5f62d454f5 100644
--- a/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml
+++ b/install_files/ansible-base/roles/tor-hidden-services/tasks/check_tor_service_config_for_admins.yml
@@ -23,8 +23,7 @@
 - name: Confirm service state matches declared config
   assert:
     that: >
-      (true in _v3_services_state_info and v3_onion_services) or
-      (true not in _v3_services_state_info and not v3_onion_services)
+      not (not v3_onion_services and true in _v3_services_state_info)
     msg: >
       ERROR. The 'sdconfig' settings do not specify v3 Onion Services,
       but v3 Onion Services were found on the server. If your SecureDrop

From f24ffc0ac95ee108aae5c9a5bacb302867e37f44 Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@freedom.press>
Date: Wed, 21 Aug 2019 13:18:10 -0700
Subject: [PATCH 46/46] Supports SSH-over-LAN in v2->v3 Onion migration

The dynamic logic to accommodate for SSH-over-LAN includes the
"restrict-direct-access" role, which assumes that Onion Services already
exist on the remote host. That's not the case when migrating from v2 to
v3 Onion services, so let's override the dynamic include logic to
instruct the role *not* to wait for the hostname files to be created.

Later in the playbook, the restrict-direct-access role will run without
the override, fetching back the client auth config that now exists,
since the tor role will also have run by this point.
---
 .../roles/restrict-direct-access/defaults/main.yml           | 5 +++++
 .../ansible-base/roles/restrict-direct-access/tasks/main.yml | 1 +
 install_files/ansible-base/securedrop-prod.yml               | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
index 49ef6e62f8..73d3c8917b 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
+++ b/install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml
@@ -21,6 +21,11 @@ admin_net_int:
     cmd: "/sbin/route -n get "
     rgx: "(?<=interface: )\\w+"
 
+# Whether to fetch back client-auth settings from the remote hosts.
+# We make this conditional to support disabling during dynamic role includes,
+# required for the ssh-over-lan strategy.
+fetch_tor_client_auth_configs: true
+
 # v2 Tor onion services are on / v3 Tor onion services are off by default for backwards
 # compatibility. Note that new install after 1.0 will have v3 enabled by sdconfig which
 # will override these variables.
diff --git a/install_files/ansible-base/roles/restrict-direct-access/tasks/main.yml b/install_files/ansible-base/roles/restrict-direct-access/tasks/main.yml
index 1127edf4b3..2be2d68ca2 100644
--- a/install_files/ansible-base/roles/restrict-direct-access/tasks/main.yml
+++ b/install_files/ansible-base/roles/restrict-direct-access/tasks/main.yml
@@ -1,5 +1,6 @@
 ---
 - include: fetch_tor_config.yml
+  when: fetch_tor_client_auth_configs
 
 - include: dh_moduli.yml
 
diff --git a/install_files/ansible-base/securedrop-prod.yml b/install_files/ansible-base/securedrop-prod.yml
index 6526173f94..87b8421912 100755
--- a/install_files/ansible-base/securedrop-prod.yml
+++ b/install_files/ansible-base/securedrop-prod.yml
@@ -37,6 +37,9 @@
     - name: Include restrict role early when using ssh over localnet
       include_role:
         name: restrict-direct-access
+      vars:
+        # Don't wait for tor client auth, might not exist yet
+        fetch_tor_client_auth_configs: false
       when:
         - not enable_ssh_over_tor
         - sd_dir_check.stat.exists