From c9d2b029ffc72e2afaef20c605a6768e086348e8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Sep 2022 15:30:44 +0100 Subject: [PATCH 1/7] Check if Rust lib needs rebuilding. --- rust/Cargo.toml | 4 +++ rust/build.rs | 44 ++++++++++++++++++++++++++ rust/src/lib.rs | 10 ++++++ stubs/synapse/synapse_rust.pyi | 1 + synapse/__init__.py | 5 +++ synapse/util/rust.py | 58 ++++++++++++++++++++++++++++++++++ 6 files changed, 122 insertions(+) create mode 100644 rust/build.rs create mode 100644 synapse/util/rust.py diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 0a9760cafcd1..deddf3cec262 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -19,3 +19,7 @@ name = "synapse.synapse_rust" [dependencies] pyo3 = { version = "0.16.5", features = ["extension-module", "macros", "abi3", "abi3-py37"] } + +[build-dependencies] +blake2 = "0.10.4" +hex = "0.4.3" diff --git a/rust/build.rs b/rust/build.rs new file mode 100644 index 000000000000..e20a2e911d53 --- /dev/null +++ b/rust/build.rs @@ -0,0 +1,44 @@ +use std::path::PathBuf; + +use blake2::{Blake2b512, Digest}; + +fn main() -> Result<(), std::io::Error> { + let mut dirs = vec![PathBuf::from("src")]; + + let mut paths = Vec::new(); + + while let Some(path) = dirs.pop() { + let mut entries = std::fs::read_dir(path)? + .map(|res| res.map(|e| e.path())) + .collect::, std::io::Error>>()?; + + entries.sort(); + + let mut new_dirs = Vec::new(); + + for entry in entries { + if entry.is_dir() { + new_dirs.push(entry); + } else { + paths.push(entry.to_str().expect("valid rust paths").to_string()); + } + } + + dirs.append(&mut new_dirs); + } + + let mut hasher = Blake2b512::new(); + + paths.sort(); + + for path in paths { + let bytes = std::fs::read(path)?; + hasher.update(bytes); + } + + let hex_digest = hex::encode(hasher.finalize()); + println!("cargo:warning={hex_digest}"); + println!("cargo:rustc-env=SYNAPSE_RUST_DIGEST={hex_digest}"); + + Ok(()) +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index fc4eb39154fc..a0e6569f054e 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,5 +1,13 @@ use pyo3::prelude::*; +/// Returns the hash of all the rust source files at the time it was compiled. +/// +/// Used by python to detect if the rust library is outdated. +#[pyfunction] +fn get_rust_file_digest() -> &'static str { + env!("SYNAPSE_RUST_DIGEST") +} + /// Formats the sum of two numbers as string. #[pyfunction] #[pyo3(text_signature = "(a, b, /)")] @@ -11,5 +19,7 @@ fn sum_as_string(a: usize, b: usize) -> PyResult { #[pymodule] fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; + + m.add_function(wrap_pyfunction!(get_rust_file_digest, m)?)?; Ok(()) } diff --git a/stubs/synapse/synapse_rust.pyi b/stubs/synapse/synapse_rust.pyi index 5b51ba05d780..8658d3138f89 100644 --- a/stubs/synapse/synapse_rust.pyi +++ b/stubs/synapse/synapse_rust.pyi @@ -1 +1,2 @@ def sum_as_string(a: int, b: int) -> str: ... +def get_rust_file_digest() -> str: ... diff --git a/synapse/__init__.py b/synapse/__init__.py index b1369aca8f72..1bed6393bdb6 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -20,6 +20,8 @@ import os import sys +from synapse.util.rust import check_rust_lib_up_to_date + # Check that we're not running on an unsupported Python version. if sys.version_info < (3, 7): print("Synapse requires Python 3.7 or above.") @@ -78,3 +80,6 @@ from synapse.util.patch_inline_callbacks import do_patch do_patch() + + +check_rust_lib_up_to_date() diff --git a/synapse/util/rust.py b/synapse/util/rust.py new file mode 100644 index 000000000000..c112f4666e8b --- /dev/null +++ b/synapse/util/rust.py @@ -0,0 +1,58 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +from glob import glob +from hashlib import blake2b + +import synapse +from synapse.synapse_rust import get_rust_file_digest + + +def check_rust_lib_up_to_date() -> None: + """For editable installs check if the rust library is outdated and needs to + be rebuilt. + """ + + if not _dist_is_editable(): + return + + synapse_dir = os.path.dirname(synapse.__file__) + hash = _hash_rust_files_in_directory(f"{synapse_dir}/../rust/src") + + if hash != get_rust_file_digest(): + raise Exception("Rust module outdated. Please rebuild using `poetry install`") + + +def _hash_rust_files_in_directory(directory: str) -> str: + paths = glob("**/*.rs", recursive=True, root_dir=directory) + paths.sort() + + hasher = blake2b() + + for path in paths: + with open(os.path.join(directory, path), "rb") as f: + hasher.update(f.read()) + + return hasher.hexdigest() + + +def _dist_is_editable() -> bool: + """Is distribution an editable install?""" + for path_item in sys.path: + egg_link = os.path.join(path_item, "matrix-synapse.egg-link") + if os.path.isfile(egg_link): + return True + return False From a40d951e8507848ff4f8578ea8d277706ebd24ac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Sep 2022 18:17:04 +0100 Subject: [PATCH 2/7] Newsfile --- changelog.d/13759.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/13759.misc diff --git a/changelog.d/13759.misc b/changelog.d/13759.misc new file mode 100644 index 000000000000..f91c51248309 --- /dev/null +++ b/changelog.d/13759.misc @@ -0,0 +1 @@ +Add a check for editable installs if the Rust library needs rebuilding. From 678f303fb1ab0df6acc2735d7bb0e5a7f39192d7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 9 Sep 2022 09:28:40 +0100 Subject: [PATCH 3/7] Comment --- rust/build.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/rust/build.rs b/rust/build.rs index e20a2e911d53..7597766696fa 100644 --- a/rust/build.rs +++ b/rust/build.rs @@ -1,3 +1,10 @@ +//! This build script is calculates the hash of all files in the `src/` +//! directory and adds it as an environment variable during build time. +//! +//! This is used so that the python code can detect when the built native module +//! does not match the source in-tree, helping to detect the case where the +//! source has been updated but the library hasn't been rebuilt. + use std::path::PathBuf; use blake2::{Blake2b512, Digest}; @@ -6,7 +13,6 @@ fn main() -> Result<(), std::io::Error> { let mut dirs = vec![PathBuf::from("src")]; let mut paths = Vec::new(); - while let Some(path) = dirs.pop() { let mut entries = std::fs::read_dir(path)? .map(|res| res.map(|e| e.path())) @@ -27,17 +33,16 @@ fn main() -> Result<(), std::io::Error> { dirs.append(&mut new_dirs); } - let mut hasher = Blake2b512::new(); - paths.sort(); + let mut hasher = Blake2b512::new(); + for path in paths { let bytes = std::fs::read(path)?; hasher.update(bytes); } let hex_digest = hex::encode(hasher.finalize()); - println!("cargo:warning={hex_digest}"); println!("cargo:rustc-env=SYNAPSE_RUST_DIGEST={hex_digest}"); Ok(()) From eafdfa77109ac2e0fa85920ae893dfa38853f9e6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 9 Sep 2022 09:42:35 +0100 Subject: [PATCH 4/7] Fix up --- rust/src/lib.rs | 1 - synapse/util/rust.py | 31 +++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/rust/src/lib.rs b/rust/src/lib.rs index a0e6569f054e..ba42465fb80b 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -19,7 +19,6 @@ fn sum_as_string(a: usize, b: usize) -> PyResult { #[pymodule] fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; - m.add_function(wrap_pyfunction!(get_rust_file_digest, m)?)?; Ok(()) } diff --git a/synapse/util/rust.py b/synapse/util/rust.py index c112f4666e8b..f356913f2c82 100644 --- a/synapse/util/rust.py +++ b/synapse/util/rust.py @@ -30,14 +30,41 @@ def check_rust_lib_up_to_date() -> None: return synapse_dir = os.path.dirname(synapse.__file__) - hash = _hash_rust_files_in_directory(f"{synapse_dir}/../rust/src") + synapse_root = os.path.abspath(os.path.join(synapse_dir, "..")) + + # Double check we've not gone into site-packages... + if os.path.basename(synapse_root) == "site-packages": + return + + # ... and it looks like the root of a python project. + if not os.path.exists("pyproject.toml"): + return + + # Get the hash of all Rust source files + hash = _hash_rust_files_in_directory(os.path.join(synapse_root, "rust", "src")) if hash != get_rust_file_digest(): raise Exception("Rust module outdated. Please rebuild using `poetry install`") def _hash_rust_files_in_directory(directory: str) -> str: - paths = glob("**/*.rs", recursive=True, root_dir=directory) + """Get the hash of all files in a directory (recursively)""" + + directory = os.path.abspath(directory) + + paths = [] + + dirs = [directory] + while dirs: + dir = dirs.pop() + with os.scandir(dir) as d: + for f in d: + if f.is_dir(): + dirs.append(f.path) + else: + paths.append(f.path) + + # We sort to make sure that we get a consistent and well-defined ordering. paths.sort() hasher = blake2b() From f069ed9838c814f6fa93f794379ecffff9b5530e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 9 Sep 2022 09:49:33 +0100 Subject: [PATCH 5/7] Lint --- synapse/util/rust.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/synapse/util/rust.py b/synapse/util/rust.py index f356913f2c82..30ecb9ffd96d 100644 --- a/synapse/util/rust.py +++ b/synapse/util/rust.py @@ -14,7 +14,6 @@ import os import sys -from glob import glob from hashlib import blake2b import synapse @@ -58,11 +57,11 @@ def _hash_rust_files_in_directory(directory: str) -> str: while dirs: dir = dirs.pop() with os.scandir(dir) as d: - for f in d: - if f.is_dir(): - dirs.append(f.path) + for entry in d: + if entry.is_dir(): + dirs.append(entry.path) else: - paths.append(f.path) + paths.append(entry.path) # We sort to make sure that we get a consistent and well-defined ordering. paths.sort() From 2ea965b3ff1a926ea35a834539c0cb9d5053f2ca Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2022 09:56:04 +0100 Subject: [PATCH 6/7] Update rust/build.rs Co-authored-by: Patrick Cloke --- rust/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/build.rs b/rust/build.rs index 7597766696fa..2aa0fd263fad 100644 --- a/rust/build.rs +++ b/rust/build.rs @@ -1,4 +1,4 @@ -//! This build script is calculates the hash of all files in the `src/` +//! This build script calculates the hash of all files in the `src/` //! directory and adds it as an environment variable during build time. //! //! This is used so that the python code can detect when the built native module From b79efc0eb47261d732123bcae53b214752835b9a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2022 09:58:21 +0100 Subject: [PATCH 7/7] Inline adding new directories --- rust/build.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/rust/build.rs b/rust/build.rs index 2aa0fd263fad..2117975e56f7 100644 --- a/rust/build.rs +++ b/rust/build.rs @@ -20,17 +20,13 @@ fn main() -> Result<(), std::io::Error> { entries.sort(); - let mut new_dirs = Vec::new(); - for entry in entries { if entry.is_dir() { - new_dirs.push(entry); + dirs.push(entry) } else { paths.push(entry.to_str().expect("valid rust paths").to_string()); } } - - dirs.append(&mut new_dirs); } paths.sort();