Generate reference docs from proto files

build/Makefile

@@ -131,7 +131,7 @@ build: binary-archive build-image
 # Build all binaries
 build-all-binaries: ensure-build-image build-linux-binary build-macos-binary build-windows-binary
 
-# Checks for changes to protobuf definitions and generates rust code if there is any.
+# Checks for changes to protobuf definitions and generates rust code and reference documentation if there is any.
 gen-protobuf: ensure-build-image
 	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) \
 		--entrypoint=cargo $(BUILD_IMAGE_TAG) run -p proto-gen -- generate

build/README.md

@@ -39,14 +39,16 @@ To build a production release, run:
 
 ##### Generating Protobuf Files
 
-If you are adding or editing the `.proto` files in this repository, they will need to be regenerated. To do this,
-you will need [protoc](https://grpc.io/docs/protoc-installation/) installed locally.
+If you are adding or editing the `.proto` files in this repository, they will need to be regenerated along with the 
+reference documentation. To do this,
+you will need [protoc](https://grpc.io/docs/protoc-installation/) and 
+[protoc-gen-doc](https://github.com/pseudomuto/protoc-gen-doc) installed locally.
 
 Then run:
 
 `cargo run -p proto-gen -- generate`
 
-Which will identify changes in protobuf definitions and regenerate code appropriately. 
+Which will identify changes in protobuf definitions and regenerate code and documentation appropriately. 
 
 #### Testing
 

build/build-image/Dockerfile

@@ -30,6 +30,13 @@ RUN set -eux && \
     npm install -g browser-sync && \
     echo "source /etc/bash_completion" >> /root/.bashrc
 
+# install protoc-gen-doc
+RUN cd /tmp && \
+    wget --quiet -O protoc-gen-doc.tar.gz https://github.com/pseudomuto/protoc-gen-doc/releases/download/v1.5.1/protoc-gen-doc_1.5.1_linux_amd64.tar.gz && \
+    tar -xf protoc-gen-doc.tar.gz && \
+    mv ./protoc-gen-doc /usr/local/bin/ && \
+    rm protoc-gen-doc.tar.gz
+
 # install gcloud
 # Credit: https://cloud.google.com/sdk/docs/install#deb
 RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && apt-get update -y && \
@@ -75,6 +82,7 @@ RUN set -eux; \
     cargo install --locked cargo-about && \
     cargo install --locked mdbook-variables && \
     cargo install --locked cargo-deny && \
+    cargo install --locked proto-gen && \
     rustup --version && \
     cargo --version && \
     rustc --version

crates/proto-gen/Cargo.toml

@@ -1,3 +1,19 @@
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 [package]
 name = "proto-gen"
 version = "0.1.0"

crates/proto-gen/gen.rs

@@ -91,15 +91,13 @@ fn install() {
         if !child.wait().expect("tar is not installed").success() {
             panic!("failed to extract proto-gen binary from tarball");
         }
-    } else {
-        if !Command::new("cargo")
-            .args(["install", "--locked", "-f", "proto-gen"])
-            .status()
-            .expect("cargo not installed")
-            .success()
-        {
-            panic!("failed to install proto-gen via cargo");
-        }
+    } else if !Command::new("cargo")
+        .args(["install", "--locked", "-f", "proto-gen"])
+        .status()
+        .expect("cargo not installed")
+        .success()
+    {
+        panic!("failed to install proto-gen via cargo");
     }
 }
 
@@ -226,6 +224,50 @@ fn execute(which: &str) {
     if !cmd.status().expect("proto-gen was not installed").success() {
         panic!("proto-gen {which} failed");
     }
+
+    if which == "generate" {
+        docs(files);
+    }
+}
+
+fn docs(files: &[(&str, &[&str])]) {
+    let mut cmd = Command::new("protoc");
+
+    let quilkin_protos: Vec<&(&str, &[&str])> = files
+        .iter()
+        .filter(|item| item.0 == "proto/quilkin")
+        .collect();
+    let includes: Vec<&(&str, &[&str])> = files
+        .iter()
+        .filter(|item| item.0 != "proto/quilkin")
+        .collect();
+
+    for (dir, files) in includes {
+        if files.is_empty() {
+            cmd.args(["-I", dir]);
+        } else {
+            for file in *files {
+                cmd.args(["-I".into(), format!("{dir}/{file}.proto")]);
+            }
+        }
+    }
+
+    cmd.args(["--doc_out", "./docs/src/services/xds/proto"]);
+    cmd.args(["--doc_opt", "markdown,index.md"]);
+
+    for (dir, files) in quilkin_protos {
+        for file in *files {
+            cmd.arg(format!("{dir}/{file}.proto"));
+        }
+    }
+
+    if !cmd
+        .status()
+        .expect("protoc-gen-doc was not installed")
+        .success()
+    {
+        panic!("protoc-gen-doc failed");
+    }
 }
 
 fn copy() {
@@ -294,7 +336,7 @@ fn copy() {
     ];
 
     let args: Vec<_> = std::env::args().skip(2).collect();
-    let name = args.get(0).expect("must provide source name");
+    let name = args.first().expect("must provide source name");
     let path = args.get(1).expect("must provide path");
 
     let Some(ri) = REPOS.iter().find(|r| r.name == name) else {

crates/quilkin-proto/Cargo.toml

@@ -1,3 +1,19 @@
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 [package]
 name = "quilkin-proto"
 version = "0.1.0"

crates/quilkin-proto/src/generated/quilkin/relay/v1alpha1.rs

@@ -95,10 +95,8 @@ pub mod aggregated_control_plane_discovery_service_client {
             self.inner = self.inner.max_encoding_message_size(limit);
             self
         }
-        /// The RPC protocol begins with a single empty DiscoveryResponse
-        /// initiated by the management server, after that this behaves
-        /// the same as the management server xDS protocol, except with
-        /// DiscoveryRequests initiated by the server rather than the client.
+        /// This RPC is not supported but remains here as part of the xDS standard,
+        /// as Quilkin only uses a delta configuration transport for performance reasons.
         pub async fn stream_aggregated_resources(
             &mut self,
             request: impl tonic::IntoStreamingRequest<
@@ -129,6 +127,7 @@ pub mod aggregated_control_plane_discovery_service_client {
             ));
             self.inner.streaming(req, path, codec).await
         }
+        /// Delta (Incremental) xDS implementation
         pub async fn delta_aggregated_resources(
             &mut self,
             request: impl tonic::IntoStreamingRequest<
@@ -176,10 +175,8 @@ pub mod aggregated_control_plane_discovery_service_server {
                 >,
             > + Send
             + 'static;
-        /// The RPC protocol begins with a single empty DiscoveryResponse
-        /// initiated by the management server, after that this behaves
-        /// the same as the management server xDS protocol, except with
-        /// DiscoveryRequests initiated by the server rather than the client.
+        /// This RPC is not supported but remains here as part of the xDS standard,
+        /// as Quilkin only uses a delta configuration transport for performance reasons.
         async fn stream_aggregated_resources(
             &self,
             request: tonic::Request<
@@ -200,6 +197,7 @@ pub mod aggregated_control_plane_discovery_service_server {
             >
             + Send
             + 'static;
+        /// Delta (Incremental) xDS implementation
         async fn delta_aggregated_resources(
             &self,
             request: tonic::Request<

crates/test/Cargo.toml

@@ -1,3 +1,19 @@
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 [package]
 name = "qt"
 version = "0.1.0"

docs/htmltest.yaml

@@ -26,6 +26,7 @@ IgnoreInternalURLs:
   - "../../print.html"
 IgnoreDirs:
   - "api"
+  - "book/services/xds/proto/"
 HTTPHeaders:
   User-Agent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
   Accept: "text/html"

docs/src/SUMMARY.md

@@ -37,7 +37,7 @@
     - [Providers]()
         - [Agones](./services/xds/providers/agones.md)
         - [Filesystem](./services/xds/providers/filesystem.md)
-
+    - [Protobuf Reference](./services/xds/proto/index.md)
 ---
 
 - [Relay](./services/relay.md)

docs/src/services/xds.md

@@ -19,34 +19,29 @@ $ quilkin manage --help
 
 ## Overview
 
-In addition to static configuration provided upon startup, a Quiklin proxy's configuration can also be updated at runtime. The proxy can be configured on startup to talk to a set of management servers which provide it with updates throughout its lifecycle.
+In addition to static configuration provided upon startup, a Quiklin proxy's configuration can also be updated at
+runtime. The proxy can be configured on startup to talk to a set of management servers which provide it with updates
+throughout its lifecycle.
 
-Communication between the proxy and management server uses the [xDS gRPC protocol][xDS], similar to an [envoy proxy]. xDS is one of the standard configuration mechanisms for software proxies and as a result, Quilkin can be setup to discover configuration resources from any API compatible server. Also, given that the protocol is [well specified][xDS-protocol], it is similarly straight-forward to implement a custom server to suit any deployment's needs.
+Communication between the proxy and management server uses the [xDS gRPC protocol][xDS], similar to an [envoy proxy].
+xDS is one of the standard configuration mechanisms for software proxies and as a result, Quilkin can be setup to
+discover configuration resources from any API compatible server. Also, given that the protocol
+is [well specified][xDS-protocol], it is similarly straight-forward to implement a custom server to suit any
+deployment's needs.
 
-As described within the [xDS-api] documentation, the xDS API comprises a set of resource discovery APIs, each serving a specific set of configuration resource types, while the protocol itself comes in several [variants][xds-variants].
-Quilkin implements the **Aggregated Discovery Service (ADS)** _State of the World (SotW)_ variant with gRPC.
+As described within the [xDS-api] documentation, the xDS API comprises a set of resource discovery APIs, each serving a
+specific set of configuration resource types, while the protocol itself comes in several [variants][xds-variants].
+Quilkin implements the **Aggregated Discovery Service (ADS)** _[Incremental xDS][incremental-xds]_ variant with gRPC.
 
-## Supported APIs
+## xDS Configuration Resources
 
-Since the range of resources configurable by the xDS API extends that of Quilkin's domain (i.e being UDP based, Quilkin does not have a need for HTTP/TCP resources), only a subset of the API is supported. The following lists these relevant parts and any limitation to the provided support as a result:
+Heavily influenced by xDS's [Cluster Discovery Service][CDS], [Endpoint Discovery Service][EDS], and 
+[Listener Discovery Service][LDS], Quilkin utilises its own custom Configuration xDS resources 
+[(reference)][config.proto] to enable dynamic configuration of Datacenters, Clusters, Endpoints and Filters through 
+the Aggregated Discovery Service.
 
-- **Cluster Discovery Service [(CDS)][CDS]**: Provides information about known clusters and their membership information.
-  * The proxy uses these resources to discover clusters and their endpoints.
-  * While cluster topology information like [locality] can be provided in the configuration, the proxy currently does not use this information (support may be included in the future however).
-  * Any [load balancing information][lbpolicy] included in this resource is ignored. For load balancing, use [Quilkin filters][filters-doc] instead.
-  * Only [cluster discovery type] `STATIC` and `EDS` is supported. Configuration including other discovery types e.g `LOGICAL_DNS` is rejected.
-
-- **Endpoint Discovery Service [(EDS)][EDS]**: Provides information about endpoints.
-  * The proxy uses these resources to discover information about endpoints like their IP addresses.
-  * Endpoints may provide [Endpoint Metadata][endpoint-metadata] via the [metadata][xds-endpoint-metadata] field. These metadata will be visible to filters as part of the corresponding endpoints information when processing packets.
-  * Only [socket addresses] are supported on an endpoint's address configuration - i.e an IP address and port number combination. Configuration including any other type of addressing e.g named pipes will be rejected.
-  * Any [load balancing information][clapolicy] included in this resource is ignored. For load balancing, use [Quilkin filters][filters-doc] instead.
-
-- **Listener Discovery Service [(LDS)][LDS]**: Provides information about [Filters and Filter Chains][filters-doc].
-  * Only the `name` and `filter_chains` fields in the [Listener resource][listener-resource] are used by the proxy. The rest are ignored.
-  * Since Quilkin only uses one filter chain per proxy, at most one filter chain can be provided in the resource. Otherwise the configuration is rejected.
-  * Only the list of [filters][xds-filters] specified in the [filter chain][xds-filter-chain] is used by the proxy - i.e other fields like `filter_chain_match` are ignored. This list also specifies the order that the corresponding filter chain will be constructed.
-  * gRPC proto configuration for Quilkin's built-in filters [can be found here][filter-protos]. They are equivalent to the filter's static configuration.
+This [above reference][config.proto] also provides documentation for all the ProtoBuf definitions for all 
+[Filters][filters-doc] that are available in Quilkin.
 
 ## Connecting to an xDS management server
 
@@ -55,25 +50,17 @@ the `management_servers` [command line](../../api/quilkin/struct.Proxy.html#stru
 
 
 [xDS]: https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#xds-rest-and-grpc-protocol
+[incremental-xds]: https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#incremental-xds
 [envoy proxy]: https://www.envoyproxy.io/docs/envoy/latest/
 [xDS-protocol]: https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#the-xds-transport-protocol
 [xDS-api]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration
+[config.proto]: ./xds/proto/index.md#quilkin_config_v1alpha1_config-proto
 [CDS]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration#cds
 [EDS]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration#eds
 [LDS]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration#lds
-[cluster discovery type]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-discoverytype
-[lbpolicy]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-lbpolicy
-[clapolicy]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/endpoint/v3/endpoint.proto#config-endpoint-v3-clusterloadassignment-policy
 [locality]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/base.proto#config-core-v3-locality
-[socket addresses]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#config-core-v3-address
 [filters-doc]: ./proxy/filters.md
-[listener-resource]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/listener/v3/listener.proto#config-listener-v3-listener
-[xds-filters]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/listener/v3/listener_components.proto#envoy-v3-api-msg-config-listener-v3-filter
-[xds-filter-chain]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/listener/v3/listener_components.proto#config-listener-v3-filterchain
 [xds-variants]: https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#variants-of-the-xds-transport-protocol
 [filter-protos]: https://github.com/googleforgames/quilkin/tree/{{GITHUB_REF_NAME}}/proto/quilkin/filters
 [xds-endpoint-metadata]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/base.proto#envoy-v3-api-msg-config-core-v3-metadata
-[endpoint-metadata]: proxy.md#endpoint-metadata
-[control-plane]: https://github.com/googleforgames/quilkin/tree/{{GITHUB_REF_NAME}}/xds
-[Kubernetes]: https://kubernetes.io/