Fix build

.cargo/config.toml

@@ -0,0 +1,2 @@
+[alias]
+xtask = "run -q --package xtask --"

.ci/DockerFile

@@ -27,5 +27,6 @@ COPY elasticsearch/Cargo.toml ./elasticsearch/Cargo.toml
 COPY elasticsearch/src ./elasticsearch/src
 COPY elasticsearch/build.rs ./elasticsearch/build.rs
 COPY yaml_test_runner ./yaml_test_runner
+COPY xtask ./xtask
 
 RUN cargo build --tests

.ci/functions/imports.sh

@@ -26,10 +26,11 @@ if [[ -z $es_node_name ]]; then
   export es_node_name=instance
   export elastic_password=changeme
   export elasticsearch_image=elasticsearch
-  export elasticsearch_url=https://elastic:${elastic_password}@${es_node_name}:9200
+  export elasticsearch_scheme="https"
   if [[ $TEST_SUITE != "platinum" ]]; then
-    export elasticsearch_url=http://${es_node_name}:9200
+    export elasticsearch_scheme="http"
   fi
+  export elasticsearch_url=${elasticsearch_scheme}://elastic:${elastic_password}@${es_node_name}:9200
   export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
   export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"
 

.ci/run-elasticsearch.sh

@@ -7,13 +7,18 @@
 # Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'.
 # Export the NUMBER_OF_NODES variable to start more than 1 node
 
-# Version 1.1.0
+# Version 1.5.0
 # - Initial version of the run-elasticsearch.sh script
 # - Deleting the volume should not dependent on the container still running
 # - Fixed `ES_JAVA_OPTS` config
 # - Moved to STACK_VERSION and TEST_VERSION
 # - Refactored into functions and imports
 # - Support NUMBER_OF_NODES
+# - Added 5 retries on docker pull for fixing transient network errors
+# - Added flags to make local CCR configurations work
+# - Added action.destructive_requires_name=false as the default will be true in v8
+# - Added ingest.geoip.downloader.enabled=false as it causes false positives in testing
+# - Moved ELASTIC_PASSWORD to the base arguments for "Security On by default"
 
 script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
@@ -27,6 +32,7 @@ cluster_name=${moniker}${suffix}
 
 declare -a volumes
 environment=($(cat <<-END
+  --env ELASTIC_PASSWORD=$elastic_password
   --env node.name=$es_node_name
   --env cluster.name=$cluster_name
   --env cluster.initial_master_nodes=$master_node_name
@@ -36,6 +42,8 @@ environment=($(cat <<-END
   --env node.attr.testattr=test
   --env path.repo=/tmp
   --env repositories.url.allowed_urls=http://snapshot.test*
+  --env action.destructive_requires_name=false
+  --env ingest.geoip.downloader.enabled=false
 END
 ))
 if [[ "$TEST_SUITE" == "platinum" ]]; then
@@ -49,6 +57,7 @@ if [[ "$TEST_SUITE" == "platinum" ]]; then
     --env xpack.security.http.ssl.certificate=certs/testnode.crt
     --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt
     --env xpack.security.transport.ssl.enabled=true
+    --env xpack.security.transport.ssl.verification_mode=certificate
     --env xpack.security.transport.ssl.key=certs/testnode.key
     --env xpack.security.transport.ssl.certificate=certs/testnode.crt
     --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt
@@ -67,6 +76,17 @@ if [[ "$TEST_SUITE" == "platinum" ]]; then
   cert_validation_flags="--insecure --cacert /usr/share/elasticsearch/config/certs/ca.crt --resolve ${es_node_name}:443:127.0.0.1"
 fi
 
+# Pull the container, retry on failures up to 5 times with
+# short delays between each attempt. Fixes most transient network errors.
+docker_pull_attempts=0
+until [ "$docker_pull_attempts" -ge 5 ]
+do
+   docker pull docker.elastic.co/elasticsearch/"$elasticsearch_container" && break
+   docker_pull_attempts=$((docker_pull_attempts+1))
+   echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..."
+   sleep 10
+done
+
 NUMBER_OF_NODES=${NUMBER_OF_NODES-1}
 http_port=9200
 for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do
@@ -92,7 +112,7 @@ END
   docker run \
     --name "$node_name" \
     --network "$network_name" \
-    --env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
+    --env "ES_JAVA_OPTS=-Xms1g -Xmx1g -da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions" \
     "${environment[@]}" \
     "${volumes[@]}" \
     --publish "$http_port":9200 \
@@ -112,4 +132,3 @@ END
   fi
 
 done
-

.gitignore

@@ -6,8 +6,8 @@ Cargo.lock
 *.iml
 .vscode/
 *.log
-yaml_test_runner/yaml/
-yaml_test_runner/tests/oss
+yaml_test_runner/tests/free
 yaml_test_runner/tests/xpack
 yaml_test_runner/tests/mod.rs
 test_results/
+checkout/

Cargo.toml

@@ -2,5 +2,11 @@
 members = [
     "api_generator",
     "elasticsearch",
-    "yaml_test_runner"
-]
+    "yaml_test_runner",
+    "xtask"
+]
+
+# See https://jakedeichert.com/blog/reducing-rust-incremental-compilation-times-on-macos-by-70-percent/
+# and https://github.com/rust-lang/cargo/pull/9298
+[profile.dev]
+split-debuginfo = "unpacked"

Makefile.toml

@@ -16,7 +16,7 @@ category = "Elasticsearch"
 description = "Sets ELASTICSEARCH_URL environment variable if not already set for later tasks when free test suite used"
 private = true
 condition = { env = { "TEST_SUITE" = "free" }, env_not_set = ["ELASTICSEARCH_URL"] }
-env = { "ELASTICSEARCH_URL" = "http://localhost:9200" }
+env = { "ELASTICSEARCH_URL" = "http://elastic:changeme@localhost:9200" }
 
 [tasks.set-platinum-env]
 category = "Elasticsearch"
@@ -25,6 +25,16 @@ private = true
 condition = { env = { "TEST_SUITE" = "platinum" }, env_not_set = ["ELASTICSEARCH_URL"] }
 env = { "ELASTICSEARCH_URL" = "https://elastic:changeme@localhost:9200" }
 
+[tasks.download-specs]
+category = "Elasticsearch"
+description = '''Download Rest API specs and YAML tests'''
+private = true
+command = "cargo"
+# cargo-make insists on installing cargo-xtask and ignores .cargo/config.toml
+#args = ["xtask", "download-specs", "--url", "${ELASTICSEARCH_URL}"]
+args = ["run", "-q", "-p", "xtask", "--", "download-specs", "--url", "${ELASTICSEARCH_URL}"]
+dependencies = ["start-elasticsearch"]
+
 [tasks.run-yaml-test-runner]
 category = "Elasticsearch"
 description = '''
@@ -34,7 +44,7 @@ The commit to use is retrieved from the running Elasticsearch instance
 private = true
 command = "cargo"
 args = ["run", "-p", "yaml_test_runner", "--", "-u", "${ELASTICSEARCH_URL}"]
-dependencies = ["start-elasticsearch"]
+dependencies = ["download-specs"]
 
 [tasks.test-yaml-test-runner]
 category = "Elasticsearch"
@@ -64,6 +74,7 @@ category = "Elasticsearch"
 private = true
 command = "cargo"
 args = ["run", "-p", "api_generator"]
+dependencies = ["download-specs"]
 
 [tasks.create-test-results-dir]
 category = "Elasticsearch"

api_generator/Cargo.toml

@@ -9,9 +9,9 @@ edition = "2018"
 license = "Apache-2.0"
 
 [dependencies]
+anyhow = "1.0"
 array_tool = "1.0.3"
 dialoguer = "0.3.0"
-failure = "0.1.5"
 flate2 = "~1"
 globset = "~0.4"
 Inflector = "0.11.4"

api_generator/docs/namespaces/fleet.md

@@ -0,0 +1 @@
+[The Fleet APIs](https://www.elastic.co/guide/en/elasticsearch/reference/master/fleet-apis.html) support the use of Elasticsearch as a data store for internal agent and action data. These APIs are experimental and for internal use by Fleet only.

api_generator/docs/namespaces/shutdown.md

@@ -0,0 +1 @@
+The [shutdown APIs](https://www.elastic.co/guide/en/elasticsearch/reference/master/node-lifecycle-api.html) allows preparing nodes for temporary or permanent shutdown, monitor the shutdown status, and enable a previously shut-down node to resume normal operations.

api_generator/src/bin/run.rs

@@ -19,107 +19,43 @@
 extern crate api_generator;
 extern crate dialoguer;
 
-use api_generator::{generator, rest_spec};
-use dialoguer::Input;
-use std::{
-    fs::{self, File},
-    io::Write,
-    path::PathBuf,
-};
+use anyhow::{bail, Context};
+use api_generator::generator;
+use std::{fs, path::PathBuf};
 
-fn main() -> Result<(), failure::Error> {
+fn main() -> anyhow::Result<()> {
     simple_logger::SimpleLogger::new()
         .with_level(log::LevelFilter::Info)
         .init()
         .unwrap();
 
-    // This must be run from the repo root directory, with cargo make generate-api
-    let download_dir = fs::canonicalize(PathBuf::from("./api_generator/rest_specs"))?;
-    let generated_dir = fs::canonicalize(PathBuf::from("./elasticsearch/src"))?;
-    let last_downloaded_version =
-        PathBuf::from("./api_generator/rest_specs/last_downloaded_version");
-
-    let mut download_specs = false;
-    let mut answer = String::new();
-    let default_branch = if last_downloaded_version.exists() {
-        fs::read_to_string(&last_downloaded_version)?
-    } else {
-        String::from("master")
-    };
-    let mut branch = default_branch.clone();
+    let stack_version = std::env::var("STACK_VERSION").context("Missing STACK_VERSION env var")?;
 
-    while answer != "y" && answer != "n" {
-        answer = Input::new()
-            .default(String::from("n"))
-            .show_default(false)
-            .with_prompt("Download rest specifications [y/N]")
-            .interact()
-            .unwrap()
-            .to_lowercase();
-        download_specs = answer == "y";
-    }
-
-    if download_specs {
-        branch = Input::new()
-            .default(default_branch.clone())
-            .show_default(false)
-            .with_prompt(
-                format!(
-                    "Branch to download specification from [default {}]",
-                    default_branch
-                )
-                .as_str(),
-            )
-            .interact()
-            .unwrap();
+    // This must be run from the repo root directory, with cargo make generate-api
+    let download_dir = PathBuf::from(&format!("./checkout/{}/rest-api-spec/api", stack_version));
 
-        fs::remove_dir_all(&download_dir)?;
-        fs::create_dir_all(&download_dir)?;
-        rest_spec::download_specs(&branch, &download_dir)?;
-        File::create(&last_downloaded_version)?.write_all(branch.as_bytes())?;
+    if !download_dir.is_dir() {
+        bail!("No specs found at {:?}", download_dir);
     }
 
-    // only offer to generate if there are downloaded specs
-    if download_dir
-        .read_dir()
-        .map(|mut r| r.next().is_some())
-        .unwrap_or(false)
-    {
-        let mut generate_code = true;
-        answer = String::new();
-        while answer != "y" && answer != "n" {
-            answer = Input::new()
-                .default(String::from("y"))
-                .show_default(false)
-                .with_prompt(
-                    format!("Generate code from rest specifications {} [Y/n]", branch).as_str(),
-                )
-                .interact()
-                .unwrap()
-                .to_lowercase();
-            generate_code = answer == "y";
-        }
-
-        if generate_code {
-            // Delete previously generated files
-            let mut generated = generated_dir.clone();
-            generated.push(generator::GENERATED_TOML);
-
-            if generated.exists() {
-                let files =
-                    toml::from_str::<generator::GeneratedFiles>(&fs::read_to_string(generated)?)?;
+    // let download_dir = fs::canonicalize(PathBuf::from("./api_generator/rest_specs"))?;
+    let generated_dir = fs::canonicalize(PathBuf::from("./elasticsearch/src"))?;
 
-                for f in files.written {
-                    let mut generated_file = generated_dir.clone();
-                    generated_file.push(f);
-                    let _ = fs::remove_file(generated_file); // ignore missing files
-                }
-            }
+    // Delete previously generated files
+    let mut generated = generated_dir.clone();
+    generated.push(generator::GENERATED_TOML);
+    if generated.exists() {
+        let files = toml::from_str::<generator::GeneratedFiles>(&fs::read_to_string(generated)?)?;
 
-            // and generate!
-            generator::generate(&branch, &download_dir, &generated_dir)?;
+        for f in files.written {
+            let mut generated_file = generated_dir.clone();
+            generated_file.push(f);
+            let _ = fs::remove_file(generated_file); // ignore missing files
         }
     }
 
+    // and generate!
+    generator::generate(&download_dir, &generated_dir)?;
+
     Ok(())
 }

api_generator/src/generator/code_gen/namespace_clients.rs

@@ -25,7 +25,7 @@ use quote::Tokens;
 use std::path::PathBuf;
 
 /// Generates the source code for a namespaced client
-pub fn generate(api: &Api, docs_dir: &PathBuf) -> Result<Vec<(String, String)>, failure::Error> {
+pub fn generate(api: &Api, docs_dir: &PathBuf) -> anyhow::Result<Vec<(String, String)>> {
     let mut output = Vec::new();
 
     for (namespace_name, namespace) in &api.namespaces {

api_generator/src/generator/code_gen/params.rs

@@ -22,7 +22,7 @@ use inflector::Inflector;
 use quote::Tokens;
 use regex::Regex;
 
-pub fn generate(api: &Api) -> Result<String, failure::Error> {
+pub fn generate(api: &Api) -> anyhow::Result<String> {
     let mut tokens = quote!(
         use serde::{Serialize, Deserialize};
     );

api_generator/src/generator/code_gen/root.rs

@@ -25,7 +25,7 @@ use quote::Tokens;
 use std::path::PathBuf;
 
 /// Generates the source code for the methods on the root of Elasticsearch
-pub fn generate(api: &Api, docs_dir: &PathBuf) -> Result<String, failure::Error> {
+pub fn generate(api: &Api, docs_dir: &PathBuf) -> anyhow::Result<String> {
     let mut tokens = Tokens::new();
     tokens.append(use_declarations());