diff --git a/tools/ci-cdk/README.md b/tools/ci-cdk/README.md index 691489eeeb..cbbd304293 100644 --- a/tools/ci-cdk/README.md +++ b/tools/ci-cdk/README.md @@ -21,7 +21,7 @@ From there, you can just point the `canary-runner` to the `cdk-outputs.json` to ```bash cd canary-runner -cargo run -- run --sdk-release-tag --musl --cdk-outputs ../cdk-outputs.json +cargo run -- run --sdk-release-tag --musl --cdk-output ../cdk-outputs.json ``` __NOTE:__ You may want to add a `--profile` to the `deploy` command to select a specific credential diff --git a/tools/ci-cdk/canary-lambda/src/canary.rs b/tools/ci-cdk/canary-lambda/src/canary.rs index 67689fd093..0f3a986a09 100644 --- a/tools/ci-cdk/canary-lambda/src/canary.rs +++ b/tools/ci-cdk/canary-lambda/src/canary.rs @@ -84,7 +84,7 @@ impl CanaryEnv { // Amazon Transcribe starts returning different output for the same audio. let expected_transcribe_result = env::var("CANARY_EXPECTED_TRANSCRIBE_RESULT") .unwrap_or_else(|_| { - "Good day to you transcribe. This is Polly talking to you from the Rust S. D. K." + "Good day to you transcribe. This is Polly talking to you from the Rust SDK." .to_string() }); diff --git a/tools/ci-cdk/canary-lambda/src/latest/transcribe_canary.rs b/tools/ci-cdk/canary-lambda/src/latest/transcribe_canary.rs index 8f6420fc1b..065539b97f 100644 --- a/tools/ci-cdk/canary-lambda/src/latest/transcribe_canary.rs +++ b/tools/ci-cdk/canary-lambda/src/latest/transcribe_canary.rs @@ -3,12 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -use crate::canary::CanaryError; use crate::mk_canary; +use anyhow::bail; use async_stream::stream; use aws_config::SdkConfig; use aws_sdk_transcribestreaming as transcribe; use bytes::BufMut; +use edit_distance::edit_distance; use transcribe::primitives::Blob; use transcribe::types::{ AudioEvent, AudioStream, LanguageCode, MediaEncoding, TranscriptResultStream, @@ -64,18 +65,20 @@ pub async fn transcribe_canary( } } - if expected_transcribe_result != full_message.trim() { - Err(CanaryError(format!( + let dist = edit_distance(&expected_transcribe_result, full_message.trim()); + let max_edit_distance = 10; + if dist > max_edit_distance { + bail!( "Transcription from Transcribe doesn't look right:\n\ Expected: `{}`\n\ - Actual: `{}`\n", + Actual: `{}`\n. The maximum allowed edit distance is {}. This had an edit distance of {}", expected_transcribe_result, - full_message.trim() - )) - .into()) - } else { - Ok(()) + full_message.trim(), + max_edit_distance, + dist + ) } + Ok(()) } fn pcm_data() -> Vec { diff --git a/tools/ci-cdk/canary-runner/src/build_bundle.rs b/tools/ci-cdk/canary-runner/src/build_bundle.rs index 9a212bd7c8..eaea69835e 100644 --- a/tools/ci-cdk/canary-runner/src/build_bundle.rs +++ b/tools/ci-cdk/canary-runner/src/build_bundle.rs @@ -54,6 +54,7 @@ uuid = { version = "0.8", features = ["v4"] } tokio-stream = "0" tracing-texray = "0.1.1" reqwest = { version = "0.11.14", features = ["rustls-tls"], default-features = false } +edit-distance = "2" "#; const REQUIRED_SDK_CRATES: &[&str] = &[ @@ -451,6 +452,7 @@ uuid = { version = "0.8", features = ["v4"] } tokio-stream = "0" tracing-texray = "0.1.1" reqwest = { version = "0.11.14", features = ["rustls-tls"], default-features = false } +edit-distance = "2" aws-config = { path = "some/sdk/path/aws-config", features = ["behavior-version-latest"] } aws-sdk-s3 = { path = "some/sdk/path/s3" } aws-sdk-ec2 = { path = "some/sdk/path/ec2" } @@ -515,6 +517,7 @@ uuid = { version = "0.8", features = ["v4"] } tokio-stream = "0" tracing-texray = "0.1.1" reqwest = { version = "0.11.14", features = ["rustls-tls"], default-features = false } +edit-distance = "2" aws-config = { version = "0.46.0", features = ["behavior-version-latest"] } aws-sdk-s3 = "0.20.0" aws-sdk-ec2 = "0.19.0"