Cherry-pick test changes from master to release/1.2 (Azure#4068)

This PR is a set of pending cherry-picks for release/1.2. It contains mostly test changes without product changes:

- Policy integration tests
- Policy E2E tests
- Improvements to the MQTT pipeline to report test results

builds/checkin/mqtt.yaml

@@ -5,14 +5,16 @@ pr:
       - master
       - release/*
 jobs:
-
   ################################################################################
   - job: check_run_pipeline
-    ################################################################################
+  ################################################################################
     displayName: Check pipeline preconditions (changes ARE in builds or mqtt)
     pool:
       vmImage: "ubuntu-16.04"
     steps:
+      - checkout: self
+        submodules: false
+        fetchDepth: 3
       - bash: |
           git log -m -1 --name-only --first-parent --pretty="" | egrep -i '^(builds|mqtt)'
           if [[ $? == 0 ]]; then
@@ -24,13 +26,16 @@ jobs:
 
   ################################################################################
   - job: linux_amd64
-    ################################################################################
+  ################################################################################
     displayName: Linux amd64
     dependsOn: check_run_pipeline
     condition: eq(dependencies.check_run_pipeline.outputs['check_files.RUN_PIPELINE'], 'true')
     pool:
       vmImage: "ubuntu-16.04"
     steps:
+      - checkout: self
+        submodules: false # mqtt broker does not use submodules
+        fetchDepth: 3
       - script: echo "##vso[task.setvariable variable=RUST_BACKTRACE;]1"
         displayName: Set env variables
       - bash: scripts/linux/generic-rust/install.sh --project-root "mqtt"
@@ -39,18 +44,28 @@ jobs:
         displayName: Build with no default features
       - bash: scripts/linux/generic-rust/build.sh --project-root "mqtt" --packages "mqttd/Cargo.toml" --manifest-path
         displayName: Build with default features
-      - bash: mqtt/build/linux/test.sh
+      - bash: mqtt/build/linux/test.sh --report test-results.xml
         displayName: Test
+      - task: PublishTestResults@2
+        displayName: Publish test results
+        inputs:
+          testResultsFormat: "JUnit"
+          testResultsFiles: "**/test-results.xml"
+          failTaskOnFailedTests: true
+        condition: succeededOrFailed()
 
   ################################################################################
   - job: style_check
-    ################################################################################
+  ################################################################################
     displayName: Style Check
     dependsOn: check_run_pipeline
     condition: eq(dependencies.check_run_pipeline.outputs['check_files.RUN_PIPELINE'], 'true')
     pool:
       vmImage: "ubuntu-16.04"
     steps:
+      - checkout: self
+        submodules: false # mqtt broker does not use submodules
+        fetchDepth: 3
       - bash: scripts/linux/generic-rust/install.sh --project-root "mqtt"
         displayName: Install Rust
       - bash: scripts/linux/generic-rust/format.sh --project-root "mqtt"

builds/ci/mqtt.yaml

@@ -13,6 +13,9 @@ jobs:
     pool:
       vmImage: "ubuntu-16.04"
     steps:
+      - checkout: self
+        submodules: false
+        fetchDepth: 3
       - bash: |
           git log -m -1 --name-only --first-parent --pretty="" | egrep -i '^(builds|mqtt)'
           if [[ $? == 0 ]]; then
@@ -31,6 +34,9 @@ jobs:
     pool:
       vmImage: 'ubuntu-16.04'
     steps:
+      - checkout: self
+        submodules: false # mqtt broker does not use submodules
+        fetchDepth: 3
       - task: Bash@3
         displayName: Install Rust
         inputs:
@@ -50,6 +56,14 @@ jobs:
         displayName: Test
         inputs:
           filePath: mqtt/build/linux/test.sh
+          arguments: --report test-results.xml
+      - task: PublishTestResults@2
+        displayName: Publish test results
+        inputs:
+          testResultsFormat: "JUnit"
+          testResultsFiles: "**/test-results.xml"
+          failTaskOnFailedTests: true
+        condition: succeededOrFailed()
 
 ################################################################################
   - job: linux_arm32v7
@@ -60,6 +74,9 @@ jobs:
     pool:
       vmImage: 'ubuntu-16.04'
     steps:
+      - checkout: self
+        submodules: false # mqtt broker does not use submodules
+        fetchDepth: 3
       - script: |
           echo "##vso[task.setvariable variable=RUSTUP_HOME;]$(Agent.WorkFolder)/rustup"
           echo "##vso[task.setvariable variable=CARGO_HOME;]$(Agent.WorkFolder)/cargo"
@@ -73,41 +90,24 @@ jobs:
       - script: cargo install cross --version 0.1.16
         displayName: Install cross
       - task: Bash@3
-        displayName: Build
+        displayName: Build with no default features
         inputs:
           filePath: scripts/linux/generic-rust/build.sh
           arguments: --project-root "mqtt" --packages "mqttd/Cargo.toml" --manifest-path --no-default-features --features "generic" --target armv7-unknown-linux-gnueabihf --cargo cross
       - task: Bash@3
-        displayName: Build
+        displayName: Build with default features
         inputs:
           filePath: scripts/linux/generic-rust/build.sh
           arguments: --project-root "mqtt" --packages "mqttd/Cargo.toml" --manifest-path --target armv7-unknown-linux-gnueabihf --cargo cross
       - task: Bash@3
         displayName: Test
         inputs:
           filePath: mqtt/build/linux/test.sh
-          arguments: --target armv7-unknown-linux-gnueabihf --cargo cross
-
-################################################################################
-  - job: style_check
-################################################################################
-    displayName: Style Check
-    dependsOn: check_run_pipeline
-    condition: eq(dependencies.check_run_pipeline.outputs['check_files.RUN_PIPELINE'], 'true')
-    pool:
-      vmImage: 'ubuntu-16.04'
-    steps:
-      - task: Bash@3
-        displayName: Install Rust
-        inputs:
-          filePath: scripts/linux/generic-rust/install.sh
-          arguments: --project-root "mqtt"
-      - task: Bash@3
-        displayName: Format Code
-        inputs:
-          filePath: scripts/linux/generic-rust/format.sh
-          arguments: --project-root "mqtt"
-      - task: Bash@3
-        displayName: Clippy
+          arguments: --target armv7-unknown-linux-gnueabihf --cargo cross --report test-results.xml
+      - task: PublishTestResults@2
+        displayName: Publish test results
         inputs:
-          filePath: mqtt/build/linux/clippy.sh
+          testResultsFormat: "JUnit"
+          testResultsFiles: "**/test-results.xml"
+          failTaskOnFailedTests: true
+        condition: succeededOrFailed()

edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.Core/config/BrokerProperties.cs

@@ -1,10 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 namespace Microsoft.Azure.Devices.Edge.Hub.Core.Config
 {
-    using System;
-    using System.Linq;
-    using Microsoft.Azure.Devices.Edge.Util;
-    using Microsoft.Azure.Devices.Edge.Util.Json;
     using Newtonsoft.Json;
 
     /// <summary>

mqtt/build/linux/test.sh

@@ -28,7 +28,10 @@ usage()
     echo ""
     echo "options"
     echo " -h, --help          Print this help and exit."
+    echo " -t, --target        Target architecture."
     echo " -r, --release       Release build? (flag, default: false)"
+    echo " -c, --cargo         Path of cargo installation."
+    echo " --report            Optional. Generates the xml test report with specified name."
     exit 1;
 }
 
@@ -44,17 +47,18 @@ process_args()
             TARGET="$arg"
             save_next_arg=0
         elif [ $save_next_arg -eq 2 ]; then
-            RELEASE="true"
+            CARGO="$arg"
             save_next_arg=0
         elif [ $save_next_arg -eq 3 ]; then
-            CARGO="$arg"
+            REPORT="$arg"
             save_next_arg=0
         else
             case "$arg" in
                 "-h" | "--help" ) usage;;
                 "-t" | "--target" ) save_next_arg=1;;
-                "-r" | "--release" ) save_next_arg=2;;
-                "-c" | "--cargo" ) save_next_arg=3;;
+                "-r" | "--release" ) RELEASE="--release";;
+                "-c" | "--cargo" ) save_next_arg=2;;
+                "--report" ) save_next_arg=3;;
                 * ) usage;;
             esac
         fi
@@ -63,8 +67,18 @@ process_args()
 
 process_args "$@"
 
-if [[ -z ${RELEASE} ]]; then
-    cd "$PROJECT_ROOT" && $CARGO test --workspace --all-features --target "$TARGET"
+if [[ -z ${REPORT} ]]; then
+    echo  $CARGO test --no-fail-fast --workspace --all-features --target "$TARGET" "$RELEASE"
+    cd "$PROJECT_ROOT" && $CARGO test --no-fail-fast --workspace --all-features \
+        --target "$TARGET" "$RELEASE"
 else
-    cd "$PROJECT_ROOT" && $CARGO test --workspace --all-features --release --target "$TARGET"
+    # Get cargo2junit to report test results to Azure Pipelines
+    $CARGO install cargo2junit
+
+    cd "$PROJECT_ROOT" && $CARGO test --no-fail-fast --workspace --all-features \
+        --target "$TARGET" "$RELEASE" \
+        -- -Z unstable-options --format json | tee test-result.json
+
+    # Convert test results to junit format.
+    cat test-result.json | cargo2junit > $REPORT
 fi

mqtt/mqtt-broker-tests-util/src/server.rs

@@ -77,7 +77,7 @@ where
     P: MakePacketProcessor + Clone + Send + Sync + 'static,
 {
     lazy_static! {
-        static ref PORT: AtomicU32 = AtomicU32::new(5555);
+        static ref PORT: AtomicU32 = AtomicU32::new(8889);
     }
 
     let port = PORT.fetch_add(1, Ordering::SeqCst);

mqtt/mqtt-broker/src/persist.rs

@@ -544,32 +544,14 @@ mod tests {
     use tempfile::TempDir;
 
     use crate::{
-        persist::{ConsolidatedState, FileFormat, FilePersistor, Persist, VersionedFileFormat},
+        persist::{FileFormat, FilePersistor, Persist, VersionedFileFormat},
         proptest::arb_broker_snapshot,
         BrokerSnapshot,
     };
 
     proptest! {
         #[test]
-        fn consolidate_simple(state in arb_broker_snapshot()) {
-            let (expected_retained, expected_sessions) = state.clone().into_parts();
-
-            let consolidated: ConsolidatedState = state.into();
-            prop_assert_eq!(expected_retained.len(), consolidated.retained.len());
-            prop_assert_eq!(expected_sessions.len(), consolidated.sessions.len());
-
-            let state: BrokerSnapshot = consolidated.into();
-            let (result_retained, result_sessions) = state.into_parts();
-
-            prop_assert_eq!(expected_retained, result_retained);
-            prop_assert_eq!(expected_sessions.len(), result_sessions.len());
-            for i in 0..expected_sessions.len(){
-                prop_assert_eq!(expected_sessions[i].clone().into_parts(), result_sessions[i].clone().into_parts());
-            }
-        }
-
-        #[test]
-        fn consolidate_roundtrip(state in arb_broker_snapshot()) {
+        fn broker_state_versioned_file_format_persistance_test(state in arb_broker_snapshot()) {
             let (expected_retained, expected_sessions) = state.clone().into_parts();
             let format = VersionedFileFormat;
             let mut buffer = vec![0_u8; 10 * 1024 * 1024];
@@ -581,10 +563,7 @@ mod tests {
             let (result_retained, result_sessions) = state.into_parts();
 
             prop_assert_eq!(expected_retained, result_retained);
-            prop_assert_eq!(expected_sessions.len(), result_sessions.len());
-            for i in 0..expected_sessions.len(){
-                prop_assert_eq!(expected_sessions[i].clone().into_parts(), result_sessions[i].clone().into_parts());
-            }
+            prop_assert_eq!(expected_sessions, result_sessions);
         }
     }
 

mqtt/mqtt-broker/src/proptest.rs

@@ -5,53 +5,30 @@ use bytes::Bytes;
 use mqtt3::proto;
 use proptest::{
     bool,
-    collection::{hash_map, hash_set, vec, vec_deque},
+    collection::{hash_map, vec, vec_deque},
     num,
     prelude::*,
 };
 
 use crate::{
-    session::identifiers::{IdentifiersInUse, PacketIdentifiers},
     AuthId, BrokerSnapshot, ClientId, ClientInfo, Publish, Segment, SessionSnapshot, Subscription,
     TopicFilter,
 };
 
 prop_compose! {
     pub fn arb_broker_snapshot()(
-        retained in hash_map(arb_topic(), arb_publication(), 0..20),
-        sessions in vec(arb_session_snapshot(), 0..10),
+        retained in hash_map(arb_topic(), arb_publication(), 0..5),
+        sessions in vec(arb_session_snapshot(), 0..5),
     ) -> BrokerSnapshot {
         BrokerSnapshot::new(retained, sessions)
     }
 }
 
-prop_compose! {
-    pub(crate) fn arb_packet_identifiers()(
-        in_use in arb_identifiers_in_use(),
-        previous in arb_packet_identifier(),
-    ) -> PacketIdentifiers {
-        PacketIdentifiers::new(in_use, previous)
-    }
-}
-
-pub(crate) fn arb_identifiers_in_use() -> impl Strategy<Value = IdentifiersInUse> {
-    vec(num::usize::ANY, PacketIdentifiers::SIZE).prop_map(|v| {
-        let mut array = [0; PacketIdentifiers::SIZE];
-        let nums = &v[..array.len()];
-        array.copy_from_slice(nums);
-        IdentifiersInUse(Box::new(array))
-    })
-}
-
 prop_compose! {
     pub fn arb_session_snapshot()(
         client_info in arb_client_info(),
-        subscriptions in hash_map(arb_topic(), arb_subscription(), 0..10),
-        _packet_identifiers in arb_packet_identifiers(),
-        waiting_to_be_sent in vec_deque(arb_publication(), 0..10),
-        _waiting_to_be_released in hash_map(arb_packet_identifier(), arb_proto_publish(), 0..10),
-        _waiting_to_be_acked in hash_map(arb_packet_identifier(), arb_publish(), 0..10),
-        _waiting_to_be_completed in hash_set(arb_packet_identifier(), 0..10),
+        subscriptions in hash_map(arb_topic(), arb_subscription(), 0..5),
+        waiting_to_be_sent in vec_deque(arb_publication(), 0..5),
     ) -> SessionSnapshot {
         SessionSnapshot::from_parts(
             client_info,
@@ -81,7 +58,7 @@ prop_compose! {
 prop_compose! {
     pub fn arb_subscribe()(
         packet_identifier in arb_packet_identifier(),
-        subscribe_to in proptest::collection::vec(arb_subscribe_to(), 1..10)
+        subscribe_to in proptest::collection::vec(arb_subscribe_to(), 1..5)
     ) -> proto::Subscribe {
         proto::Subscribe {
             packet_identifier,
@@ -105,7 +82,7 @@ prop_compose! {
 prop_compose! {
     pub fn arb_unsubscribe()(
         packet_identifier in arb_packet_identifier(),
-        unsubscribe_from in proptest::collection::vec(arb_topic_filter_weighted(), 1..10)
+        unsubscribe_from in proptest::collection::vec(arb_topic_filter_weighted(), 1..5)
     ) -> proto::Unsubscribe {
         proto::Unsubscribe {
             packet_identifier,
@@ -194,7 +171,7 @@ pub fn arb_topic() -> impl Strategy<Value = String> {
 }
 
 pub fn arb_payload() -> impl Strategy<Value = Bytes> {
-    vec(num::u8::ANY, 0..1024).prop_map(Bytes::from)
+    vec(num::u8::ANY, 0..128).prop_map(Bytes::from)
 }
 
 prop_compose! {

mqtt/mqtt-broker/src/session/identifiers.rs

@@ -90,11 +90,6 @@ impl PacketIdentifiers {
     /// We use a bitshift instead of `usize::pow` because the latter is not a const fn
     pub(crate) const SIZE: usize = (1 << 16) / (mem::size_of::<usize>() * 8);
 
-    #[cfg(any(test, feature = "proptest"))]
-    pub(crate) fn new(in_use: IdentifiersInUse, previous: proto::PacketIdentifier) -> Self {
-        Self { in_use, previous }
-    }
-
     pub(crate) fn reserve(&mut self) -> Result<proto::PacketIdentifier, Error> {
         let start = self.previous;
         let mut current = start;