Update to rust 1.52

.github/workflows/tests.yaml

@@ -45,8 +45,8 @@ jobs:
           PROPTEST_CASES: 2500
           RUSTFLAGS: -D warnings -C target-feature=+avx,+avx2,+sse4.2
         with:
-          version: "0.16.0"
-          args: " --exclude-files target* tremor-cli tremor-api deprecated **/errors.rs --out Lcov --all"
+          version: "0.18.0-alpha3"
+          args: " --avoid-cfg-tarpaulin --exclude-files target* tremor-cli tremor-api **/errors.rs --out Lcov --all"
       - name: Coveralls
         uses: coverallsapp/github-action@master
         with:

CHANGELOG.md

@@ -22,6 +22,8 @@
 - Include `cncf::otel` stdlib sources in deb package
 - Add `/usr/local/share/tremor` to default `TREMOR_PATH` also for all packages as a well-known directory for custom tremor-script libraries and modules.
 - Record the partition number assigned during rebalancing when running Kafka.
+- Fix bug in HDR histogram implementation when using emit without reset.
+- Fix bug in mean that invalid values would be counted as part of the total number of values.
 
 ## 0.11.1
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.51.0 as builder
+FROM rust:1.52.1 as builder
 
 # Avoid warnings by switching to noninteractive
 ENV DEBIAN_FRONTEND=noninteractive

Dockerfile.learn

@@ -1,4 +1,4 @@
-FROM rust:1.51.0 as builder
+FROM rust:1.52.1 as builder
 
 RUN cargo install --features=ssl websocat
 

Dockerfile.native

@@ -1,4 +1,4 @@
-FROM rust:1.51.0 as builder
+FROM rust:1.52.1 as builder
 
 # Avoid warnings by switching to noninteractive
 ENV DEBIAN_FRONTEND=noninteractive

rust-toolchain

@@ -1 +1 @@
-1.51.0
+1.52.1

src/lib.rs

@@ -23,10 +23,10 @@
     clippy::unnecessary_unwrap,
     clippy::pedantic
 )]
-// TODOthis is needed due to a false positive in clippy
+// TODO this is needed due to a false positive in clippy
 // https://github.com/rust-lang/rust/issues/83125
 // we will need this in 1.52.0
-// #![allow(proc_macro_back_compat)]
+#![allow(proc_macro_back_compat)]
 
 #[macro_use]
 extern crate serde_derive;

src/preprocessor.rs

@@ -502,7 +502,8 @@ mod test {
             }
         }
 
-            fn textual_length_pre_post(length in 1..100_usize) {
+        #[test]
+        fn textual_length_pre_post(length in 1..100_usize) {
             let data = vec![1_u8; length];
             let mut pre_p = pre::TextualLength::default();
             let mut post_p = post::TextualLength::default();
@@ -821,31 +822,31 @@ mod test {
     fn test_gzip() -> Result<()> {
         let int = "snot".as_bytes();
         assert_simple_symmetric!(int, Gzip, "gzip");
-        assert_decompress!(int, Lz4, "lz4");
+        assert_decompress!(int, Gzip, "gzip");
         Ok(())
     }
 
     #[test]
     fn test_zlib() -> Result<()> {
         let int = "snot".as_bytes();
         assert_simple_symmetric!(int, Zlib, "zlib");
-        assert_decompress!(int, Lz4, "lz4");
+        assert_decompress!(int, Zlib, "zlib");
         Ok(())
     }
 
     #[test]
     fn test_snappy() -> Result<()> {
         let int = "snot".as_bytes();
         assert_simple_symmetric!(int, Snappy, "snap");
-        assert_decompress!(int, Lz4, "lz4");
+        assert_decompress!(int, Snappy, "snap");
         Ok(())
     }
 
     #[test]
     fn test_xz2() -> Result<()> {
         let int = "snot".as_bytes();
         assert_simple_symmetric!(int, Xz2, "xz2");
-        assert_decompress!(int, Lz4, "lz4");
+        assert_decompress!(int, Xz2, "xz2");
         Ok(())
     }
 

tests/script_error.rs

@@ -134,6 +134,9 @@ test_cases!(
     pp_cyclic,
     pp_nest_cyclic,
     // INSERT
+    double_const_mod,
+    bin_invalid_bits,
+    bin_invalid_type,
     merge_ident,
     select_ident,
     function_already_defined,

tests/script_errors/bin_invalid_bits/error.txt

@@ -0,0 +1,3 @@
+Error: 
+    1 | <<event:100>>
+      |   ^^^^^^^^^ negative bits or bits > 64 are are not allowed: 100

tests/script_errors/bin_invalid_bits/script.tremor

@@ -0,0 +1 @@
+<<event:100>>

tests/script_errors/bin_invalid_type/error.txt

@@ -0,0 +1,3 @@
+Error: 
+    1 | <<event/snot>>
+      |   ^^^^^^^^^^ Not a valid data type: 'snot'

tests/script_errors/bin_invalid_type/script.tremor

@@ -0,0 +1 @@
+<<event/snot>>

tests/script_errors/double_const_mod/error.txt

@@ -0,0 +1,6 @@
+Error: 
+    1 | mod test with
+    2 | const a = 1;
+    3 | const a = 2;
+      | ^^^^^^^^^^^ Can't declare the constant `a` twice
+    4 | end;

tests/script_errors/double_const_mod/script.tremor

@@ -0,0 +1,4 @@
+mod test with
+const a = 1;
+const a = 2;
+end;

tests/script_errors/lexer_triple_colon/error.txt

@@ -1,5 +1,5 @@
 Error: 
     1 | #  we can't have triple colons
     2 | a:::b()
-      |  ^^^ Found the token `:::` but expected one of `!=`, `%`, `&`, `(`, `)`, `*`, `+`, `,`, `-`, `.`, `.`, `/`, `:`, `::`, `;`, `<`, `<<`, `<=`, `<end-of-stream>`, `==`, `=>`, `>`, `>=`, `>>`, `>>>`, `[`, `]`, `^`, `and`, `case`, `default`, `end`, `of`, `or`, `when`, `xor`, `|`, `}`
+      |  ^^^ Found the token `:::` but expected one of `!=`, `%`, `&`, `(`, `)`, `*`, `+`, `,`, `-`, `.`, `.`, `/`, `:`, `::`, `;`, `<`, `<<`, `<=`, `<end-of-stream>`, `==`, `=>`, `>`, `>=`, `>>`, `>>>`, `[`, `]`, `^`, `and`, `case`, `default`, `end`, `of`, `or`, `when`, `xor`, `}`
       |      NOTE: Did you mean to use `::`?

tremor-cli/src/debug.rs

@@ -69,7 +69,7 @@ where
     Ok(())
 }
 
-fn dbg_tokens<W>(h: &mut W, lexemes: Vec<Spanned<Token>>) -> Result<()>
+fn dbg_tokens<W>(h: &mut W, lexemes: Vec<Spanned>) -> Result<()>
 where
     W: Highlighter,
 {

tremor-cli/src/run.rs

@@ -168,8 +168,8 @@ impl Egress {
             is_interactive,
             is_pretty,
             buffer,
-            postprocessor,
             codec,
+            postprocessor,
         })
     }
 

tremor-pipeline/src/lib.rs

@@ -23,10 +23,10 @@
     clippy::unnecessary_unwrap,
     clippy::pedantic
 )]
-// TODOthis is needed due to a false positive in clippy
+// TODO this is needed due to a false positive in clippy
 // https://github.com/rust-lang/rust/issues/83125
 // we will need this in 1.52.0
-// #![allow(proc_macro_back_compat)]
+#![allow(proc_macro_back_compat)]
 
 #[macro_use]
 extern crate serde_derive;

tremor-pipeline/src/op/qos/wal.rs

@@ -47,14 +47,14 @@ impl AddAssign<u64> for Idx {
 impl Add<u64> for Idx {
     type Output = Idx;
     fn add(self, rhs: u64) -> Self::Output {
-        Idx::from(u64::from(self) + rhs)
+        Idx::from(u64::from(&self) + rhs)
     }
 }
 
 impl Add<usize> for Idx {
     type Output = Idx;
     fn add(self, rhs: usize) -> Self::Output {
-        Idx::from(u64::from(self) + rhs as u64)
+        Idx::from(u64::from(&self) + rhs as u64)
     }
 }
 
@@ -65,20 +65,6 @@ impl From<&Idx> for u64 {
     }
 }
 
-impl From<&mut Idx> for u64 {
-    fn from(i: &mut Idx) -> u64 {
-        let mut rdr = Cursor::new(&i.0);
-        rdr.read_u64::<BigEndian>().unwrap_or(0)
-    }
-}
-
-impl From<Idx> for u64 {
-    fn from(i: Idx) -> u64 {
-        let mut rdr = Cursor::new(&i.0);
-        rdr.read_u64::<BigEndian>().unwrap_or(0)
-    }
-}
-
 impl From<IVec> for Idx {
     fn from(v: IVec) -> Self {
         let mut rdr = Cursor::new(v);
@@ -98,7 +84,7 @@ impl Idx {
         self.0 = unsafe { mem::transmute(v.to_be()) };
     }
     fn set_min(&mut self, v: u64) {
-        if v < u64::from(*self) {
+        if v < u64::from(&*self) {
             self.0 = unsafe { mem::transmute(v.to_be()) };
         }
     }
@@ -109,24 +95,12 @@ impl AsRef<[u8]> for Idx {
     }
 }
 
-impl From<Idx> for IVec {
-    fn from(i: Idx) -> Self {
-        IVec::from(&i.0)
-    }
-}
-
 impl From<&Idx> for IVec {
     fn from(i: &Idx) -> Self {
         IVec::from(&i.0)
     }
 }
 
-impl From<&mut Idx> for IVec {
-    fn from(i: &mut Idx) -> Self {
-        IVec::from(&i.0)
-    }
-}
-
 #[derive(Debug, Clone, Deserialize, serde::Deserialize, serde::Serialize)]
 pub struct Config {
     /// Maximum number of events to read per tick/event when filling
@@ -193,17 +167,13 @@ pub struct Wal {
 }
 
 op!(WalFactory(_uid, node) {
-    if let Some(map) = &node.config {
-        let config: Config = Config::new(map)?;
-
-        if let (None, None) = (config.max_elements, config.max_bytes) {
-            Err(ErrorKind::BadOpConfig("WAL operator needs at least one of `max_elements` or `max_bytes` config entries.".to_owned()).into())
-        } else {
-            Ok(Box::new(Wal::new(node.id.to_string(), config)?))
-        }
+    let map = node.config.as_ref().ok_or_else(|| ErrorKind::MissingOpConfig(node.id.to_string()))?;
+    let config: Config = Config::new(&map)?;
 
+    if config.max_elements.or(config.max_bytes).is_none() {
+        Err(ErrorKind::BadOpConfig("WAL operator needs at least one of `max_elements` or `max_bytes` config entries.".to_string()).into())
     } else {
-        Err(ErrorKind::MissingOpConfig(node.id.to_string()).into())
+        Ok(Box::new(Wal::new(node.id.to_string(), config)?))
     }
 });
 
@@ -372,7 +342,7 @@ impl Operator for Wal {
                     insight.id.track(&e.id);
                 }
 
-                let current_confirmed = self.confirmed.map(u64::from).unwrap_or_default();
+                let current_confirmed = self.confirmed.map(|v| u64::from(&v)).unwrap_or_default();
                 if event_id < current_confirmed {
                     warn!(
                         "trying to fail a message({}) that was already confirmed({})",
@@ -750,4 +720,20 @@ mod test {
         }
         Ok(())
     }
+
+    #[test]
+    fn from() -> Result<()> {
+        assert_eq!(42, u64::from(&(Idx::from(40u64) + 2u64)));
+        assert_eq!(42, u64::from(&(Idx::from(40u64) + 2usize)));
+
+        Ok(())
+    }
+
+    #[test]
+    fn as_ref() -> Result<()> {
+        let i = Idx::from(42u64);
+        let s: &[u8] = i.as_ref();
+        assert_eq!(&[0, 0, 0, 0, 0, 0, 0, 42u8][..], s);
+        Ok(())
+    }
 }