New Value Network: nn-f004da0ebf25.network (#87)

New threat inputs. Both nets are now zstd compressed also.

Passed STC:
Elo: 48.06 ± 3.3 (95%) LOS: 100.0%
Total: 20000 W: 7332 L: 4583 D: 8085
Ptnml(0-2): 325, 1657, 3886, 3208, 924
nElo: 70.78 ± 5.0 (95%) PairsRatio: 2.08
https://tests.montychess.org/tests/view/6771c6231678cfe6b8265528

Passed VLTC SMP:
Elo: 35.00 ± 6.8 (95%) LOS: 100.0%
Total: 2560 W: 709 L: 452 D: 1399
Ptnml(0-2): 2, 201, 628, 436, 13
nElo: 69.47 ± 14.0 (95%) PairsRatio: 2.21
https://tests.montychess.org/tests/view/6771be701678cfe6b826551e

Removed redundant biases
Passed LTC:
Elo: 35.77 ± 4.0 (95%) LOS: 100.0%
Total: 10000 W: 3123 L: 2097 D: 4780
Ptnml(0-2): 72, 863, 2246, 1605, 214
nElo: 61.12 ± 7.0 (95%) PairsRatio: 1.95
https://tests.montychess.org/tests/view/677d1a4e8e3267ba3c2ca00b

Bench: 1733801
Co-authored-by: jw1912 <jamie.whiting.dev@gmail.com>

.gitignore

@@ -7,6 +7,7 @@
 *.pdb
 *.network
 *.epd
+*.zst
 checkpoints
 data
 nets

Cargo.toml

@@ -13,6 +13,7 @@ codegen-units = 1
 
 [features]
 embed = []
+raw = []
 datagen = []
 uci-minimal = []
 tunable = []
@@ -23,7 +24,9 @@ resolver = "2"
 
 [dependencies]
 memmap2 = "0.9.5"
+zstd = "0.13.2"
 
 [build-dependencies]
 sha2 = "0.10.8"
-chrono = "0.4.38"
+chrono = "0.4.38"
+zstd = { version = "0.13.2", features = ["zstdmt"] }

Makefile

@@ -12,6 +12,9 @@ endif
 
 default:
 	cargo rustc --release --bin monty --features=embed -- -C target-cpu=native --emit link=$(NAME)
+
+raw:
+	cargo rustc --release --bin monty --features=embed,raw -- -C target-cpu=native --emit link=$(NAME)
 
 montytest:
 	cargo +stable rustc --release --bin monty --features=uci-minimal,tunable -- -C target-cpu=native --emit link=$(NAME)
@@ -20,8 +23,4 @@ noembed:
 	cargo rustc --release --bin monty -- -C target-cpu=native --emit link=$(NAME)
 
 gen:
-	cargo rustc --release --package datagen --bin datagen -- -C target-cpu=native --emit link=$(NAME)
-
-release:
-	cargo rustc --release --bin monty --features=embed -- --emit link=$(OLD)
-	cargo rustc --release --bin monty --features=embed -- -C target-cpu=x86-64-v2 -C target-feature=+avx2 --emit link=$(AVX2)
+	cargo rustc --release --package datagen --bin datagen -- -C target-cpu=native --emit link=$(NAME)

build.rs

@@ -37,26 +37,66 @@ fn main() {
     // Get the build version name
     get_name();
 
-    // Extract the file names from the respective source files
-    let value_file_name = extract_network_name("src/networks/value.rs", "ValueFileDefaultName");
-    let policy_file_name = extract_network_name("src/networks/policy.rs", "PolicyFileDefaultName");
+    // Declare variables for file names
+    let value_file_name;
+    let policy_file_name;
+    let value_path;
+    let policy_path;
 
-    // Define fixed paths where the networks will be stored
-    let value_path = "value.network";
-    let policy_path = "policy.network";
+    // Extract the file names from the respective source files
+    #[cfg(feature = "raw")]
+    {
+        value_file_name = extract_network_name("src/networks/value.rs", "ValueFileDefaultName");
+        policy_file_name = extract_network_name("src/networks/policy.rs", "PolicyFileDefaultName");
+        value_path = "value.network";
+        policy_path = "policy.network";
+    }
+    #[cfg(not(feature = "raw"))]
+    {
+        value_file_name = extract_network_name("src/networks/value.rs", "CompressedValueName");
+        policy_file_name = extract_network_name("src/networks/policy.rs", "CompressedPolicyName");
+        value_path = "value.network.zst";
+        policy_path = "policy.network.zst";
+    }
 
     // Validate and download the network files if needed
-    validate_and_download_network(&value_file_name, &value_path);
-    validate_and_download_network(&policy_file_name, &policy_path);
+    validate_and_download_network(&value_file_name, &value_path, should_validate(&value_path));
+    validate_and_download_network(
+        &policy_file_name,
+        &policy_path,
+        should_validate(&policy_path),
+    );
+
+    #[cfg(feature = "raw")]
+    {
+        compress_with_zstd(&value_path);
+        compress_with_zstd(&policy_path);
+    }
 
     // Set up cargo instructions to track changes
     println!("cargo:rerun-if-changed=src/networks/value.rs");
     println!("cargo:rerun-if-changed=src/networks/policy.rs");
-    println!("cargo:rerun-if-changed={}", value_path);
-    println!("cargo:rerun-if-changed={}", policy_path);
 }
 
-#[cfg(not(feature = "embed"))]
+#[cfg(all(feature = "embed", feature = "raw"))]
+fn should_validate(dest_path: &str) -> bool {
+    let path = Path::new(dest_path);
+
+    // Construct the compressed path with `.zst` extension
+    let compressed_path = path.with_extension("zst");
+
+    // Check if both paths exist
+    path.exists() && compressed_path.exists()
+}
+
+#[cfg(all(feature = "embed", not(feature = "raw")))]
+fn should_validate(dest_path: &str) -> bool {
+    let path = Path::new(dest_path);
+
+    path.exists()
+}
+
+#[cfg(not(any(feature = "embed", feature = "raw")))]
 fn main() {
     // Get the build version name
     get_name();
@@ -86,14 +126,13 @@ fn extract_network_name(file_path: &str, const_name: &str) -> String {
 }
 
 #[cfg(feature = "embed")]
-fn validate_and_download_network(expected_name: &str, dest_path: &str) {
+fn validate_and_download_network(expected_name: &str, dest_path: &str, should_validate: bool) {
     let path = Path::new(dest_path);
-
     // Extract the expected SHA-256 prefix from the expected file name
     let expected_prefix = extract_sha_prefix(expected_name);
 
     // If the file exists, calculate its SHA-256 and check the first 12 characters
-    if path.exists() {
+    if should_validate {
         if let Ok(existing_sha) = calculate_sha256(path) {
             println!("Expected SHA-256 prefix: {}", expected_prefix);
             println!("Actual SHA-256: {}", &existing_sha[..12]);
@@ -120,6 +159,37 @@ fn validate_and_download_network(expected_name: &str, dest_path: &str) {
 
     // Download the correct network file
     download_network(expected_name, dest_path);
+    println!("cargo:rerun-if-changed={}", dest_path);
+}
+
+#[cfg(all(feature = "embed", feature = "raw"))]
+fn compress_with_zstd(input_path: &str) {
+    use std::fs::File;
+    use std::io::{copy, BufReader, BufWriter};
+    use zstd::Encoder;
+
+    let output_path = format!("{}.zst", input_path);
+
+    let input_file = File::open(input_path).expect("Failed to open input file");
+    let output_file = File::create(output_path.clone()).expect("Failed to create output file");
+
+    let mut reader = BufReader::new(input_file);
+    let writer = BufWriter::new(output_file);
+
+    // Initialize the Zstd encoder with compression level 22
+    let mut encoder = Encoder::new(writer, 22).expect("Failed to create Zstd encoder");
+    // Use 4 threads
+    encoder
+        .multithread(4)
+        .expect("Failed to set multithreaded compression");
+
+    // Copy the data from the reader and write to the encoder
+    copy(&mut reader, &mut encoder).expect("Failed to compress data");
+
+    // Finalize the compression process
+    encoder.finish().expect("Failed to finish compression");
+
+    println!("cargo:rerun-if-changed={}", output_path);
 }
 
 #[cfg(feature = "embed")]

src/chess.rs

@@ -1,6 +1,6 @@
 mod attacks;
 mod board;
-mod consts;
+pub mod consts;
 mod frc;
 mod moves;
 

src/chess/attacks.rs

@@ -103,16 +103,16 @@ impl Attacks {
     });
 }
 
-struct File;
+pub struct File;
 impl File {
-    const A: u64 = 0x0101_0101_0101_0101;
-    const H: u64 = Self::A << 7;
+    pub const A: u64 = 0x0101_0101_0101_0101;
+    pub const H: u64 = Self::A << 7;
 }
 
 const EAST: [u64; 64] = init!(|sq, 64| (0xFF << (sq & 56)) ^ (1 << sq) ^ WEST[sq]);
 const WEST: [u64; 64] = init!(|sq, 64| (0xFF << (sq & 56)) & ((1 << sq) - 1));
 const DIAG: u64 = DIAGS[7];
-const DIAGS: [u64; 15] = [
+pub const DIAGS: [u64; 15] = [
     0x0100_0000_0000_0000,
     0x0201_0000_0000_0000,
     0x0402_0100_0000_0000,

src/chess/consts.rs

@@ -1,5 +1,5 @@
-use super::attacks::line_through;
-use crate::init;
+use super::attacks::{line_through, File, DIAGS};
+use crate::{init, init_add_assign};
 
 pub struct Side;
 impl Side {
@@ -148,3 +148,60 @@ pub static ZVALS: ZobristVals = {
 pub const PHASE_VALS: [i32; 8] = [0, 0, 0, 1, 1, 2, 4, 0];
 
 pub const SEE_VALS: [i32; 8] = [0, 0, 100, 450, 450, 650, 1250, 0];
+
+pub struct ValueOffsets;
+impl ValueOffsets {
+    pub const PAWN: usize = 0;
+    pub const KNIGHT: usize = Self::PAWN + 6 * ValueIndices::PAWN;
+    pub const BISHOP: usize = Self::KNIGHT + 12 * ValueIndices::KNIGHT[64];
+    pub const ROOK: usize = Self::BISHOP + 10 * ValueIndices::BISHOP[64];
+    pub const QUEEN: usize = Self::ROOK + 10 * ValueIndices::ROOK[64];
+    pub const KING: usize = Self::QUEEN + 12 * ValueIndices::QUEEN[64];
+    pub const END: usize = Self::KING + 8 * ValueIndices::KING[64];
+}
+
+pub struct ValueIndices;
+impl ValueIndices {
+    pub const PAWN: usize = 84;
+    pub const KNIGHT: [usize; 65] =
+        init_add_assign!(|sq, 0, 64| ValueAttacks::KNIGHT[sq].count_ones() as usize);
+    pub const BISHOP: [usize; 65] =
+        init_add_assign!(|sq, 0, 64| ValueAttacks::BISHOP[sq].count_ones() as usize);
+    pub const ROOK: [usize; 65] =
+        init_add_assign!(|sq, 0, 64| ValueAttacks::ROOK[sq].count_ones() as usize);
+    pub const QUEEN: [usize; 65] =
+        init_add_assign!(|sq, 0, 64| ValueAttacks::QUEEN[sq].count_ones() as usize);
+    pub const KING: [usize; 65] =
+        init_add_assign!(|sq, 0, 64| ValueAttacks::KING[sq].count_ones() as usize);
+}
+
+pub struct ValueAttacks;
+impl ValueAttacks {
+    pub const KNIGHT: [u64; 64] = init!(|sq, 64| {
+        let n = 1 << sq;
+        let h1 = ((n >> 1) & 0x7f7f_7f7f_7f7f_7f7f) | ((n << 1) & 0xfefe_fefe_fefe_fefe);
+        let h2 = ((n >> 2) & 0x3f3f_3f3f_3f3f_3f3f) | ((n << 2) & 0xfcfc_fcfc_fcfc_fcfc);
+        (h1 << 16) | (h1 >> 16) | (h2 << 8) | (h2 >> 8)
+    });
+
+    pub const BISHOP: [u64; 64] = init!(|sq, 64| {
+        let rank = sq / 8;
+        let file = sq % 8;
+        DIAGS[file + rank].swap_bytes() ^ DIAGS[7 + file - rank]
+    });
+
+    pub const ROOK: [u64; 64] = init!(|sq, 64| {
+        let rank = sq / 8;
+        let file = sq % 8;
+        (0xFF << (rank * 8)) ^ (File::A << file)
+    });
+
+    pub const QUEEN: [u64; 64] = init!(|sq, 64| Self::BISHOP[sq] | Self::ROOK[sq]);
+
+    pub const KING: [u64; 64] = init!(|sq, 64| {
+        let mut k = 1 << sq;
+        k |= (k << 8) | (k >> 8);
+        k |= ((k & !File::A) >> 1) | ((k & !File::H) << 1);
+        k ^ (1 << sq)
+    });
+}