From cc269b95a690facab7c56f5364fb735c84351fb6 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Fri, 4 Feb 2022 18:22:29 +0530 Subject: [PATCH 01/30] Re added missing cache_hit metric --- src/rpc/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 80b9e97..9cad1c8 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -78,6 +78,7 @@ pub async fn fetch( let (moderation_status, categories, document) = match db_results.get(0) { Some(result) => { + metrics::MODERATION.with_label_values(&["cache_hit"]).inc(); info!( "Found cached results for id={}, blocked={}, categories:{:?}, provider:{:?}", req_id, result.blocked, result.categories, result.provider From 3e0eb3a35e77e6dfe769604a1e6eb3bc3f97852b Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Wed, 9 Feb 2022 01:23:02 +0530 Subject: [PATCH 02/30] Added dev deps, updated others --- Cargo.lock | 195 +++++++++++++++++++++++++++-------------------------- Cargo.toml | 12 ++-- 2 files changed, 106 insertions(+), 101 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8cf954..9916582 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,15 +48,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33df8f310ad3e5937d55b9e92e1404241b4f91b7c1da7822b89fde04caaacd2c" +checksum = "12b964849038df43a2b4e0c20e29b67451af5a93108d757dd58b9e82f41a0ee8" dependencies = [ "aws-http", "aws-sdk-sso", @@ -81,9 +81,9 @@ dependencies = [ [[package]] name = "aws-endpoint" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4862c0314a90e9e6bdbc9625236aeebd194e66f98066d1f1e3f2f32b867f9ecd" +checksum = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6" dependencies = [ "aws-smithy-http", "aws-types", @@ -94,9 +94,9 @@ dependencies = [ [[package]] name = "aws-http" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae844a9180de89ae4dcf3ea8f21230b84f181eed52335123f0e3a435da21d0f" +checksum = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -109,9 +109,9 @@ dependencies = [ [[package]] name = "aws-sdk-rekognition" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e57587b7ac1fe83c958cb1d362a3a570f42d0338df17a21f827c6507337f05d" +checksum = "7a7b64ac36eb697ce5733651262f3e0a78c460b0e2cd0e695148049248a51619" dependencies = [ "aws-endpoint", "aws-http", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce2416ba8a1a22c20777a59327780ed9095f102ed0625d3560008f2c6ab9daf" +checksum = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9" dependencies = [ "aws-endpoint", "aws-http", @@ -156,9 +156,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d34588414f5077e48761d2977dfd2a66a1d2df80034cff50010893b4fa584f" +checksum = "222fcabbf95f1f13c4e28cab95c9a4bb02606f998b1ea2800713b6866be5701d" dependencies = [ "aws-endpoint", "aws-http", @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c34df5bea9c58505f7cb4346dc56842f0300f5244827e078f359302b223386" +checksum = "d85b9f081af2c73ee25642de1a35fa9ba7b2432e54f6bf42242e478ae53c3beb" dependencies = [ "aws-endpoint", "aws-http", @@ -200,9 +200,9 @@ dependencies = [ [[package]] name = "aws-sig-auth" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dba38995b2c5531e56877e0771746545ac6cd3893261d8cd5753b69bd39ceff2" +checksum = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc" dependencies = [ "aws-sigv4", "aws-smithy-eventstream", @@ -215,9 +215,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d23b934ee5886df8c49c47ca0a65230d2407eaa71c2a883cbdf87c5d25c458cd" +checksum = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c" dependencies = [ "aws-smithy-eventstream", "aws-smithy-http", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.35.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5fcbffea194e3b5c20471f5dc12d042a9edee49aadbb0efb25315b6dc9dd5d" +checksum = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e" dependencies = [ "futures-util", "pin-project-lite", @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "aws-smithy-client" -version = "0.35.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4120423bf4bfe09332eb9c83300d2c20a7ce58f1ace34fadbc87fae4db2c6b3" +checksum = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -271,9 +271,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.35.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6edce1459b6cc8ca0bb8bbf93dbc24f3d89aa9ccc21f73233d83c2bee1756caa" +checksum = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9" dependencies = [ "aws-smithy-types", "bytes", @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.35.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0cd7cf4c3eab32ccbaab8a3c2a128d5fee49c59414b8f0e1be66380ab5870a" +checksum = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696" dependencies = [ "aws-smithy-eventstream", "aws-smithy-types", @@ -303,9 +303,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-tower" -version = "0.35.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af28d1e5455d9362208e12aa17221a8c27dd430e85578a0a27964c1f1eed42c0" +checksum = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d" dependencies = [ "aws-smithy-http", "bytes", @@ -318,18 +318,18 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.35.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a73ad42c7528114053c4ace79dc2b560cb3fbeb4c677246da066bd639fb61e40" +checksum = "3cc19c372b0a561aa6bfc5dfdd917da7c7b1641d3bc9049ca4d7b197bb616a09" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.35.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e66f2ced4b96990a00bd774aa6fd02bd485d1fce081219637e2907c588f9dee4" +checksum = "8254e49a237e9dc0301a4683c424a825f4220420b241ec4eb51e959a70626d8a" dependencies = [ "aws-smithy-types", "urlencoding", @@ -337,9 +337,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "0.35.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193abb2559d65d6eaeacc45dd3764cb8f821a90425f6b051a8fd17ea12cbd0d1" +checksum = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f" dependencies = [ "itoa 1.0.1", "num-integer", @@ -349,9 +349,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.35.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e222a20a9a91c396fe1689faae16499fa857de08071a8a86d2b734319eac405" +checksum = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4" dependencies = [ "thiserror", "xmlparser", @@ -359,9 +359,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcd5d5fbab40e704ae3d0f3c602131f4c691475617270bf793cf1c47d9d24e41" +checksum = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -418,9 +418,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d36a02058e76b040de25a4464ba1c80935655595b661505c8b39b664828b95" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" dependencies = [ "generic-array", ] @@ -517,9 +517,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "core-foundation" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ "core-foundation-sys", "libc", @@ -542,9 +542,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2209c310e29876f7f0b2721e7e26b84aff178aa3da5d091f9bfbf47669e60e3" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ "cfg-if", ] @@ -572,9 +572,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762" +checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" dependencies = [ "cfg-if", "crossbeam-utils", @@ -585,9 +585,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcae03edb34f947e64acdb1c33ec169824e20657e9ecb61cef6c8c74dcb8120" +checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" dependencies = [ "cfg-if", "lazy_static", @@ -647,7 +647,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b697d66081d42af4fba142d56918a3cb21dc8eb63372c6b85d14f44fb9c5979b" dependencies = [ - "block-buffer 0.10.0", + "block-buffer 0.10.2", "crypto-common", "generic-array", "subtle", @@ -810,9 +810,9 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" [[package]] name = "futures" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28560757fe2bb34e79f907794bb6b22ae8b0e5c669b638a1132f2592b19035b4" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" dependencies = [ "futures-channel", "futures-core", @@ -825,9 +825,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3dda0b6588335f360afc675d0564c17a77a2bda81ca178a4b6081bd86c7f0b" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" dependencies = [ "futures-core", "futures-sink", @@ -835,15 +835,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" [[package]] name = "futures-executor" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29d6d2ff5bb10fb95c85b8ce46538a2e5f5e7fdc755623a7d4529ab8a4ed9d2a" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" dependencies = [ "futures-core", "futures-task", @@ -852,15 +852,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9d34af5a1aac6fb380f735fe510746c38067c5bf16c7fd250280503c971b2" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" [[package]] name = "futures-macro" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbd947adfffb0efc70599b3ddcf7b5597bb5fa9e245eb99f62b3a5f7bb8bd3c" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ "proc-macro2", "quote", @@ -869,21 +869,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3055baccb68d74ff6480350f8d6eb8fcfa3aa11bdc1a1ae3afdd0514617d508" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" [[package]] name = "futures-task" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee7c6485c30167ce4dfb83ac568a849fe53274c831081476ee13e0dce1aad72" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" [[package]] name = "futures-util" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b5cf40b47a271f77a8b1bec03ca09044d99d2372c0de244e66430761127164" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ "futures-channel", "futures-core", @@ -943,9 +943,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9de88456263e249e241fcd211d3954e2c9b0ef7ccfc235a444eb367cae3689" +checksum = "d9f1f717ddc7b2ba36df7e871fd88db79326551d3d6f1fc406fbfd28b582ff8e" dependencies = [ "bytes", "fnv", @@ -1262,9 +1262,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0005d08a8f7b65fb8073cb697aa0b12b631ed251ce73d862ce50eeb52ce3b50" +checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c" [[package]] name = "libgit2-sys" @@ -1298,9 +1298,9 @@ checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "lock_api" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" dependencies = [ "scopeguard", ] @@ -1476,6 +1476,7 @@ dependencies = [ "log", "log4rs", "prometheus", + "rand", "rust-embed", "serde", "serde_json", @@ -1796,9 +1797,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.25.2" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47c327e191621a2158159df97cdbc2e7074bb4e940275e35abf38eb3d2595754" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" [[package]] name = "quote" @@ -2128,9 +2129,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09d3c15d814eda1d6a836f2f2b56a6abc1446c8a34351cb3180d3db92ffe4ce" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" dependencies = [ "bitflags", "core-foundation", @@ -2141,9 +2142,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e90dd10c41c6bfc633da6e0c659bd25d31e0791e5974ac42970267d59eba87f7" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" dependencies = [ "core-foundation-sys", "libc", @@ -2151,9 +2152,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" +checksum = "0486718e92ec9a68fbed73bb5ef687d71103b142595b406835649bebd33f72c7" dependencies = [ "serde", ] @@ -2201,9 +2202,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd186dd4e1748b2798a2e86789dd77f5834ecda0bf15db76962e8e104bfc9bd" +checksum = "d7868ad3b8196a8a0aea99a8220b124278ee5320a55e4fde97794b6f85b1a377" dependencies = [ "serde", ] @@ -2285,9 +2286,9 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "socket2" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f82496b90c36d70af5fcd482edaa2e0bd16fade569de1330405fecbbdac736b" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" dependencies = [ "libc", "winapi", @@ -2419,9 +2420,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.15.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" +checksum = "0c27a64b625de6d309e8c57716ba93021dccf1b3b5c97edd6d3dd2d2135afc0a" dependencies = [ "bytes", "libc", @@ -2576,9 +2577,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +checksum = "2d8d93354fe2a8e50d5953f5ae2e47a3fc2ef03292e7ea46e3cc38f549525fb9" dependencies = [ "cfg-if", "log", @@ -2589,9 +2590,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" +checksum = "8276d9a4a3a558d7b7ad5303ad50b53d58264641b82914b7ada36bd762e7a716" dependencies = [ "proc-macro2", "quote", @@ -2600,9 +2601,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +checksum = "03cfcb51380632a72d3111cb8d3447a8d908e577d31beeac006f836383d29a23" dependencies = [ "lazy_static", ] @@ -2902,6 +2903,6 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc222aec311c323c717f56060324f32b82da1ce1dd81d9a09aa6a9030bfe08db" +checksum = "7c88870063c39ee00ec285a2f8d6a966e5b6fb2becc4e8dac77ed0d370ed6006" diff --git a/Cargo.toml b/Cargo.toml index 9d1fb15..cecb9b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ version = "1.4.0" authors = ["Cryptonomic Inc."] edition = "2018" build = "build.rs" +exclude = ["/docker", "/docs", "/lib", "/log", "proxy.conf", "/*.yml", "/.github", "Dockerfile", "LICENSE", "README.md", ".gitignore", ".dockerignore"] [build-dependencies] built = { version = "0.5", features = ["git2"] } @@ -36,14 +37,17 @@ rust-embed="6.3" jemallocator = { version = "0.3", optional = true } # deps for aws -aws-config = "0.5.2" -aws-sdk-rekognition = "0.5.2" -aws-sdk-s3 = "0.5.2" -aws-types = "0.5.2" +aws-config = "0.6.0" +aws-sdk-rekognition = "0.6.0" +aws-sdk-s3 = "0.6.0" +aws-types = "0.6.0" # Overrides to address vulns time = "0.3" +[dev-dependencies] +rand = "0.8.4" + [profile.release] lto = true From 74a16c8af733476296baed82941da418d28f8fa6 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Wed, 9 Feb 2022 01:26:42 +0530 Subject: [PATCH 03/30] Updated deps --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9916582..a693575 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1033,9 +1033,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" +checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" [[package]] name = "httpdate" From 6487b6f39540c4afea8e43550d889137687fa18e Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Wed, 9 Feb 2022 01:53:31 +0530 Subject: [PATCH 04/30] Improved clarity of log message --- src/rpc/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 9cad1c8..3f9188a 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -29,7 +29,7 @@ async fn fetch_document( ) -> Result, Errors> { let cache_key = sha256(url.as_bytes()); let cached_doc = ctx.cache.as_ref().and_then(|cache| { - debug!("Fetched document from cache, url:{}", url); + debug!("Checking document cache for document, url:{}", url); cache.get(&cache_key) }); @@ -80,7 +80,7 @@ pub async fn fetch( Some(result) => { metrics::MODERATION.with_label_values(&["cache_hit"]).inc(); info!( - "Found cached results for id={}, blocked={}, categories:{:?}, provider:{:?}", + "Found cached moderation results for id={}, blocked={}, categories:{:?}, provider:{:?}", req_id, result.blocked, result.categories, result.provider ); let document = if !result.blocked || params.force { @@ -92,7 +92,7 @@ pub async fn fetch( } None => { metrics::MODERATION.with_label_values(&["cache_miss"]).inc(); - info!("No cached results found for id={}", req_id); + info!("No cached moderation results found for id={}", req_id); let document = fetch_document(ctx.clone(), req_id, ¶ms.url).await?; let max_document_size = ctx.moderation_provider.max_document_size(); let supported_types = ctx.moderation_provider.supported_types(); @@ -100,10 +100,12 @@ pub async fn fetch( metrics::MODERATION.with_label_values(&["requests"]).inc(); + info!("Submitting moderation request for id:{}", req_id); // Resize the image if required or reformat to png if required let mod_response = if document.bytes.len() as u64 >= max_document_size || !supported_types.contains(&document_type) { + info!("Image resizing required, id={}", req_id); let resized_doc = document.resize_image(max_document_size)?; ctx.moderation_provider.moderate(&resized_doc).await? } else { From 3cf581c8adcb1ee7e272e0e45171a7f844584a18 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Wed, 9 Feb 2022 01:54:46 +0530 Subject: [PATCH 05/30] Added new error variants for image resizing ops --- src/rpc/error.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rpc/error.rs b/src/rpc/error.rs index b89f9c7..2b63806 100644 --- a/src/rpc/error.rs +++ b/src/rpc/error.rs @@ -26,6 +26,7 @@ pub enum Errors { InvalidUri, InvalidOrBlockedHost, TimedOut, + ImageResizeError, } impl Errors { @@ -48,6 +49,7 @@ impl Errors { 111, "Connection/Request/Response from the destination timed out".to_string(), ), + Errors::ImageResizeError => (112, "Image Resize Error".to_string()), }; RpcError { From dd348668964f0f6b214b316b61309108ed3011cc Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Wed, 9 Feb 2022 01:55:38 +0530 Subject: [PATCH 06/30] Refactored resizing code to use better hueristics. Added unit tests. --- src/document.rs | 261 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 218 insertions(+), 43 deletions(-) diff --git a/src/document.rs b/src/document.rs index 2e1f74b..5763aae 100644 --- a/src/document.rs +++ b/src/document.rs @@ -1,21 +1,21 @@ extern crate base64; extern crate hyper; -use std::io::Cursor; - use crate::cache::ByteSizeable; -use crate::moderation::SupportedMimeTypes; use crate::rpc::error::Errors; +use image::io::Reader as ImageReader; +use std::cmp::max; +use std::io::Cursor; use base64::encode; use hyper::body::Bytes; -use image::DynamicImage; -use image::ImageFormat; -use image::{self, GenericImageView}; -use log::{debug, error}; +use image::{DynamicImage, GenericImageView, ImageFormat, ImageOutputFormat}; +use log::{error, info, warn}; use uuid::Uuid; +const MINIMUM_IMAGE_DIMENSION: u32 = 1024_u32; + #[derive(Clone)] pub struct Document { pub id: Uuid, @@ -32,52 +32,227 @@ impl ByteSizeable for Document { } impl Document { - fn load_image(&self, image_type: SupportedMimeTypes) -> Result { - let cursor = Cursor::new(&self.bytes); - let img = match image_type { - SupportedMimeTypes::ImageBmp => image::load(cursor, ImageFormat::Bmp), - SupportedMimeTypes::ImageGif => image::load(cursor, ImageFormat::Gif), - SupportedMimeTypes::ImageJpeg => image::load(cursor, ImageFormat::Jpeg), - SupportedMimeTypes::ImagePng => image::load(cursor, ImageFormat::Png), - SupportedMimeTypes::ImageTiff => image::load(cursor, ImageFormat::Tiff), - SupportedMimeTypes::Unsupported => image::load(cursor, ImageFormat::Jpeg), //TODO - }; - img.map_err(|e| { - error!("Unable to open image, reason={}", e); - Errors::InternalError - }) + fn load_image(&self) -> Result { + ImageReader::new(Cursor::new(&self.bytes)) + .with_guessed_format() + .map_err(|e| { + error!("Unable to open image, id={}, reason={}", self.id, e); + Errors::ImageResizeError + }) + .and_then(|r| { + r.decode().map_err(|e| { + error!("Unable to open image, id={}, reason={}", self.id, e); + Errors::ImageResizeError + }) + }) } - pub fn resize_image(&self, max_size: u64) -> Result { - let image_type = SupportedMimeTypes::from_string(&self.content_type); - let img = self.load_image(image_type)?; + fn resize_parameters(x_dim: u32, y_dim: u32) -> (u32, u32) { + let aspect_ratio = x_dim as f64 / y_dim as f64; + + if x_dim <= MINIMUM_IMAGE_DIMENSION || y_dim <= MINIMUM_IMAGE_DIMENSION { + return (max(x_dim / 2_u32, 1_u32), max(y_dim / 2_u32, 1_u32)); + } + + if aspect_ratio > 1_f64 { + let new_x_dim = aspect_ratio * (MINIMUM_IMAGE_DIMENSION as f64); + (new_x_dim as u32, MINIMUM_IMAGE_DIMENSION) + } else { + let new_y_dim = aspect_ratio * (MINIMUM_IMAGE_DIMENSION as f64); + (MINIMUM_IMAGE_DIMENSION, new_y_dim as u32) + } + } + + fn resize(&self, img: DynamicImage, max_size: u64) -> Result, Errors> { + let dim_floor = 32_u32; let (x_dim, y_dim) = img.dimensions(); - let scale = self.bytes.len() as f64 / max_size as f64; - let scale_factor: u32 = 2_u32.pow(scale.max(0_f64) as u32); - debug!("Image resize: scale={}, factor={}", scale, scale_factor); - let (x_dim_new, y_dim_new) = (x_dim / scale_factor, y_dim / scale_factor); - debug!( - "Image resize: New dimensions x={}, y={}", - x_dim_new, y_dim_new + let (new_x_dim, new_y_dim) = Self::resize_parameters(x_dim, y_dim); + info!( + "Image resizing, id={}, x={}, y={}, new_x={}, new_y={}", + self.id, x_dim, y_dim, new_x_dim, new_y_dim ); - let new_img = img.resize(x_dim_new, y_dim_new, image::imageops::FilterType::Nearest); //TODO this is expensive - let mut bytes: Vec = Vec::new(); - match new_img.write_to(&mut bytes, image::ImageOutputFormat::Png) { - Ok(_) => Ok(Document { - id: self.id, - content_length: bytes.len() as u64, - content_type: String::from("image/png"), - bytes: Bytes::copy_from_slice(bytes.as_slice()), - url: self.url.clone(), - }), + let new_img = img.resize(new_x_dim, new_y_dim, image::imageops::FilterType::Nearest); + let mut cursor = Cursor::new(Vec::new()); + match new_img.write_to(&mut cursor, ImageOutputFormat::Png) { + Ok(_) => { + let bytes = cursor.into_inner(); + info!( + "Image resizing result, id={}, len={}, new_len={}", + self.id, + self.bytes.len(), + bytes.len() + ); + if bytes.len() as u64 > max_size { + warn!("Resizing did not reduce image size enough to fit max moderation size, id={}, max_size={}", self.id, max_size); + if new_x_dim < dim_floor || new_y_dim < dim_floor { + warn!("Image dimension(s) is smaller than {} pixels but file size is greater than max moderation size, id={}, max_size={}", dim_floor, self.id, max_size ); + Ok(bytes) + } else { + self.resize(new_img, max_size) + } + } else { + Ok(bytes) + } + } Err(e) => { - error!("Error writing out image to buffer, reason={}", e); - Err(Errors::InternalError) + error!( + "Error writing out image to buffer, id={}, reason={}", + self.id, e + ); + Err(Errors::ImageResizeError) } } } + pub fn resize_image(&self, max_size: u64) -> Result { + if (self.bytes.len() as u64) < max_size { + warn!("Image resize is not required as size is below max_size, id={}, size={}, max_size={}", self.id, self.bytes.len(), max_size); + return Ok(self.clone()); + } + info!( + "Image info, id={}, len={}, type={}", + self.id, + self.bytes.len(), + self.content_type + ); + + let img = self.load_image()?; + let bytes = self.resize(img, max_size)?; + Ok(Document { + id: self.id, + content_length: bytes.len() as u64, + content_type: String::from("image/png"), + bytes: Bytes::copy_from_slice(bytes.as_slice()), + url: self.url.clone(), + }) + } + pub fn to_url(&self) -> String { format!("data:{};base64,{}", self.content_type, encode(&self.bytes)) } } + +#[cfg(test)] +mod tests { + use super::*; + use image::{GenericImage, ImageOutputFormat, Rgba}; + use rand::Rng; + + const X_SIZE: u32 = 1600; + const Y_SIZE: u32 = 1400; + + fn construct_image(x_dim: u32, y_dim: u32) -> Vec { + let mut new_image = DynamicImage::new_rgba8(x_dim, y_dim); + let mut rng = rand::thread_rng(); + + for x in 0..x_dim - 1 { + for y in 0..y_dim - 1 { + let rgb: Rgba = Rgba([ + rng.gen_range(0..255), + rng.gen_range(0..255), + rng.gen_range(0..255), + 255, + ]); + new_image.put_pixel(x, y, rgb); + } + } + let mut cursor = Cursor::new(Vec::new()); + + new_image + .write_to(&mut cursor, ImageOutputFormat::Png) + .unwrap(); + + cursor.into_inner() + } + + fn construct_document(image_bytes: &[u8]) -> Document { + let len = image_bytes.len() as u64; + Document { + id: Uuid::new_v4(), + content_type: "image/png".to_string(), + content_length: len, + bytes: Bytes::copy_from_slice(image_bytes), + url: "http://localhost.com/test.png".to_string(), + } + } + + #[test] + fn test_to_url() { + let bytes = "hello world".as_bytes(); + let document = construct_document(bytes); + let encoded = document.to_url(); + assert_eq!(encoded.as_str(), "data:image/png;base64,aGVsbG8gd29ybGQ="); + } + + #[test] + fn test_resize_parameters() { + let x_dim = 2048; + let y_dim = 1500; + let (x, y) = Document::resize_parameters(x_dim, y_dim); + assert!(x < x_dim); + assert_eq!(y, MINIMUM_IMAGE_DIMENSION); + + let x_dim = 1500; + let y_dim = 2048; + let (x, y) = Document::resize_parameters(x_dim, y_dim); + assert!(y < y_dim); + assert_eq!(x, MINIMUM_IMAGE_DIMENSION); + + let x_dim = 512; + let y_dim = 20480; + let (x, y) = Document::resize_parameters(x_dim, y_dim); + assert_eq!(x, x_dim / 2); + assert_eq!(y, y_dim / 2); + + let x_dim = 20480; + let y_dim = 512; + let (x, y) = Document::resize_parameters(x_dim, y_dim); + assert_eq!(x, x_dim / 2); + assert_eq!(y, y_dim / 2); + + let x_dim = 512; + let y_dim = 1; + let (x, y) = Document::resize_parameters(x_dim, y_dim); + assert_eq!(x, x_dim / 2); + assert_eq!(y, 1); + + let x_dim = 1; + let y_dim = 512; + let (x, y) = Document::resize_parameters(x_dim, y_dim); + assert_eq!(x, 1); + assert_eq!(y, y_dim / 2); + } + + #[test] + fn test_image_functions() { + let image_bytes = construct_image(X_SIZE, Y_SIZE); + assert!(!image_bytes.is_empty()); + let document = construct_document(image_bytes.as_slice()); + + // Check if image can be loaded + let loaded_image = document.load_image(); + assert!(loaded_image.is_ok()); + let dimensions = loaded_image.unwrap().dimensions(); + assert_eq!(dimensions, (X_SIZE, Y_SIZE)); + + // Check resize logic + let max_size_5mb = 1024_u64 * 1024_u64 * 5_u64; + let max_size_100mb = max_size_5mb * 20_u64; + + // No resize required + let new_document = document.resize_image(max_size_100mb); + assert!(new_document.is_ok()); + assert_eq!(document.bytes.len(), new_document.unwrap().bytes.len()); + + // Resize required + let new_document = document.resize_image(max_size_5mb); + assert!(new_document.is_ok()); + let new_document = new_document.unwrap(); + assert!(new_document.bytes.len() < document.bytes.len()); + let loaded_image = new_document.load_image(); + assert!(loaded_image.is_ok()); + let dimensions = loaded_image.unwrap().dimensions(); + //TODO: Recheck why after img.resize is the y dimension of the image is off by -1 + assert_eq!(dimensions.1, MINIMUM_IMAGE_DIMENSION - 1); + } +} From 238fc3d358db11baaccdf04e89217a6b4aec0345 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Wed, 9 Feb 2022 02:09:26 +0530 Subject: [PATCH 07/30] Removed unused import --- src/document.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/document.rs b/src/document.rs index 5763aae..e02c241 100644 --- a/src/document.rs +++ b/src/document.rs @@ -10,7 +10,7 @@ use std::io::Cursor; use base64::encode; use hyper::body::Bytes; -use image::{DynamicImage, GenericImageView, ImageFormat, ImageOutputFormat}; +use image::{DynamicImage, GenericImageView, ImageOutputFormat}; use log::{error, info, warn}; use uuid::Uuid; From 029e464b90268080e0135080d9ff01ed0bd1c427 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Wed, 9 Feb 2022 02:56:05 +0530 Subject: [PATCH 08/30] Switched to tikv jemallocator --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- Cargo.toml | 9 +++++---- src/main.rs | 7 ++++--- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a693575..1d54427 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1206,27 +1206,6 @@ dependencies = [ "regex", ] -[[package]] -name = "jemalloc-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" -dependencies = [ - "cc", - "fs_extra", - "libc", -] - -[[package]] -name = "jemallocator" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" -dependencies = [ - "jemalloc-sys", - "libc", -] - [[package]] name = "jobserver" version = "0.1.24" @@ -1471,7 +1450,6 @@ dependencies = [ "hyper-timeout", "hyper-tls", "image", - "jemallocator", "lazy_static", "log", "log4rs", @@ -1481,6 +1459,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.1", + "tikv-jemallocator", "time 0.3.7", "tokio", "tokio-postgres", @@ -2383,6 +2362,27 @@ dependencies = [ "weezl", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.4.2+5.2.1-patched.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5844e429d797c62945a566f8da4e24c7fe3fbd5d6617fd8bf7a0b7dc1ee0f22e" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c14a5a604eb8715bc5785018a37d00739b180bcf609916ddf4393d33d49ccdf" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.1.43" diff --git a/Cargo.toml b/Cargo.toml index cecb9b0..7f64f9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,6 @@ image = "0.23" dns-lookup = "1" anyhow = "1.0" rust-embed="6.3" -jemallocator = { version = "0.3", optional = true } # deps for aws aws-config = "0.6.0" @@ -45,12 +44,14 @@ aws-types = "0.6.0" # Overrides to address vulns time = "0.3" +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemallocator = "0.4" + + [dev-dependencies] rand = "0.8.4" [profile.release] lto = true -[features] -default = ["jemalloc"] -jemalloc = ["jemallocator"] \ No newline at end of file +[features] \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 1b26923..ce1acb9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,8 +21,8 @@ use std::{ sync::Arc, }; -#[cfg(feature = "jemalloc")] -use jemallocator::Jemalloc; +#[cfg(not(target_env = "msvc"))] +use tikv_jemallocator::Jemalloc; use hyper::{ service::{make_service_fn, service_fn}, @@ -42,7 +42,8 @@ pub mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } -#[cfg(feature = "jemalloc")] + +#[cfg(not(target_env = "msvc"))] #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; From 4c0230748182a7581003a1626caf8fa697d26921 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Wed, 9 Feb 2022 04:23:17 +0530 Subject: [PATCH 09/30] Added env variable for configuring jemallocator --- .cargo/config.toml | 3 +++ .dockerignore | 3 ++- Cargo.toml | 2 +- Dockerfile | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..4268328 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +[env] +# Remember to change the below variable in `Dockerfile` +_RJEM_MALLOC_CONF="background_thread:true,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true" \ No newline at end of file diff --git a/.dockerignore b/.dockerignore index 20e7121..68e052c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,4 +15,5 @@ lib/* **/dist **/build .github -**/*.md \ No newline at end of file +**/*.md +.cargo \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 7f64f9f..69a83d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ aws-types = "0.6.0" time = "0.3" [target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemallocator = "0.4" +tikv-jemallocator = { version="0.4", features=[ "background_threads_runtime_support", "background_threads"] } [dev-dependencies] diff --git a/Dockerfile b/Dockerfile index 6926acd..b5cfd84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,4 +23,6 @@ RUN apt-get update && apt-get upgrade -y && apt-get install ca-certificates -y & COPY --from=builder /opt/img_proxy/target/release/nft_image_proxy /opt/img_proxy/nft_image_proxy RUN mkdir -p /opt/img_proxy/sql COPY sql/ /opt/img_proxy/sql +# Remember to change the below variable in `.cargo/config.toml` +ENV _RJEM_MALLOC_CONF="background_thread:true,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true" CMD sleep 5 && /opt/img_proxy/nft_image_proxy From bd304106f6480e15acc7c944d75ac4f5fee952b6 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Thu, 10 Feb 2022 15:23:34 +0530 Subject: [PATCH 10/30] Updated project version --- Cargo.lock | 17 ++++++++--------- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d54427..c6384e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -595,9 +595,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d6b536309245c849479fba3da410962a43ed8e51c26b729208ec0ac2798d0" +checksum = "a4600d695eb3f6ce1cd44e6e291adceb2cc3ab12f20a33777ecd0bf6eba34e06" dependencies = [ "generic-array", ] @@ -643,13 +643,12 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b697d66081d42af4fba142d56918a3cb21dc8eb63372c6b85d14f44fb9c5979b" +checksum = "8cb780dce4f9a8f5c087362b3a4595936b2019e7c8b30f2c3e9a7e94e6ae9837" dependencies = [ "block-buffer 0.10.2", "crypto-common", - "generic-array", "subtle", ] @@ -987,7 +986,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddca131f3e7f2ce2df364b57949a9d47915cfbd35e46cfee355ccebbf794d6a2" dependencies = [ - "digest 0.10.1", + "digest 0.10.2", ] [[package]] @@ -1339,7 +1338,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6a38fc55c8bbc10058782919516f88826e70320db6d206aebc49611d24216ae" dependencies = [ - "digest 0.10.1", + "digest 0.10.2", ] [[package]] @@ -1430,7 +1429,7 @@ dependencies = [ [[package]] name = "nft_image_proxy" -version = "1.4.0" +version = "1.4.1" dependencies = [ "anyhow", "async-trait", @@ -2233,7 +2232,7 @@ checksum = "99c3bd8169c58782adad9290a9af5939994036b76187f7b4f0e6de91dbbfc0ec" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.1", + "digest 0.10.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 69a83d3..4434170 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nft_image_proxy" -version = "1.4.0" +version = "1.4.1" authors = ["Cryptonomic Inc."] edition = "2018" build = "build.rs" From ebc681b6d75fabfcc013674d2f9199328ec9d5e5 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 08:29:21 +0530 Subject: [PATCH 11/30] Updated minimum image size to 128 --- src/document.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/document.rs b/src/document.rs index e02c241..0b712c1 100644 --- a/src/document.rs +++ b/src/document.rs @@ -64,7 +64,7 @@ impl Document { } fn resize(&self, img: DynamicImage, max_size: u64) -> Result, Errors> { - let dim_floor = 32_u32; + let dim_floor = 128_u32; let (x_dim, y_dim) = img.dimensions(); let (new_x_dim, new_y_dim) = Self::resize_parameters(x_dim, y_dim); info!( From afc8d0ce1dacec4fe9bfd1108b5014ccfae58c81 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 09:06:34 +0530 Subject: [PATCH 12/30] Renaming variables --- src/document.rs | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/document.rs b/src/document.rs index 0b712c1..2c48e20 100644 --- a/src/document.rs +++ b/src/document.rs @@ -14,7 +14,12 @@ use image::{DynamicImage, GenericImageView, ImageOutputFormat}; use log::{error, info, warn}; use uuid::Uuid; -const MINIMUM_IMAGE_DIMENSION: u32 = 1024_u32; +// The X or Y resolution (depending on aspect ration) that is +// used as a first attempt target by image resizing +const NOMINAL_IMAGE_DIMENSION: u32 = 1024_u32; + +// The minimum dimension for either X or Y +const MINIMUM_IMAGE_DIMENSION: u32 = 128_u32; #[derive(Clone)] pub struct Document { @@ -50,21 +55,20 @@ impl Document { fn resize_parameters(x_dim: u32, y_dim: u32) -> (u32, u32) { let aspect_ratio = x_dim as f64 / y_dim as f64; - if x_dim <= MINIMUM_IMAGE_DIMENSION || y_dim <= MINIMUM_IMAGE_DIMENSION { + if x_dim <= NOMINAL_IMAGE_DIMENSION || y_dim <= NOMINAL_IMAGE_DIMENSION { return (max(x_dim / 2_u32, 1_u32), max(y_dim / 2_u32, 1_u32)); } if aspect_ratio > 1_f64 { - let new_x_dim = aspect_ratio * (MINIMUM_IMAGE_DIMENSION as f64); - (new_x_dim as u32, MINIMUM_IMAGE_DIMENSION) + let new_x_dim = aspect_ratio * (NOMINAL_IMAGE_DIMENSION as f64); + (new_x_dim as u32, NOMINAL_IMAGE_DIMENSION) } else { - let new_y_dim = aspect_ratio * (MINIMUM_IMAGE_DIMENSION as f64); - (MINIMUM_IMAGE_DIMENSION, new_y_dim as u32) + let new_y_dim = aspect_ratio * (NOMINAL_IMAGE_DIMENSION as f64); + (NOMINAL_IMAGE_DIMENSION, new_y_dim as u32) } } fn resize(&self, img: DynamicImage, max_size: u64) -> Result, Errors> { - let dim_floor = 128_u32; let (x_dim, y_dim) = img.dimensions(); let (new_x_dim, new_y_dim) = Self::resize_parameters(x_dim, y_dim); info!( @@ -84,8 +88,8 @@ impl Document { ); if bytes.len() as u64 > max_size { warn!("Resizing did not reduce image size enough to fit max moderation size, id={}, max_size={}", self.id, max_size); - if new_x_dim < dim_floor || new_y_dim < dim_floor { - warn!("Image dimension(s) is smaller than {} pixels but file size is greater than max moderation size, id={}, max_size={}", dim_floor, self.id, max_size ); + if new_x_dim < MINIMUM_IMAGE_DIMENSION || new_y_dim < MINIMUM_IMAGE_DIMENSION { + warn!("Image dimension(s) is smaller than {} pixels but file size is greater than max moderation size, id={}, max_size={}", MINIMUM_IMAGE_DIMENSION, self.id, max_size ); Ok(bytes) } else { self.resize(new_img, max_size) @@ -190,13 +194,13 @@ mod tests { let y_dim = 1500; let (x, y) = Document::resize_parameters(x_dim, y_dim); assert!(x < x_dim); - assert_eq!(y, MINIMUM_IMAGE_DIMENSION); + assert_eq!(y, NOMINAL_IMAGE_DIMENSION); let x_dim = 1500; let y_dim = 2048; let (x, y) = Document::resize_parameters(x_dim, y_dim); assert!(y < y_dim); - assert_eq!(x, MINIMUM_IMAGE_DIMENSION); + assert_eq!(x, NOMINAL_IMAGE_DIMENSION); let x_dim = 512; let y_dim = 20480; @@ -253,6 +257,6 @@ mod tests { assert!(loaded_image.is_ok()); let dimensions = loaded_image.unwrap().dimensions(); //TODO: Recheck why after img.resize is the y dimension of the image is off by -1 - assert_eq!(dimensions.1, MINIMUM_IMAGE_DIMENSION - 1); + assert_eq!(dimensions.1, NOMINAL_IMAGE_DIMENSION - 1); } } From 824a4a7b6d659d265d90b6502127ff9244e6e2ff Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 09:09:21 +0530 Subject: [PATCH 13/30] Scaled aws max doc size limit by 1.5 to account for base64 encoding --- src/aws/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aws/mod.rs b/src/aws/mod.rs index dc7da52..18d7829 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -62,7 +62,9 @@ impl ModerationProvider for Rekognition { } fn max_document_size(&self) -> u64 { - 5242880 // As per AWS documentation, 5 MB binary limit + // As per AWS documentation, 5 MB binary limit then scaled by + // generous encoding margin + (5242880_f64 / 1.5_f64).ceil() as u64 } } From 6844d65c9d1a76b7c179ffe6d2fc748c66216022 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 09:18:55 +0530 Subject: [PATCH 14/30] Added metrics for resizing logic --- src/document.rs | 7 ++++++- src/metrics.rs | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/document.rs b/src/document.rs index 2c48e20..05fafc4 100644 --- a/src/document.rs +++ b/src/document.rs @@ -1,8 +1,8 @@ extern crate base64; extern crate hyper; -use crate::cache::ByteSizeable; use crate::rpc::error::Errors; +use crate::{cache::ByteSizeable, metrics}; use image::io::Reader as ImageReader; use std::cmp::max; use std::io::Cursor; @@ -69,6 +69,7 @@ impl Document { } fn resize(&self, img: DynamicImage, max_size: u64) -> Result, Errors> { + metrics::IMAGE_RESIZE.with_label_values(&["request"]).inc(); let (x_dim, y_dim) = img.dimensions(); let (new_x_dim, new_y_dim) = Self::resize_parameters(x_dim, y_dim); info!( @@ -87,18 +88,22 @@ impl Document { bytes.len() ); if bytes.len() as u64 > max_size { + metrics::IMAGE_RESIZE.with_label_values(&["retry"]).inc(); warn!("Resizing did not reduce image size enough to fit max moderation size, id={}, max_size={}", self.id, max_size); if new_x_dim < MINIMUM_IMAGE_DIMENSION || new_y_dim < MINIMUM_IMAGE_DIMENSION { + metrics::IMAGE_RESIZE.with_label_values(&["dim_low"]).inc(); warn!("Image dimension(s) is smaller than {} pixels but file size is greater than max moderation size, id={}, max_size={}", MINIMUM_IMAGE_DIMENSION, self.id, max_size ); Ok(bytes) } else { self.resize(new_img, max_size) } } else { + metrics::IMAGE_RESIZE.with_label_values(&["success"]).inc(); Ok(bytes) } } Err(e) => { + metrics::IMAGE_RESIZE.with_label_values(&["failed"]).inc(); error!( "Error writing out image to buffer, id={}, reason={}", self.id, e diff --git a/src/metrics.rs b/src/metrics.rs index 0c3ebd3..e6d86aa 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -91,6 +91,14 @@ lazy_static! { &["protocol"] ) .unwrap(); + pub static ref IMAGE_RESIZE: IntCounterVec = IntCounterVec::new( + Opts::new( + "image_resize", + "Counts various metrics around Image resizing" + ), + &["metric"] + ) + .unwrap(); } pub fn init_registry() { @@ -120,6 +128,7 @@ pub fn init_registry() { REGISTRY .register(Box::new(URI_DESTINATION_PROTOCOL.clone())) .unwrap(); + REGISTRY.register(Box::new(IMAGE_RESIZE.clone())).unwrap(); #[cfg(not(target_os = "macos"))] let pc = ProcessCollector::for_self(); #[cfg(not(target_os = "macos"))] From 778c734079fd807abf93bb8243f111e7d6346a21 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 09:22:34 +0530 Subject: [PATCH 15/30] Changed metric name --- src/document.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/document.rs b/src/document.rs index 05fafc4..5b23567 100644 --- a/src/document.rs +++ b/src/document.rs @@ -91,7 +91,9 @@ impl Document { metrics::IMAGE_RESIZE.with_label_values(&["retry"]).inc(); warn!("Resizing did not reduce image size enough to fit max moderation size, id={}, max_size={}", self.id, max_size); if new_x_dim < MINIMUM_IMAGE_DIMENSION || new_y_dim < MINIMUM_IMAGE_DIMENSION { - metrics::IMAGE_RESIZE.with_label_values(&["dim_low"]).inc(); + metrics::IMAGE_RESIZE + .with_label_values(&["dim_floor_hit"]) + .inc(); warn!("Image dimension(s) is smaller than {} pixels but file size is greater than max moderation size, id={}, max_size={}", MINIMUM_IMAGE_DIMENSION, self.id, max_size ); Ok(bytes) } else { From e073f0ba4454cbfdb7e144ea22df6b1e3af6d6ec Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 09:38:44 +0530 Subject: [PATCH 16/30] AWS labels now use parent name --- src/aws/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 18d7829..0e32004 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -33,12 +33,12 @@ impl ModerationProvider for Rekognition { let labels = result.moderation_labels.unwrap_or_default(); let labels = labels .into_iter() - .filter(|l| l.name.is_some()) // Remove empty labels + .filter(|l| l.parent_name.is_some()) // Remove empty labels .map(|l| { - let name = l.name.unwrap(); // This is safe - Rekognition::normalize_category(name.as_str()) + l.parent_name() + .map(|n| Rekognition::normalize_category(n)) + .or_else(ModerationCategories::Unknown) }) - .filter(|l| *l != ModerationCategories::Unknown) .collect(); debug!( @@ -51,7 +51,7 @@ impl ModerationProvider for Rekognition { }) } Err(e) => { - error!("Moderation failed, reason:{}", e); + error!("Moderation failed, id={}, reason:{}", document.id, e); Err(Errors::ModerationFailed) } } From 326ff9d52eb227b8d93f9aebb518ef8152af307a Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 09:39:56 +0530 Subject: [PATCH 17/30] AWS labels now use parent name --- src/aws/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 18d7829..538e290 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -33,12 +33,12 @@ impl ModerationProvider for Rekognition { let labels = result.moderation_labels.unwrap_or_default(); let labels = labels .into_iter() - .filter(|l| l.name.is_some()) // Remove empty labels + .filter(|l| l.parent_name.is_some()) // Remove empty labels .map(|l| { - let name = l.name.unwrap(); // This is safe - Rekognition::normalize_category(name.as_str()) + l.parent_name() + .map(|n| Rekognition::normalize_category(n)) + .unwrap_or(ModerationCategories::Unknown) }) - .filter(|l| *l != ModerationCategories::Unknown) .collect(); debug!( @@ -51,7 +51,7 @@ impl ModerationProvider for Rekognition { }) } Err(e) => { - error!("Moderation failed, reason:{}", e); + error!("Moderation failed, id={}, reason:{}", document.id, e); Err(Errors::ModerationFailed) } } From 54e171109e19b72ec572060fd1c81b2fb30e5f21 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 10:52:05 +0530 Subject: [PATCH 18/30] Updated labeling logic --- src/aws/mod.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 4c08025..3baa6a7 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -34,25 +34,28 @@ impl ModerationProvider for Rekognition { let labels = labels .into_iter() .map(|l| { - if let Some(parent_name) = l.parent_name() { - let category = Rekognition::normalize_category(parent_name); + debug!("Moderation labels: id={}, name={:?}, parent={:?}", document.id, l.name(), l.parent_name()); + let parent_name = l.parent_name().filter(|n| !n.is_empty()); + let name = l.name(); + if let Some(cat) = parent_name { + let category = Rekognition::normalize_category(cat); if category == ModerationCategories::Unknown { warn!( "Moderation category has no enum, id={}, cat={}", - document.id, parent_name + document.id, cat ); } category - } else if let Some(name) = l.name() { + } else if let Some(cat) = name { warn!( "Moderation category has no parent, id={}, cat={}", - document.id, name + document.id, cat ); - let category = Rekognition::normalize_category(name); + let category = Rekognition::normalize_category(cat); if category == ModerationCategories::Unknown { warn!( "Moderation category has no enum, id={}, cat={}", - document.id, name + document.id, cat ); } category From 7198ef7d371563f65959ae011019ccd9b30f6bd9 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 11:51:22 +0530 Subject: [PATCH 19/30] Fixed bad merge --- Cargo.lock | 105 +++++++++-------------------------------------------- Cargo.toml | 4 -- 2 files changed, 17 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2418e9a..cb13302 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -341,7 +341,7 @@ version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f" dependencies = [ - "itoa 1.0.1", + "itoa", "num-integer", "ryu", "time 0.3.7", @@ -407,22 +407,9 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "block-buffer" -<<<<<<< HEAD -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.2" -======= -version = "0.10.0" ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" dependencies = [ @@ -636,30 +623,13 @@ dependencies = [ "syn", ] -[[package]] -name = "digest" -<<<<<<< HEAD -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.2" -======= -version = "0.10.1" ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cb780dce4f9a8f5c087362b3a4595936b2019e7c8b30f2c3e9a7e94e6ae9837" dependencies = [ -<<<<<<< HEAD - "block-buffer 0.10.2", -======= "block-buffer", ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef "crypto-common", "subtle", ] @@ -998,11 +968,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddca131f3e7f2ce2df364b57949a9d47915cfbd35e46cfee355ccebbf794d6a2" dependencies = [ -<<<<<<< HEAD - "digest 0.10.2", -======= "digest", ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef ] [[package]] @@ -1032,7 +998,7 @@ checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" dependencies = [ "bytes", "fnv", - "itoa 1.0.1", + "itoa", ] [[package]] @@ -1066,9 +1032,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.16" +version = "0.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55" +checksum = "043f0e083e9901b6cc658a77d1eb86f4fc650bbb977a4337dd63192826aa85dd" dependencies = [ "bytes", "futures-channel", @@ -1079,7 +1045,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 0.4.8", + "itoa", "pin-project-lite", "socket2", "tokio", @@ -1113,7 +1079,7 @@ checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" dependencies = [ "http", "hyper", - "rustls 0.20.2", + "rustls 0.20.3", "tokio", "tokio-rustls 0.23.2", ] @@ -1198,12 +1164,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" -[[package]] -name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - [[package]] name = "itoa" version = "1.0.1" @@ -1256,15 +1216,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -<<<<<<< HEAD version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c" -======= -version = "0.2.115" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a8d982fa7a96a000f6ec4cfe966de9703eccde29750df2bb8949da91b0e818d" ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef [[package]] name = "libgit2-sys" @@ -1360,11 +1314,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6a38fc55c8bbc10058782919516f88826e70320db6d206aebc49611d24216ae" dependencies = [ -<<<<<<< HEAD - "digest 0.10.2", -======= "digest", ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef ] [[package]] @@ -1455,11 +1405,7 @@ dependencies = [ [[package]] name = "nft_image_proxy" -<<<<<<< HEAD -version = "1.4.1" -======= version = "2.0.0" ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef dependencies = [ "anyhow", "async-trait", @@ -1483,18 +1429,11 @@ dependencies = [ "log", "log4rs", "prometheus", -<<<<<<< HEAD "rand", - "rust-embed", - "serde", - "serde_json", - "sha2 0.10.1", - "tikv-jemallocator", -======= "serde", "serde_json", "sha2", ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef + "tikv-jemallocator", "time 0.3.7", "tokio", "tokio-postgres", @@ -1513,9 +1452,9 @@ dependencies = [ [[package]] name = "ntapi" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" dependencies = [ "winapi", ] @@ -1947,7 +1886,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rustls 0.20.2", + "rustls 0.20.3", "rustls-native-certs 0.6.1", "rustls-pemfile", "serde", @@ -2001,9 +1940,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37e5e2290f3e040b594b1a9e04377c2c671f1a1cfd9bfdef82106ac1c113f84" +checksum = "b323592e3164322f5b193dc4302e4e36cd8d37158a712d664efae1a5c2791700" dependencies = [ "log", "ring", @@ -2156,11 +2095,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d23c1ba4cf0efd44be32017709280b32d1cea5c3f1275c3b6d9e8bc54f758085" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" dependencies = [ - "itoa 1.0.1", + "itoa", "ryu", "serde", ] @@ -2181,7 +2120,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.1", + "itoa", "ryu", "serde", ] @@ -2206,11 +2145,7 @@ checksum = "99c3bd8169c58782adad9290a9af5939994036b76187f7b4f0e6de91dbbfc0ec" dependencies = [ "cfg-if", "cpufeatures", -<<<<<<< HEAD - "digest 0.10.2", -======= "digest", ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef ] [[package]] @@ -2485,7 +2420,7 @@ version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b" dependencies = [ - "rustls 0.20.2", + "rustls 0.20.3", "tokio", "webpki 0.22.0", ] @@ -2860,12 +2795,6 @@ dependencies = [ [[package]] name = "zeroize" -<<<<<<< HEAD version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c88870063c39ee00ec285a2f8d6a966e5b6fb2becc4e8dac77ed0d370ed6006" -======= -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4062c749be08d90be727e9c5895371c3a0e49b90ba2b9592dc7afda95cc2b719" ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef diff --git a/Cargo.toml b/Cargo.toml index 885bd18..8c34953 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,7 @@ [package] name = "nft_image_proxy" -<<<<<<< HEAD -version = "1.4.1" -======= version = "2.0.0" repository = "https://github.com/Cryptonomic/ImageProxy" ->>>>>>> 25ddd5e390071213311de61df4181d9e92d01aef authors = ["Cryptonomic Inc."] edition = "2018" build = "build.rs" From 2937d0214a4eb42fc3b425d65ed1bd1c6a662555 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 12:32:18 +0530 Subject: [PATCH 20/30] Removed len check from resize validation --- src/document.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/document.rs b/src/document.rs index 5b23567..2671e96 100644 --- a/src/document.rs +++ b/src/document.rs @@ -116,10 +116,6 @@ impl Document { } pub fn resize_image(&self, max_size: u64) -> Result { - if (self.bytes.len() as u64) < max_size { - warn!("Image resize is not required as size is below max_size, id={}, size={}, max_size={}", self.id, self.bytes.len(), max_size); - return Ok(self.clone()); - } info!( "Image info, id={}, len={}, type={}", self.id, @@ -127,6 +123,18 @@ impl Document { self.content_type ); + if (self.bytes.len() as u64) < max_size { + info!( + "Likely image format change, id={}, size={}, max_size={}", + self.id, + self.bytes.len(), + max_size + ); + metrics::IMAGE_RESIZE + .with_label_values(&["format_change"]) + .inc(); + } + let img = self.load_image()?; let bytes = self.resize(img, max_size)?; Ok(Document { @@ -248,12 +256,6 @@ mod tests { // Check resize logic let max_size_5mb = 1024_u64 * 1024_u64 * 5_u64; - let max_size_100mb = max_size_5mb * 20_u64; - - // No resize required - let new_document = document.resize_image(max_size_100mb); - assert!(new_document.is_ok()); - assert_eq!(document.bytes.len(), new_document.unwrap().bytes.len()); // Resize required let new_document = document.resize_image(max_size_5mb); From 920df98ab4df0f18b851ccf54a163abc142daabc Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 13:41:08 +0530 Subject: [PATCH 21/30] Updated resize logic --- src/document.rs | 114 +++++++++++++++++++++++++++++------------------- 1 file changed, 69 insertions(+), 45 deletions(-) diff --git a/src/document.rs b/src/document.rs index 2671e96..877be4f 100644 --- a/src/document.rs +++ b/src/document.rs @@ -52,26 +52,40 @@ impl Document { }) } - fn resize_parameters(x_dim: u32, y_dim: u32) -> (u32, u32) { + fn resize_parameters(x_dim: u32, y_dim: u32, target_dim: u32, min_dim: u32) -> (u32, u32) { let aspect_ratio = x_dim as f64 / y_dim as f64; - - if x_dim <= NOMINAL_IMAGE_DIMENSION || y_dim <= NOMINAL_IMAGE_DIMENSION { - return (max(x_dim / 2_u32, 1_u32), max(y_dim / 2_u32, 1_u32)); - } - if aspect_ratio > 1_f64 { - let new_x_dim = aspect_ratio * (NOMINAL_IMAGE_DIMENSION as f64); - (new_x_dim as u32, NOMINAL_IMAGE_DIMENSION) + // X is larger, set it to NOMINAL_IMAGE_DIMENSION + let new_x_dim = target_dim as f64; + let new_y_dim = new_x_dim / aspect_ratio; + + if new_y_dim < min_dim as f64 { + let new_y_dim = min_dim as f64; + let new_x_dim = aspect_ratio * new_y_dim; + (new_x_dim.floor() as u32, new_y_dim.floor() as u32) + } else { + (new_x_dim.floor() as u32, new_y_dim.floor() as u32) + } } else { - let new_y_dim = aspect_ratio * (NOMINAL_IMAGE_DIMENSION as f64); - (NOMINAL_IMAGE_DIMENSION, new_y_dim as u32) + // Y is larger, set it to NOMINAL_IMAGE_DIMENSION + let new_y_dim = target_dim as f64; + let new_x_dim = new_y_dim * aspect_ratio; + + if new_x_dim < min_dim as f64 { + let new_x_dim = min_dim as f64; + let new_y_dim = new_x_dim / aspect_ratio; + (new_x_dim.floor() as u32, new_y_dim.floor() as u32) + } else { + (new_x_dim.floor() as u32, new_y_dim.floor() as u32) + } } } - fn resize(&self, img: DynamicImage, max_size: u64) -> Result, Errors> { + fn resize(&self, img: DynamicImage, target_dim: u32, max_size: u64) -> Result, Errors> { metrics::IMAGE_RESIZE.with_label_values(&["request"]).inc(); let (x_dim, y_dim) = img.dimensions(); - let (new_x_dim, new_y_dim) = Self::resize_parameters(x_dim, y_dim); + let (new_x_dim, new_y_dim) = + Self::resize_parameters(x_dim, y_dim, target_dim, MINIMUM_IMAGE_DIMENSION); info!( "Image resizing, id={}, x={}, y={}, new_x={}, new_y={}", self.id, x_dim, y_dim, new_x_dim, new_y_dim @@ -90,14 +104,14 @@ impl Document { if bytes.len() as u64 > max_size { metrics::IMAGE_RESIZE.with_label_values(&["retry"]).inc(); warn!("Resizing did not reduce image size enough to fit max moderation size, id={}, max_size={}", self.id, max_size); - if new_x_dim < MINIMUM_IMAGE_DIMENSION || new_y_dim < MINIMUM_IMAGE_DIMENSION { + if target_dim / 2_u32 < MINIMUM_IMAGE_DIMENSION { metrics::IMAGE_RESIZE .with_label_values(&["dim_floor_hit"]) .inc(); warn!("Image dimension(s) is smaller than {} pixels but file size is greater than max moderation size, id={}, max_size={}", MINIMUM_IMAGE_DIMENSION, self.id, max_size ); Ok(bytes) } else { - self.resize(new_img, max_size) + self.resize(new_img, target_dim / 2_u32, max_size) } } else { metrics::IMAGE_RESIZE.with_label_values(&["success"]).inc(); @@ -136,7 +150,7 @@ impl Document { } let img = self.load_image()?; - let bytes = self.resize(img, max_size)?; + let bytes = self.resize(img, NOMINAL_IMAGE_DIMENSION, max_size)?; Ok(Document { id: self.id, content_length: bytes.len() as u64, @@ -207,39 +221,49 @@ mod tests { fn test_resize_parameters() { let x_dim = 2048; let y_dim = 1500; - let (x, y) = Document::resize_parameters(x_dim, y_dim); - assert!(x < x_dim); - assert_eq!(y, NOMINAL_IMAGE_DIMENSION); + let (x, y) = Document::resize_parameters( + x_dim, + y_dim, + NOMINAL_IMAGE_DIMENSION, + MINIMUM_IMAGE_DIMENSION, + ); + assert_eq!(x, NOMINAL_IMAGE_DIMENSION); + assert!(y < y_dim); let x_dim = 1500; let y_dim = 2048; - let (x, y) = Document::resize_parameters(x_dim, y_dim); - assert!(y < y_dim); - assert_eq!(x, NOMINAL_IMAGE_DIMENSION); + let (x, y) = Document::resize_parameters( + x_dim, + y_dim, + NOMINAL_IMAGE_DIMENSION, + MINIMUM_IMAGE_DIMENSION, + ); + assert!(x < x_dim); + assert_eq!(y, NOMINAL_IMAGE_DIMENSION); - let x_dim = 512; - let y_dim = 20480; - let (x, y) = Document::resize_parameters(x_dim, y_dim); - assert_eq!(x, x_dim / 2); - assert_eq!(y, y_dim / 2); - - let x_dim = 20480; - let y_dim = 512; - let (x, y) = Document::resize_parameters(x_dim, y_dim); - assert_eq!(x, x_dim / 2); - assert_eq!(y, y_dim / 2); - - let x_dim = 512; - let y_dim = 1; - let (x, y) = Document::resize_parameters(x_dim, y_dim); - assert_eq!(x, x_dim / 2); - assert_eq!(y, 1); - - let x_dim = 1; - let y_dim = 512; - let (x, y) = Document::resize_parameters(x_dim, y_dim); - assert_eq!(x, 1); - assert_eq!(y, y_dim / 2); + // High X aspect ratio + let x_dim = 2000; + let y_dim = 200; + let (x, y) = Document::resize_parameters( + x_dim, + y_dim, + NOMINAL_IMAGE_DIMENSION, + MINIMUM_IMAGE_DIMENSION, + ); + assert!(x < x_dim); + assert_eq!(y, MINIMUM_IMAGE_DIMENSION); + + // High Y aspect ratio + let x_dim = 200; + let y_dim = 2000; + let (x, y) = Document::resize_parameters( + x_dim, + y_dim, + NOMINAL_IMAGE_DIMENSION, + MINIMUM_IMAGE_DIMENSION, + ); + assert_eq!(x, MINIMUM_IMAGE_DIMENSION); + assert!(y < y_dim); } #[test] @@ -266,6 +290,6 @@ mod tests { assert!(loaded_image.is_ok()); let dimensions = loaded_image.unwrap().dimensions(); //TODO: Recheck why after img.resize is the y dimension of the image is off by -1 - assert_eq!(dimensions.1, NOMINAL_IMAGE_DIMENSION - 1); + assert_eq!(dimensions.0, NOMINAL_IMAGE_DIMENSION); } } From 926907e4d001617b7c74bee0e667c7b5a1521fd7 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Mon, 14 Feb 2022 14:02:36 +0530 Subject: [PATCH 22/30] Removed unused import --- src/document.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/document.rs b/src/document.rs index 877be4f..ef1679a 100644 --- a/src/document.rs +++ b/src/document.rs @@ -4,7 +4,6 @@ extern crate hyper; use crate::rpc::error::Errors; use crate::{cache::ByteSizeable, metrics}; use image::io::Reader as ImageReader; -use std::cmp::max; use std::io::Cursor; use base64::encode; From 68351c034849893b2de5d6385978c99e915f0710 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Tue, 15 Feb 2022 07:31:38 +0530 Subject: [PATCH 23/30] Added moderation category metric --- src/aws/mod.rs | 5 ++++- src/document.rs | 1 - src/metrics.rs | 8 ++++++++ src/moderation.rs | 9 ++++++++- src/rpc/mod.rs | 6 ++++++ 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 3baa6a7..f4d843f 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -31,7 +31,7 @@ impl ModerationProvider for Rekognition { match self.get_moderation_labels(&document.bytes).await { Ok(result) => { let labels = result.moderation_labels.unwrap_or_default(); - let labels = labels + let mut labels: Vec = labels .into_iter() .map(|l| { debug!("Moderation labels: id={}, name={:?}, parent={:?}", document.id, l.name(), l.parent_name()); @@ -69,6 +69,9 @@ impl ModerationProvider for Rekognition { }) .collect(); + labels.sort(); + labels.dedup(); + debug!( "Moderation labels for id={}, labels={:?}", document.id, labels diff --git a/src/document.rs b/src/document.rs index 877be4f..ef1679a 100644 --- a/src/document.rs +++ b/src/document.rs @@ -4,7 +4,6 @@ extern crate hyper; use crate::rpc::error::Errors; use crate::{cache::ByteSizeable, metrics}; use image::io::Reader as ImageReader; -use std::cmp::max; use std::io::Cursor; use base64::encode; diff --git a/src/metrics.rs b/src/metrics.rs index e6d86aa..212d210 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -71,6 +71,11 @@ lazy_static! { IntCounterVec::new(Opts::new("traffic", "Traffic stats in bytes"), &["metric"]).unwrap(); pub static ref MODERATION: IntCounterVec = IntCounterVec::new(Opts::new("moderation", "Moderation stats"), &["metric"]).unwrap(); + pub static ref MODERATION_CATEGORIES: IntCounterVec = IntCounterVec::new( + Opts::new("moderation_categories", "Moderation Categories"), + &["category"] + ) + .unwrap(); pub static ref DOCUMENT_SIZE: HistogramVec = HistogramVec::new( HistogramOpts::new("doc_size", "Document Size").buckets(DOCUMENT_SIZE_BUCKETS.clone()), &["size_bytes"] @@ -129,6 +134,9 @@ pub fn init_registry() { .register(Box::new(URI_DESTINATION_PROTOCOL.clone())) .unwrap(); REGISTRY.register(Box::new(IMAGE_RESIZE.clone())).unwrap(); + REGISTRY + .register(Box::new(MODERATION_CATEGORIES.clone())) + .unwrap(); #[cfg(not(target_os = "macos"))] let pc = ProcessCollector::for_self(); #[cfg(not(target_os = "macos"))] diff --git a/src/moderation.rs b/src/moderation.rs index 1d7d85c..3f0478b 100644 --- a/src/moderation.rs +++ b/src/moderation.rs @@ -28,7 +28,7 @@ impl SupportedMimeTypes { } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] pub enum ModerationCategories { ExplicitNudity, Suggestive, @@ -42,6 +42,13 @@ pub enum ModerationCategories { Hate, Unknown, } + +impl std::fmt::Display for ModerationCategories { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + #[derive(Clone)] pub struct ModerationResponse { pub categories: Vec, diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 3f9188a..e6ef131 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -116,6 +116,12 @@ pub async fn fetch( .with_label_values(&["moderated"]) .inc_by(document.bytes.len() as u64); + mod_response.categories.iter().for_each(|c| { + metrics::MODERATION_CATEGORIES + .with_label_values(&[&c.to_string()]) + .inc() + }); + let blocked = !mod_response.categories.is_empty(); let mod_status: ModerationStatus = blocked.into(); From c053265ec2dc690fcc7d0ad5003ec5f2584b82b2 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Tue, 15 Feb 2022 08:42:21 +0530 Subject: [PATCH 24/30] Added caching layer for database queries --- Cargo.lock | 353 ++++++++++++++++++++++++++++++++++++++++++++----- Cargo.toml | 1 + src/proxy.rs | 6 +- src/rpc/mod.rs | 32 +++-- 4 files changed, 349 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c6384e2..92a831b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -385,7 +385,7 @@ dependencies = [ "async-trait", "futures-channel", "futures-util", - "parking_lot", + "parking_lot 0.11.2", "tokio", ] @@ -441,6 +441,12 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +[[package]] +name = "bytecount" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72feb31ffc86498dacdbd0fcebb56138e7177a8cc5cea4516031d15ae85a742e" + [[package]] name = "bytemuck" version = "1.7.3" @@ -469,6 +475,15 @@ dependencies = [ "either", ] +[[package]] +name = "camino" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f3132262930b0522068049f5870a856ab8affc80c70d08b6ecb785771a6fc23" +dependencies = [ + "serde", +] + [[package]] name = "cargo-lock" version = "7.0.1" @@ -481,6 +496,28 @@ dependencies = [ "url", ] +[[package]] +name = "cargo-platform" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba2ae6de944143141f6155a473a6b02f66c7c3f9f47316f802f80204ebfe6e12" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", +] + [[package]] name = "cc" version = "1.0.72" @@ -490,6 +527,12 @@ dependencies = [ "jobserver", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -546,7 +589,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -555,8 +598,8 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" dependencies = [ - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.7", ] [[package]] @@ -565,9 +608,24 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-epoch 0.9.7", + "crossbeam-utils 0.8.7", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard", ] [[package]] @@ -576,20 +634,31 @@ version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" dependencies = [ - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.7", "lazy_static", - "memoffset", + "memoffset 0.6.5", "scopeguard", ] +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", +] + [[package]] name = "crossbeam-utils" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "lazy_static", ] @@ -658,7 +727,7 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53ecafc952c4528d9b51a458d1a8904b81783feff9fde08ab6ed2545ff396872" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "socket2", "winapi", @@ -740,7 +809,16 @@ version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", +] + +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check 0.9.4", ] [[package]] @@ -764,7 +842,7 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crc32fast", "libc", "miniz_oxide 0.4.4", @@ -912,7 +990,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi", ] @@ -940,6 +1018,12 @@ dependencies = [ "url", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "h2" version = "0.3.11" @@ -1173,7 +1257,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1289,7 +1373,7 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "serde", ] @@ -1314,7 +1398,7 @@ dependencies = [ "libc", "log", "log-mdc", - "parking_lot", + "parking_lot 0.11.2", "regex", "serde", "serde-value", @@ -1326,12 +1410,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "matches" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "md-5" version = "0.10.0" @@ -1353,6 +1452,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.6.5" @@ -1409,6 +1517,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "moka" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26e1a0803406a47496169480f26cab92e78936c1180ad439ee18d36478c1509c" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.8.7", + "num_cpus", + "once_cell", + "parking_lot 0.12.0", + "quanta", + "scheduled-thread-pool", + "skeptic", + "smallvec", + "tagptr", + "thiserror", + "triomphe", + "uuid", +] + [[package]] name = "native-tls" version = "0.2.8" @@ -1452,6 +1582,7 @@ dependencies = [ "lazy_static", "log", "log4rs", + "moka", "prometheus", "rand", "rust-embed", @@ -1563,7 +1694,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", @@ -1606,7 +1737,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.1", ] [[package]] @@ -1615,7 +1756,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall 0.2.10", @@ -1623,6 +1764,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.2.10", + "smallvec", + "windows-sys", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1762,12 +1916,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fnv", "lazy_static", "libc", "memchr", - "parking_lot", + "parking_lot 0.11.2", "procfs", "protobuf", "thiserror", @@ -1779,6 +1933,33 @@ version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" +[[package]] +name = "pulldown-cmark" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34f197a544b0c9ab3ae46c359a7ec9cbbb5c7bf97054266fecb7ead794a181d6" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "quanta" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" +dependencies = [ + "crossbeam-utils 0.8.7", + "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.15" @@ -1828,6 +2009,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "raw-cpuid" +version = "10.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "929f54e29691d4e6a9cc558479de70db7aa3d98cd6fe7ab86d7507aa2886b9d2" +dependencies = [ + "bitflags", +] + [[package]] name = "rayon" version = "1.5.1" @@ -1848,7 +2038,7 @@ checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils", + "crossbeam-utils 0.8.7", "lazy_static", "num_cpus", ] @@ -2073,6 +2263,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot 0.11.2", +] + [[package]] name = "scoped_threadpool" version = "0.1.9" @@ -2218,7 +2417,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", "opaque-debug", @@ -2230,7 +2429,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99c3bd8169c58782adad9290a9af5939994036b76187f7b4f0e6de91dbbfc0ec" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.2", ] @@ -2250,6 +2449,21 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a86232ab60fa71287d7f2ddae4a7073f6b7aac33631c3015abb556f08c6d0a3e" +[[package]] +name = "skeptic" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" +dependencies = [ + "bytecount", + "cargo_metadata", + "error-chain", + "glob", + "pulldown-cmark", + "tempfile", + "walkdir", +] + [[package]] name = "slab" version = "0.4.5" @@ -2278,6 +2492,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "stringprep" version = "0.1.2" @@ -2305,13 +2525,19 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tempfile" version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand", "libc", "redox_syscall 0.2.10", @@ -2429,7 +2655,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.11.2", "pin-project-lite", "signal-hook-registry", "tokio-macros", @@ -2479,7 +2705,7 @@ dependencies = [ "fallible-iterator", "futures", "log", - "parking_lot", + "parking_lot 0.11.2", "percent-encoding", "phf", "pin-project-lite", @@ -2580,7 +2806,7 @@ version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d8d93354fe2a8e50d5953f5ae2e47a3fc2ef03292e7ea46e3cc38f549525fb9" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -2613,6 +2839,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" +[[package]] +name = "triomphe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c45e322b26410d7260e00f64234810c2f17d7ece356182af4df8f7ff07890f09" +dependencies = [ + "memoffset 0.6.5", + "serde", + "stable_deref_trait", +] + [[package]] name = "try-lock" version = "0.2.3" @@ -2634,6 +2871,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check 0.9.4", +] + [[package]] name = "unicode-bidi" version = "0.3.7" @@ -2749,7 +2995,7 @@ version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] @@ -2774,7 +3020,7 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -2876,6 +3122,49 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" + +[[package]] +name = "windows_i686_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" + +[[package]] +name = "windows_i686_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" + [[package]] name = "winreg" version = "0.7.0" diff --git a/Cargo.toml b/Cargo.toml index 4434170..c78562c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ image = "0.23" dns-lookup = "1" anyhow = "1.0" rust-embed="6.3" +moka = "0.7" # deps for aws aws-config = "0.6.0" diff --git a/src/proxy.rs b/src/proxy.rs index e1d466c..088d0cb 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -3,7 +3,7 @@ extern crate tokio_postgres; use crate::cache::{get_cache, Cache}; use crate::config::{Cors, SecurityConfig}; -use crate::db::{DatabaseFactory, DatabaseProvider}; +use crate::db::{DatabaseFactory, DatabaseProvider, DbModerationRow}; use crate::dns::StandardDnsResolver; use crate::document::Document; @@ -27,10 +27,10 @@ use crate::{ }; use chrono::Utc; - use hyper::header::HeaderValue; use hyper::{Body, Method, Request, Response, StatusCode}; use log::{debug, error}; +use moka::sync::Cache as MokaCache; use prometheus::Encoder; use rust_embed::RustEmbed; use serde::de; @@ -52,6 +52,7 @@ pub struct Context { pub moderation_provider: Box, pub http_client_provider: HttpClientWrapper, pub cache: Option + Send + Sync>>, + pub db_cache: Arc>, } impl Context { @@ -74,6 +75,7 @@ impl Context { moderation_provider, http_client_provider: http_client, cache: get_cache(&config.cache_config), + db_cache: Arc::new(MokaCache::new(10000)), }) } } diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index e6ef131..2af54e0 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -66,15 +66,27 @@ pub async fn fetch( info!("Document id={} has forced flag enabled.", req_id); } - let urls = vec![params.url.clone()]; - let db_results = ctx - .database - .get_moderation_result(&urls) - .await - .map_err(|e| { - error!("Error querying database for id={}, reason={}", req_id, e); - Errors::InternalError - })?; + let db_results = if let Some(result) = ctx.db_cache.get(¶ms.url) { + info!( + "Database moderation query skipped. Using cached results, id={}", + req_id + ); + vec![result] + } else { + info!("Querying database for moderation results, id={}", req_id); + let results = ctx + .database + .get_moderation_result(&[params.url.clone()]) + .await + .map_err(|e| { + error!("Error querying database for id={}, reason={}", req_id, e); + Errors::InternalError + })?; + results.iter().for_each(|r| { + ctx.db_cache.insert(r.url.clone(), r.clone()); + }); + results + }; let (moderation_status, categories, document) = match db_results.get(0) { Some(result) => { @@ -264,6 +276,7 @@ pub async fn describe_report( #[cfg(test)] mod tests { use hyper::body::Bytes; + use moka::sync::Cache as MokaCache; use uuid::Uuid; use crate::config::{Host, IpfsGatewayConfig}; @@ -327,6 +340,7 @@ mod tests { moderation_provider: Box::new(moderation_provider), http_client_provider, cache: None, + db_cache: Arc::new(MokaCache::new(10)), }; Arc::new(context) From 5f255d4d8beb5c4dd481fe308bc98a3e84b05f08 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Tue, 15 Feb 2022 08:44:34 +0530 Subject: [PATCH 25/30] Improved logging --- src/rpc/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 2af54e0..10e527c 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -82,6 +82,7 @@ pub async fn fetch( error!("Error querying database for id={}, reason={}", req_id, e); Errors::InternalError })?; + info!("Query result, id={}, rows={}", req_id, results.len()); results.iter().for_each(|r| { ctx.db_cache.insert(r.url.clone(), r.clone()); }); @@ -92,7 +93,7 @@ pub async fn fetch( Some(result) => { metrics::MODERATION.with_label_values(&["cache_hit"]).inc(); info!( - "Found cached moderation results for id={}, blocked={}, categories:{:?}, provider:{:?}", + "Database has moderation results for id={}, blocked={}, categories:{:?}, provider:{:?}", req_id, result.blocked, result.categories, result.provider ); let document = if !result.blocked || params.force { @@ -104,7 +105,7 @@ pub async fn fetch( } None => { metrics::MODERATION.with_label_values(&["cache_miss"]).inc(); - info!("No cached moderation results found for id={}", req_id); + info!("Database has no moderation results for id={}", req_id); let document = fetch_document(ctx.clone(), req_id, ¶ms.url).await?; let max_document_size = ctx.moderation_provider.max_document_size(); let supported_types = ctx.moderation_provider.supported_types(); From fdfed2edfbe5147db1a7b9d052a42365d1612cf4 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Tue, 15 Feb 2022 09:12:53 +0530 Subject: [PATCH 26/30] Added metrics for db cache --- src/rpc/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 10e527c..61e88e3 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -71,9 +71,15 @@ pub async fn fetch( "Database moderation query skipped. Using cached results, id={}", req_id ); + metrics::CACHE_METRICS + .with_label_values(&["db_cache", "hits"]) + .inc(); vec![result] } else { info!("Querying database for moderation results, id={}", req_id); + metrics::CACHE_METRICS + .with_label_values(&["db_cache", "misses"]) + .inc(); let results = ctx .database .get_moderation_result(&[params.url.clone()]) From c9912e3bc72aafdda1d3c1325f102bd2463e1a15 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Tue, 15 Feb 2022 09:28:03 +0530 Subject: [PATCH 27/30] Added metrics for ipfs fallback gateway --- src/http/mod.rs | 3 ++- src/metrics.rs | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/http/mod.rs b/src/http/mod.rs index 9610ce4..18c97d7 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs @@ -149,7 +149,8 @@ impl HttpClientWrapper { && parsed_uri.scheme == UriScheme::Ipfs && self.ipfs_config.fallback.is_some() { - debug!("Using fallback gateway for req_id={}, url={}", req_id, url); + info!("Using fallback gateway for req_id={}, url={}", req_id, url); + metrics::IPFS_FALLBACK.inc(); let fallback_ipfs_config = self.ipfs_config.fallback.as_ref().unwrap(); let uri = HttpClientWrapper::construct_ipfs_uri(url, fallback_ipfs_config)?; self.do_fetch(req_id, &uri).await diff --git a/src/metrics.rs b/src/metrics.rs index 212d210..76dd214 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -31,6 +31,8 @@ lazy_static! { ) .unwrap(); pub static ref HITS: IntCounter = IntCounter::new("hits", "Total hits").unwrap(); + pub static ref IPFS_FALLBACK: IntCounter = + IntCounter::new("ipfs_fallback", "Ipfs fallback gateway usage").unwrap(); pub static ref CACHE_METRICS: IntGaugeVec = IntGaugeVec::new( Opts::new("cache_metrics", "Cache metics by cache type"), &["type", "metric"] @@ -108,6 +110,7 @@ lazy_static! { pub fn init_registry() { REGISTRY.register(Box::new(HITS.clone())).unwrap(); + REGISTRY.register(Box::new(IPFS_FALLBACK.clone())).unwrap(); REGISTRY.register(Box::new(MODERATION.clone())).unwrap(); REGISTRY.register(Box::new(API_REQUESTS.clone())).unwrap(); REGISTRY From 479d7d6ef360075fbf2a0b89c4880de5d0808bb0 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Tue, 15 Feb 2022 10:06:16 +0530 Subject: [PATCH 28/30] Added missing mod cat metrics invocation --- src/rpc/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 61e88e3..8850b4a 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -102,6 +102,11 @@ pub async fn fetch( "Database has moderation results for id={}, blocked={}, categories:{:?}, provider:{:?}", req_id, result.blocked, result.categories, result.provider ); + result.categories.iter().for_each(|c| { + metrics::MODERATION_CATEGORIES + .with_label_values(&[&c.to_string()]) + .inc() + }); let document = if !result.blocked || params.force { Some(fetch_document(ctx.clone(), req_id, ¶ms.url).await?) } else { From 5d2fdd987469e5f5a521d117d9f5baf08cdc98e5 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Tue, 15 Feb 2022 10:38:50 +0530 Subject: [PATCH 29/30] Added missing invocation of document metric --- src/rpc/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 8850b4a..77511ed 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -110,6 +110,7 @@ pub async fn fetch( let document = if !result.blocked || params.force { Some(fetch_document(ctx.clone(), req_id, ¶ms.url).await?) } else { + metrics::DOCUMENT.with_label_values(&["blocked"]).inc(); None }; (result.blocked.into(), result.categories.clone(), document) From 533fd50c02c04507f1287f663e36fe285fe108e6 Mon Sep 17 00:00:00 2001 From: bhaskarkishore <50350646+bhaskarkishore@users.noreply.github.com> Date: Tue, 15 Feb 2022 11:06:37 +0530 Subject: [PATCH 30/30] Removed unecessary dep --- Cargo.lock | 84 ++++-------------------------------------------------- Cargo.toml | 1 - 2 files changed, 6 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e19fa7..3ee0795 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -407,15 +407,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.2" @@ -701,22 +692,13 @@ dependencies = [ "syn", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cb780dce4f9a8f5c087362b3a4595936b2019e7c8b30f2c3e9a7e94e6ae9837" dependencies = [ - "block-buffer 0.10.2", + "block-buffer", "crypto-common", "subtle", ] @@ -1070,7 +1052,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddca131f3e7f2ce2df364b57949a9d47915cfbd35e46cfee355ccebbf794d6a2" dependencies = [ - "digest 0.10.2", + "digest", ] [[package]] @@ -1431,7 +1413,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6a38fc55c8bbc10058782919516f88826e70320db6d206aebc49611d24216ae" dependencies = [ - "digest 0.10.2", + "digest", ] [[package]] @@ -1579,10 +1561,9 @@ dependencies = [ "moka", "prometheus", "rand", - "rust-embed", "serde", "serde_json", - "sha2 0.10.1", + "sha2", "tikv-jemallocator", "time 0.3.7", "tokio", @@ -1675,12 +1656,6 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" version = "0.10.38" @@ -1859,7 +1834,7 @@ dependencies = [ "md-5", "memchr", "rand", - "sha2 0.10.1", + "sha2", "stringprep", ] @@ -2131,40 +2106,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "rust-embed" -version = "6.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40377bff8cceee81e28ddb73ac97f5c2856ce5522f0b260b763f434cdfae602" -dependencies = [ - "rust-embed-impl", - "rust-embed-utils", - "walkdir", -] - -[[package]] -name = "rust-embed-impl" -version = "6.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e763e24ba2bf0c72bc6be883f967f794a019fafd1b86ba1daff9c91a7edd30" -dependencies = [ - "proc-macro2", - "quote", - "rust-embed-utils", - "syn", - "walkdir", -] - -[[package]] -name = "rust-embed-utils" -version = "7.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad22c7226e4829104deab21df575e995bfbc4adfad13a595e387477f238c1aec" -dependencies = [ - "sha2 0.9.9", - "walkdir", -] - [[package]] name = "rustc_version" version = "0.4.0" @@ -2404,19 +2345,6 @@ dependencies = [ "yaml-rust", ] -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "sha2" version = "0.10.1" @@ -2425,7 +2353,7 @@ checksum = "99c3bd8169c58782adad9290a9af5939994036b76187f7b4f0e6de91dbbfc0ec" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.2", + "digest", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d11d444..8ae6c13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,6 @@ async-trait = "0.1" image = "0.23" dns-lookup = "1" anyhow = "1.0" -rust-embed="6.3" moka = "0.7" # deps for aws