feat: Add misc.is_b2c field (#1553)

* remove lettre * Add retries * feat: Add misc.is_b2c * Fix docker
reacherhq · Dec 30, 2024 · 14a6759 · 14a6759
1 parent b1377db
commit 14a6759
Show file tree

Hide file tree

Showing 17 changed files with 97,694 additions and 1,079 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Makefile b/Makefile
@@ -1,10 +1,13 @@
+###############################################################################
+# Run
+###############################################################################
+
 # Run the backend without worker mode, i.e. only enabling single-shot
 # verifications via the /v1/check_email endpoint.
 .PHONY: run
 run:
 	cd backend && cargo run --bin reacher_backend
 
-
 # Run the backend with worker mode on. This enables the /v1/bulk endpoints.
 # Make sure to have a Postgres DB and a RabbitMQ instance running.
 .PHONY: run-with-worker
@@ -30,4 +33,23 @@ changelog:
 	echo "" >> CHANGELOG.md
 	echo "All notable changes to this project will be documented in this file. The changes in this project follow [Convention Commits](https://www.conventionalcommits.org/en/v1.0.0/)." >> CHANGELOG.md
 	echo "" >> CHANGELOG.md
-	conventional-changelog -p angular -r 0 >> CHANGELOG.md
+	conventional-changelog -p angular -r 0 >> CHANGELOG.md
+
+###############################################################################
+# Update lists
+###############################################################################
+
+.PHONY: update-role-accounts
+update-role-accounts:
+# License is MIT.
+	curl https://raw.githubusercontent.com/mixmaxhq/role-based-email-addresses/refs/heads/master/index.js -o core/src/misc/roles.txt
+# Remove first line, last line, and all ' and , characters
+	sed -i.bak '1d' core/src/misc/roles.txt && rm core/src/misc/roles.txt.bak
+	sed -i.bak '$$d' core/src/misc/roles.txt && rm core/src/misc/roles.txt.bak
+	sed -i.bak 's/['\'', ]//g' core/src/misc/roles.txt && rm core/src/misc/roles.txt.bak
+
+
+.PHONY: update-free-email-providers
+update-free-email-providers:
+# License is MIT.
+	curl https://raw.githubusercontent.com/ihmpavel/free-email-domains-list/refs/heads/master/data/data.txt -o core/src/misc/b2c.txt
diff --git a/README.md b/README.md
@@ -101,7 +101,8 @@ The output will be a JSON with the below format, the fields should be self-expla
 	"is_reachable": "invalid",
 	"misc": {
 		"is_disposable": false,
-		"is_role_account": false
+		"is_role_account": false,
+		"is_b2c": true
 	},
 	"mx": {
 		"accepts_mail": true,

diff --git a/backend/openapi.json b/backend/openapi.json
@@ -409,7 +409,8 @@
 						"is_reachable": "invalid",
 						"misc": {
 							"is_disposable": false,
-							"is_role_account": true
+							"is_role_account": true,
+							"is_b2c": true
 						},
 						"mx": {
 							"accepts_mail": true,
@@ -498,12 +499,13 @@
 				"type": "string",
 				"title": "Reachable",
 				"enum": ["invalid", "unknown", "safe", "risky"],
-				"description": "An enumeration describing the confidence level that the recipient address is valid: `safe`, `risky`, `invalid`, or `unknown`. Refer to our FAQ for detailed definitions: https://docs.reacher.email/getting-started/is-reachable."
+				"description": "An enumeration describing the confidence level that the recipient address is valid: `safe`, `risky`, `invalid`, or `unknown`. Refer to our FAQ for detailed definitions: https://help.reacher.email/email-attributes-inside-json."
 			},
 			"MiscDetails": {
 				"title": "MiscDetails",
 				"type": "object",
 				"description": "Additional information about the email account.",
+				"required": ["is_disposable", "is_role_account", "is_b2c"],
 				"properties": {
 					"is_disposable": {
 						"type": "boolean",
@@ -516,9 +518,15 @@
 					"gravatar_url": {
 						"type": "string",
 						"description": "URL to the Gravatar profile picture associated with the email, if available and requested."
+					},
+					"is_b2c": {
+						"type": "boolean",
+						"x-stoplight": {
+							"id": "0cxn26qlxy8r4"
+						},
+						"description": "Is this a B2C email address?"
 					}
-				},
-				"required": ["is_disposable", "is_role_account"]
+				}
 			},
 			"MxDetails": {
 				"title": "MxDetails",
@@ -702,7 +710,7 @@
 					"duration": {
 						"$ref": "#/components/schemas/Duration"
 					},
-					"backend_name": {
+					"server_name": {
 						"type": "string",
 						"x-stoplight": {
 							"id": "2jrbdecvqh4t5"
@@ -717,7 +725,7 @@
 					"start_time",
 					"end_time",
 					"duration",
-					"backend_name",
+					"server_name",
 					"smtp"
 				]
 			},

diff --git a/backend/src/http/mod.rs b/backend/src/http/mod.rs
@@ -83,9 +83,9 @@ pub async fn run_warp_server(
 	// Run v0 bulk job listener.
 	let is_bulk_enabled = env::var("RCH_ENABLE_BULK").unwrap_or_else(|_| "0".into()) == "1";
 	let runner = if is_bulk_enabled {
-		let pg_pool = config
-			.get_pg_pool()
-			.expect("DATABASE_URL is required when RCH_ENABLE_BULK is set");
+		let pg_pool = config.get_pg_pool().expect(
+			"Please set the RCH__STORAGE__POSTGRES__DB_URL environment when RCH_ENABLE_BULK is set",
+		);
 		let runner = v0::bulk::create_job_registry(&pg_pool).await?;
 		Some(runner)
 	} else {

diff --git a/backend/src/http/v0/check_email/post.rs b/backend/src/http/v0/check_email/post.rs
@@ -67,6 +67,7 @@ impl CheckEmailRequest {
 				.unwrap_or_else(|| CheckEmailInput::default().smtp_port),
 			sentry_dsn: config.sentry_dsn.clone(),
 			backend_name: config.backend_name.clone(),
+			retries: 2,
 			..Default::default()
 		}
 	}

diff --git a/backend/tests/check_email.rs b/backend/tests/check_email.rs
@@ -23,8 +23,8 @@ mod tests {
 	use warp::http::StatusCode;
 	use warp::test::request;
 
-	const FOO_BAR_RESPONSE: &str = r#"{"input":"foo@bar","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":"","normalized_email":null,"suggestion":null}"#;
-	const FOO_BAR_BAZ_RESPONSE: &str = r#"{"input":"foo@bar.baz","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":"foo@bar.baz","domain":"bar.baz","is_valid_syntax":true,"username":"foo","normalized_email":"foo@bar.baz","suggestion":null}"#;
+	const FOO_BAR_RESPONSE: &str = r#"{"input":"foo@bar","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"is_b2c":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":"","normalized_email":null,"suggestion":null}"#;
+	const FOO_BAR_BAZ_RESPONSE: &str = r#"{"input":"foo@bar.baz","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"is_b2c":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":"foo@bar.baz","domain":"bar.baz","is_valid_syntax":true,"username":"foo","normalized_email":"foo@bar.baz","suggestion":null}"#;
 
 	fn create_backend_config(header_secret: &str) -> Arc<BackendConfig> {
 		let mut config = BackendConfig::empty();

diff --git a/cli/src/main.rs b/cli/src/main.rs
@@ -105,7 +105,8 @@ async fn main() -> Result<(), anyhow::Error> {
 		.hotmailb2c_verif_method(CONF.hotmailb2c_verif_method)
 		.check_gravatar(CONF.check_gravatar)
 		.haveibeenpwned_api_key(CONF.haveibeenpwned_api_key.clone())
-		.backend_name("reacher-cli".to_string());
+		.backend_name("reacher-cli".to_string())
+		.retries(2);
 
 	if let Some(proxy_host) = &CONF.proxy_host {
 		input = input.proxy(Some(CheckEmailInputProxy {

diff --git a/core/Cargo.toml b/core/Cargo.toml
@@ -27,7 +27,6 @@ futures = { version = "0.3.30" }
 hickory-proto = "0.24.0"
 hickory-resolver = "0.24.0"
 levenshtein = "1.0.5"
-lettre = { version = "0.11", features = ["smtp-transport"] }
 log = "0.4.22"
 mailchecker = "6.0.11"
 md5 = "0.7.0"