From 8b4f82220037710bb73c8165efb2444bb15b617a Mon Sep 17 00:00:00 2001
From: Joshua Clayton <joshua.clayton@gmail.com>
Date: Sun, 24 May 2020 23:10:57 -0400
Subject: [PATCH] Calculate token usage in parallel via Rayon

What?
=====

This swaps token usage calculation with Rayon to further speed up
processing.

|            | before | after | speedup |
| Codebase 1 |    778 |   621 |     25% |
| Codebase 2 |   1214 |   985 |     23% |
| Codebase 3 |  37980 | 36950 |      3% |
|  discourse |   1309 |   949 |     38% |

All times are measured in milliseconds and benchmarks are run with
[bench](https://hackage.haskell.org/package/bench) against release mode.
---
 Cargo.lock                               | 1 +
 crates/token_analysis/Cargo.toml         | 3 ++-
 crates/token_analysis/src/token_usage.rs | 7 ++++---
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0a22903..a4bf1fe 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -704,6 +704,7 @@ dependencies = [
  "indicatif",
  "itertools",
  "project_configuration",
+ "rayon",
  "read_ctags",
  "serde",
  "serde_json",
diff --git a/crates/token_analysis/Cargo.toml b/crates/token_analysis/Cargo.toml
index 9d23fbb..778df47 100644
--- a/crates/token_analysis/Cargo.toml
+++ b/crates/token_analysis/Cargo.toml
@@ -13,7 +13,8 @@ project_configuration = { path = "../../crates/project_configuration/" }
 serde_json = "1.0.50"
 serde = { version = "1.0.105", features = ["derive"] }
 itertools = "0.9"
-indicatif = {version = "0.14"}
+rayon = "1.1"
+indicatif = {version = "0.14", features = ["with_rayon"]}
 
 [dev-dependencies]
 totems = "0.2.7"
diff --git a/crates/token_analysis/src/token_usage.rs b/crates/token_analysis/src/token_usage.rs
index af6f2b6..2b45d86 100644
--- a/crates/token_analysis/src/token_usage.rs
+++ b/crates/token_analysis/src/token_usage.rs
@@ -1,9 +1,10 @@
 use super::analysis_filter::{AnalysisFilter, OrderField, SortOrder};
 use super::occurrence_count::FileTypeCounts;
 use super::usage_likelihood::UsageLikelihood;
-use indicatif::ProgressIterator;
+use indicatif::ParallelProgressIterator;
 use itertools::{rev, Itertools};
 use project_configuration::ProjectConfiguration;
+use rayon::prelude::*;
 use serde::Serialize;
 use token_search::{TokenSearchConfig, TokenSearchResult, TokenSearchResults};
 
@@ -46,9 +47,9 @@ impl TokenUsageResults {
         let size = &unwrapped_results.len();
 
         let results = unwrapped_results
-            .into_iter()
+            .par_iter()
             .progress_with(token_search_config.toggleable_progress_bar("🧐 Analyzing...", *size))
-            .map(|r| TokenUsage::new(config, r))
+            .map(|r| TokenUsage::new(config, r.clone()))
             .collect::<Vec<_>>();
         TokenUsageResults(results)
     }