Add updated results

pomonam · Jul 15, 2024 · 9907f27 · 9907f27
1 parent c340602
commit 9907f27
Show file tree

Hide file tree

Showing 17 changed files with 17,385 additions and 6,135 deletions.
diff --git a/examples/openwebtext/README.md b/examples/openwebtext/README.md
diff --git a/examples/openwebtext/files/database.txt b/examples/openwebtext/files/database.txt
diff --git a/examples/openwebtext/files/scores_raw/ai.txt b/examples/openwebtext/files/scores_raw/ai.txt
diff --git a/examples/openwebtext/files/canada.txt → ...s/openwebtext/files/scores_raw/canada.txt b/examples/openwebtext/files/canada.txt → ...s/openwebtext/files/scores_raw/canada.txt
diff --git a/examples/openwebtext/files/scores_raw/cow.txt b/examples/openwebtext/files/scores_raw/cow.txt
diff --git a/examples/openwebtext/files/doctor.txt → ...s/openwebtext/files/scores_raw/doctor.txt b/examples/openwebtext/files/doctor.txt → ...s/openwebtext/files/scores_raw/doctor.txt
diff --git a/...s/openwebtext/files/factor_arguments.json → ...xt/files/scores_raw/factor_arguments.json b/...s/openwebtext/files/factor_arguments.json → ...xt/files/scores_raw/factor_arguments.json
@@ -5,13 +5,13 @@
     "amp_scale": 65536.0,
     "has_shared_parameters": false,
     "covariance_max_examples": 100000,
-    "covariance_data_partitions": 1,
+    "covariance_data_partitions": 4,
     "covariance_module_partitions": 2,
     "activation_covariance_dtype": "torch.bfloat16",
     "gradient_covariance_dtype": "torch.bfloat16",
     "eigendecomposition_dtype": "torch.float64",
     "lambda_max_examples": 100000,
-    "lambda_data_partitions": 1,
+    "lambda_data_partitions": 4,
     "lambda_module_partitions": 4,
     "use_iterative_lambda_aggregation": true,
     "offload_activations_to_cpu": true,

diff --git a/examples/openwebtext/files/inflation.txt → ...penwebtext/files/scores_raw/inflation.txt b/examples/openwebtext/files/inflation.txt → ...penwebtext/files/scores_raw/inflation.txt
diff --git a/examples/openwebtext/files/scores_raw/math.txt b/examples/openwebtext/files/scores_raw/math.txt
diff --git a/examples/openwebtext/files/ml.txt → examples/openwebtext/files/scores_raw/ml.txt b/examples/openwebtext/files/ml.txt → examples/openwebtext/files/scores_raw/ml.txt
diff --git a/...webtext/files/query_dataset_metadata.json → ...es/scores_raw/query_dataset_metadata.json b/...webtext/files/query_dataset_metadata.json → ...es/scores_raw/query_dataset_metadata.json
@@ -1,5 +1,5 @@
 {
     "type": "Dataset",
-    "dataset_size": 5,
+    "dataset_size": 10,
     "indices": null
 }
diff --git a/examples/openwebtext/files/scores_raw/science.txt b/examples/openwebtext/files/scores_raw/science.txt
diff --git a/...es/openwebtext/files/score_arguments.json → ...ext/files/scores_raw/score_arguments.json b/...es/openwebtext/files/score_arguments.json → ...ext/files/scores_raw/score_arguments.json
@@ -13,7 +13,7 @@
     "aggregate_train_gradients": false,
     "use_measurement_for_self_influence": false,
     "query_gradient_svd_dtype": "torch.float32",
-    "per_sample_gradient_dtype": "torch.bfloat16",
-    "precondition_dtype": "torch.bfloat16",
+    "per_sample_gradient_dtype": "torch.float32",
+    "precondition_dtype": "torch.float32",
     "score_dtype": "torch.bfloat16"
 }
diff --git a/...webtext/files/train_dataset_metadata.json → ...es/scores_raw/train_dataset_metadata.json b/...webtext/files/train_dataset_metadata.json → ...es/scores_raw/train_dataset_metadata.json
diff --git a/examples/openwebtext/files/scores_raw/water.txt b/examples/openwebtext/files/scores_raw/water.txt
diff --git a/examples/openwebtext/files/scores_raw/water_korean.txt b/examples/openwebtext/files/scores_raw/water_korean.txt
diff --git a/examples/openwebtext/inspect_scores.py b/examples/openwebtext/inspect_scores.py
@@ -11,15 +11,15 @@
 
 
 def main():
-    scores = Analyzer.load_file("influence_results/scores_jul_11_2024/pairwise_scores.safetensors")[
+    scores = Analyzer.load_file("influence_results/openwebtext/scores_raw/pairwise_scores.safetensors")[
         "all_modules"
     ].float()
 
     train_dataset = get_openwebtext_dataset()
     eval_dataset = get_custom_dataset()
     tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, use_fast=True, trust_remote_code=True)
 
-    eval_idx = 4
+    eval_idx = 5
     sorted_scores = torch.sort(scores[eval_idx], descending=True)
     top_indices = sorted_scores.indices
 
@@ -29,9 +29,7 @@ def main():
     plt.show()
 
     print("Query Sequence:")
-    print(
-        "Prompt: " + eval_dataset[eval_idx]["prompt"] + "; Completion: " + eval_dataset[eval_idx]["completion"] + "\n"
-    )
+    print("Prompt:" + eval_dataset[eval_idx]["prompt"] + "; Completion:" + eval_dataset[eval_idx]["completion"] + "\n")
 
     print("Top Influential Sequences:")
     for i in range(100):