better data testing

.gitignore

@@ -164,6 +164,7 @@ output-logs
 models
 logs
 datasets
+./tests
 
 *.hdf5
 *.no

impsy/dataset.py

@@ -22,7 +22,7 @@ def transform_log_to_sequence_example(logfile: str, dimension: int):
     return np.array(perf_df[["t"] + data_names])
 
 
-def generate_dataset(dimension: int, source: str):
+def generate_dataset(dimension: int, source: str = "logs", destination: str = "datasets"):
     """Generate a dataset from .log files in the log directory."""
     # Load up the performances
     log_location = f"{source}/"
@@ -41,7 +41,7 @@ def generate_dataset(dimension: int, source: str):
                 print("Processing failed for", local_file)
 
     # Save Performance Data in a compressed numpy file.
-    dataset_location = "datasets/"
+    dataset_location = destination + "/"
     dataset_filename = "training-dataset-" + str(dimension) + "d.npz"
 
     # Input format is:

impsy/tests/test_data.py

@@ -5,8 +5,9 @@
 
 def test_log_to_examples():
   """Tests transform_log_to_sequence_example with a single example"""
-  test_log_file = "logs/2024-06-01T12-00-00-4d-mdrnn.log"
+  test_log_file = "tests/logs/2024-06-01T12-00-00-4d-mdrnn.log"
   test_line = "2024-06-01T12:00:00,interface,0.1,0.2,0.3,0.4"
+  os.makedirs("tests/logs", exist_ok=True)
   with open(test_log_file, "w") as file:
     file.write(test_line)
   log = dataset.transform_log_to_sequence_example(test_log_file, 4)
@@ -16,5 +17,12 @@ def test_log_to_examples():
 
 def test_dataset_command():
   """Test the dataset command runs"""
-  print("Testing dataset function...")
-  dataset.generate_dataset(dimension=2, source="logs")
+  test_log_file = "tests/logs/2024-06-01T12-00-00-4d-mdrnn.log"
+  test_line = "2024-06-01T12:00:00,interface,0.1,0.2,0.3,0.4"
+  os.makedirs("tests/logs", exist_ok=True)
+  os.makedirs("tests/datasets", exist_ok=True)
+  with open(test_log_file, "w") as file:
+    file.write(test_line)
+  dataset.generate_dataset(dimension=2, source="tests/logs", destination="tests/datasets")
+  os.remove(test_log_file)
+