Merge pull request #1145

Fix YAML syntax errors in example config
tenzir · Nov 9, 2020 · d5018fe · d5018fe
2 parents 9c693d6 + 54dc6c3
commit d5018fe
Show file tree

Hide file tree

Showing 9 changed files with 125 additions and 44 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,10 @@ Every entry has a category for which we use the following visual abbreviations:
 
 ## Unreleased
 
+- 🐞 The `vast.yaml.example` contained syntax errors. The example config file
+  now works again.
+  [#1145](https://github.com/tenzir/vast/pull/1145)
+
 - 🐞 VAST no longer starts if the specified config file does not exist.
   [#1147](https://github.com/tenzir/vast/pull/1147)
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -816,7 +816,10 @@ install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/schema"
         DESTINATION "${CMAKE_INSTALL_DATADIR}/vast")
 
 install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/integration"
-        DESTINATION "${CMAKE_INSTALL_DATADIR}/vast")
+        DESTINATION "${CMAKE_INSTALL_DATADIR}/vast"
+        PATTERN "integration/vast.yaml.example" EXCLUDE)
+install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/vast.yaml.example"
+        DESTINATION "${CMAKE_INSTALL_DATADIR}/vast/integration")
 
 install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE"
         DESTINATION "${CMAKE_INSTALL_DOCDIR}/")

diff --git a/integration/default_set.yaml b/integration/default_set.yaml
@@ -6,24 +6,35 @@ fixtures:
       node0 = Server(self.cmd,
                      ['-e', f':{VAST_PORT}', '-i', 'node0', 'start'],
                      work_dir, name='node0', port=VAST_PORT,
-                     config_file=f'{self.config_file}')
+                     config_file=self.config_file')
       node1 = Server(self.cmd,
                      ['-e', ':42124',
                       '-i', 'node1', 'start'],
                      work_dir, name='node1', port=42124,
-                     config_file=f'{self.config_file}')
+                     config_file=self.config_file)
       cmd += ['-e', f':{VAST_PORT}']
 
     exit: | # python
       node0.stop()
       node1.stop()
 
+  ExampleConfigFileTester:
+    enter: | # python
+      node = Server(self.cmd,
+                    ['-e', f':{VAST_PORT}', '-i', 'node', 'start'],
+                    work_dir, name='node', port=VAST_PORT,
+                    config_file='vast.yaml.example')
+      cmd += ['-e', f':{VAST_PORT}']
+
+    exit: | # python
+      node.stop()
+
   ServerTester:
     enter: | # python
       node = Server(self.cmd,
                     ['-e', f':{VAST_PORT}', '-i', 'node', 'start'],
                     work_dir, name='node', port=VAST_PORT,
-                    config_file=f'{self.config_file}')
+                    config_file=self.config_file)
       cmd += ['-e', f':{VAST_PORT}']
 
     exit: | # python
@@ -38,7 +49,7 @@ fixtures:
                      '--aging-query=:addr == 192.168.1.104',
                      'start'],
                     work_dir, name='node', port=VAST_PORT,
-                    config_file=f'{self.config_file}')
+                    config_file=self.config_file)
       cmd += ['-e', f':{VAST_PORT}']
 
     exit: | # python
@@ -59,7 +70,7 @@ fixtures:
                      '--disk-budget-high=5MiB',
                      '--disk-budget-low=2MiB'],
                     work_dir, name='node', port=VAST_PORT,
-                    config_file=f'{self.config_file}')
+                    config_file=self.config_file)
       cmd += ['-e', f':{VAST_PORT}']
 
     exit: | # python
@@ -397,6 +408,14 @@ tests:
       - command: import -b zeek
         input: data/zeek/conn.log.gz
       - command: count '#type ~ /suricata.*/'
+  Example config file:
+    tags: [import-export, zeek]
+    fixture: ExampleConfigFileTester
+    config-file: vast.yaml.example
+    steps:
+      - command: import -b zeek
+        input: data/zeek/conn.log.gz
+      - command: export ascii 'net.app !in ["dns", "ftp", "http", "ssl"]'
   # The idea is import just enough to hit the high-water mark, wait, and then
   # verify that most of the partitions and segments were deleted by checking
   # the number of events left in the database. This test is a bit annoying as

diff --git a/integration/integration.py b/integration/integration.py
@@ -34,6 +34,7 @@
 VAST_PORT = 42024
 STEP_TIMEOUT = 30
 CURRENT_SUBPROCS: List[subprocess.Popen] = []
+SET_DIR = Path()
 
 
 class Fixture(NamedTuple):
@@ -54,6 +55,7 @@ class Condition(NamedTuple):
 class Test(NamedTuple):
     tags: Optional[str]
     condition: Optional[Condition]
+    config_file: Optional[str]
     fixture: Optional[str]
     steps: List[Union[Step, Condition]]
 
@@ -309,19 +311,26 @@ def __init__(
         **kwargs,
     ):
         self.app = app
-        self.config_arg = f"--config={config_file}"
+        if config_file:
+            self.config_arg = f"--config={SET_DIR/config_file}"
+        else:
+            self.config_arg = None
         self.name = name
         self.cwd = work_dir / self.name
         self.port = port
-        LOGGER.debug("starting server fixture")
+        command = [self.app]
+        if self.config_arg:
+            command.append(self.config_arg)
+        command = command + args
+        LOGGER.info(f"starting server fixture: {command}")
         LOGGER.debug(f"waiting for port {self.port} to be available")
         if not wait.tcp.closed(self.port, timeout=5):
             raise RuntimeError("Port is blocked by another process.\nAborting tests...")
         self.cwd.mkdir(parents=True)
         out = open(self.cwd / "out", "w")
         err = open(self.cwd / "err", "w")
         self.process = spawn(
-            [self.app, self.config_arg] + args,
+            command,
             cwd=self.cwd,
             stdout=out,
             stderr=err,
@@ -333,13 +342,17 @@ def __init__(
 
     def stop(self):
         """Stops the server"""
-        LOGGER.debug("stopping server fixture")
+        command = [self.app]
+        if self.config_arg:
+            command.append(self.config_arg)
+        command = command + ["-e", f":{self.port}", "stop"]
+        LOGGER.debug(f"stopping server fixture: {command}")
         stop_out = open(self.cwd / "stop.out", "w")
         stop_err = open(self.cwd / "stop.err", "w")
         stop = 0
         try:
             stop = spawn(
-                [self.app, self.config_arg, "-e", f":{self.port}", "stop"],
+                command,
                 cwd=self.cwd,
                 stdout=stop_out,
                 stderr=stop_err,
@@ -410,7 +423,9 @@ def run(self, test_name, test):
         dummy_fixture = Fixture("pass", "pass")
         fixture = dummy_fixture if not test.fixture else self.fixtures.get(test.fixture)
         cmd = [self.cmd]
-        if self.config_file:
+        if test.config_file:
+            cmd.append(f"--config={test.config_file}")
+        elif self.config_file:
             cmd.append(f"--config={self.config_file}")
         fenter = Template(fixture.enter).substitute(locals())
         fexit = Template(fixture.exit).substitute(locals())
@@ -442,7 +457,7 @@ def run(self, test_name, test):
         return summary.dominant_state()
 
 
-def validate(data, set_dir):
+def validate(data):
     def is_file(path):
         return path.is_file()
 
@@ -456,16 +471,17 @@ def guard_to_condition(guard):
         return Condition(guard["guard"])
 
     def to_test(data):
+        data["config_file"] = data.pop("config-file")
         return Test(**data)
 
     def absolute_path(path):
         absolute = Path(os.path.expanduser(path))
         if not absolute.is_absolute():
-            absolute = (set_dir / path).resolve()
+            absolute = (SET_DIR / path).resolve()
         return absolute
 
     def replace_path(raw_command):
-        return raw_command.replace("@.", str(set_dir))
+        return raw_command.replace("@.", str(SET_DIR))
 
     def to_command(raw_command):
         return shlex.split(replace_path(raw_command))
@@ -499,6 +515,7 @@ def to_command(raw_command):
             {
                 schema.Optional("tags", default=None): [str],
                 schema.Optional("condition", default=None): schema.Use(Condition),
+                schema.Optional("config-file", default=None): schema.Use(absolute_path),
                 schema.Optional("fixture", default=None): str,
                 "steps": [schema.Or(step, guard)],
             },
@@ -660,9 +677,11 @@ def emit(self, record):
         args.set = Path(__file__).resolve().parent / "default_set.yaml"
     args.set = args.set.resolve()
     LOGGER.debug(f"resolved test set path to {args.set}")
+    global SET_DIR
+    SET_DIR = args.set.parent
     test_file = open(args.set, "r")
     test_dict = yaml.full_load(test_file)
-    test_dec = validate(test_dict, args.set.parent)
+    test_dec = validate(test_dict)
     # Print tests.
     if args.list is not None:
         selection = tagselect(args.list, test_dec["tests"])

diff --git a/integration/reference/example-config-file/step_00.ref b/integration/reference/example-config-file/step_00.ref
diff --git a/integration/reference/example-config-file/step_01.ref b/integration/reference/example-config-file/step_01.ref
@@ -0,0 +1,23 @@
+<2009-11-18T16:59:33.941668096, "QqEBsEl5zOf", 192.168.1.103, 1397/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 2.65s, 2943, 1440, "SF", nil, 0, "ShAdDafF", 19, 3711, 30, 2648, []>
+<2009-11-18T17:23:50.304653056, "E8oxyJ4sSf4", 192.168.1.103, 1749/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 115.93ms, 5679, 1405, "SF", nil, 0, "ShAdDafF", 19, 6447, 30, 2613, []>
+<2009-11-18T17:26:14.919133952, "tkT6ftxDPmg", 192.168.1.103, 1755/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 118.43ms, 2357, 1405, "SF", nil, 0, "ShAdDafF", 17, 3045, 28, 2533, []>
+<2009-11-18T17:37:00.302089984, "PK1psr2Dg68", 192.168.1.105, 49218/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 1.6s, 4899, 1519, "SF", nil, 0, "ShAadDfF", 18, 5631, 29, 2691, []>
+<2009-11-18T17:38:00.189256960, "U6EoRx2yUyl", 192.168.1.105, 49219/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 22.98ms, 4429, 799, "SF", nil, 0, "ShAadDfF", 18, 5161, 30, 2011, []>
+<2009-11-18T17:40:50.124592128, "3cN64Zhy0Cf", 192.168.1.105, 49220/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 114.18ms, 2750, 1466, "SF", nil, 0, "ShAadDfF", 16, 3402, 27, 2558, []>
+<2009-11-18T17:42:03.706407936, "zkdnQLD4rx7", 192.168.1.103, 1806/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 33.24ms, 4371, 1370, "SF", nil, 0, "ShAdDafF", 17, 5059, 28, 2498, []>
+<2009-11-18T17:58:29.383510016, "6uaqOCphs9g", 192.168.1.102, 1400/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 4.19s, 1522, 1370, "SF", nil, 0, "ShAdDafF", 16, 2170, 26, 2418, []>
+<2009-11-18T18:00:48.261595904, "oAiWgG5cMng", 192.168.1.102, 1404/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 113.17ms, 5247, 1370, "SF", nil, 0, "ShAdDafF", 18, 5975, 28, 2498, []>
+<2009-11-18T18:01:29.262108928, "GxYIFhoLgza", 192.168.1.102, 1405/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 21.48ms, 5796, 632, "SF", nil, 0, "ShAdDafF", 19, 6564, 28, 1760, []>
+<2009-11-18T18:09:17.457758976, "3c03Zu0ccbh", 192.168.1.105, 49336/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 115.17ms, 2430, 1466, "SF", nil, 0, "ShAadDfF", 16, 3082, 27, 2558, []>
+<2009-11-18T18:13:56.508357888, "lhnlnKqXCxf", 192.168.1.105, 49353/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 111.93ms, 2462, 1466, "SF", nil, 0, "ShAadDfF", 16, 3114, 27, 2558, []>
+<2009-11-18T18:14:19.128662016, "ufFpSyrwjwd", 192.168.1.103, 1836/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 31.99ms, 3372, 1370, "SF", nil, 0, "ShAdDafF", 16, 4020, 26, 2418, []>
+<2009-11-18T18:24:27.549041152, "mpHi6R9SU9i", 192.168.1.102, 1709/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 113.92ms, 2374, 1370, "SF", nil, 0, "ShAdDafF", 16, 3022, 26, 2418, []>
+<2009-11-18T18:25:38.108254976, "Rdz0JuTWH7h", 192.168.1.105, 49561/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 32.98ms, 4334, 1466, "SF", nil, 0, "ShAadDfF", 17, 5026, 28, 2598, []>
+<2009-11-18T19:55:41.027461888, "K1ETvrXVgr", 192.168.1.104, 1572/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 438.74ms, 1021, 308, "SF", nil, 0, "ShAdDafF", 10, 1429, 17, 996, []>
+<2009-11-18T20:51:24.692600064, "4JRIv0bBLY5", 192.168.1.104, 1604/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 279.07ms, 761, 308, "SF", nil, 0, "ShAdDafF", 10, 1169, 17, 996, []>
+<2009-11-18T20:57:20.949762048, "iZiq8ubpoz5", 192.168.1.104, 1665/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 254.84ms, 574, 308, "SF", nil, 0, "ShAdDafF", 10, 982, 17, 996, []>
+<2009-11-18T22:02:37.587842816, "9ZNqUeBHGZg", 192.168.1.103, 1934/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 770.76ms, 5732, 1370, "SF", nil, 0, "ShAdDafF", 18, 6460, 27, 2458, []>
+<2009-11-18T22:08:27.588229888, "PbX5YfEjOJg", 192.168.1.103, 2008/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 1.23s, 3460, 1370, "SF", nil, 0, "ShAdDafF", 17, 4148, 26, 2418, []>
+<2009-11-18T22:33:34.298059008, "IcCYHXxqpC5", 192.168.1.102, 1911/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 1.03s, 7079, 1370, "SF", nil, 0, "ShAdDafF", 20, 7887, 29, 2538, []>
+<2009-11-19T01:28:01.990388992, "4nUkGQ2lYSi", 192.168.1.105, 49307/tcp, 143.166.11.10, 64053/tcp, "tcp", "ftp-data", 41.39s, 0, 4255056, "RSTO", nil, 178020, "ShAdfR", 1453, 63572, 2965, 4198407, []>
+<2009-11-19T01:29:23.408284928, "EQbLqHeggqj", 192.168.1.105, 49330/tcp, 143.166.11.10, 64334/tcp, "tcp", "ftp-data", 27.61s, 0, 4255056, "RSTO", nil, 0, "ShAdfR", 1514, 67868, 3101, 4392906, []>
diff --git a/integration/vast.yaml.example b/integration/vast.yaml.example
@@ -0,0 +1 @@
+../vast.yaml.example
diff --git a/libvast/src/system/spawn_index.cpp b/libvast/src/system/spawn_index.cpp
@@ -37,12 +37,12 @@ maybe_actor spawn_index(node_actor* self, spawn_arguments& args) {
   if (!fs)
     return make_error(ec::lookup_error, "failed to find filesystem actor");
   namespace sd = vast::defaults::system;
-  auto handle
-    = self->spawn(index, fs, args.dir / args.label,
-                  opt("vast.max-partition-size", sd::max_partition_size),
-                  opt("vast.in-mem-partitions", sd::max_in_mem_partitions),
-                  opt("vast.taste-partitions", sd::taste_partitions),
-                  opt("vast.query-supervisors", sd::num_query_supervisors));
+  auto handle = self->spawn(
+    index, fs, args.dir / args.label,
+    opt("vast.max-partition-size", sd::max_partition_size),
+    opt("vast.max-resident-partitions", sd::max_in_mem_partitions),
+    opt("vast.max-taste-partitions", sd::taste_partitions),
+    opt("vast.max-queries", sd::num_query_supervisors));
   VAST_VERBOSE(self, "spawned the index");
   if (auto accountant = self->state.registry.find_by_label("accountant"))
     self->send(handle, caf::actor_cast<accountant_type>(accountant));