From 4bb42c6060859c7f161848ee56cf5fa0702d69ce Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Mon, 22 Jan 2024 17:06:54 -0700 Subject: [PATCH] test longitudinal snapshot --- .github/workflows/codecov.yml | 11 +- contrib/longitudinal_snapshot.py | 6 +- test/regression/CMakeLists.txt | 1 + test/regression/gufi_tool.py.in | 36 ++-- .../regression/longitudinal_snapshot.expected | 192 ++++++++++++++++++ test/regression/longitudinal_snapshot.sh.in | 106 ++++++++++ test/regression/setup.sh.in | 2 + 7 files changed, 326 insertions(+), 28 deletions(-) create mode 100644 test/regression/longitudinal_snapshot.expected create mode 100755 test/regression/longitudinal_snapshot.sh.in diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index d17824914..fe32d1d62 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -123,11 +123,12 @@ jobs: ${{ github.workspace }}/build/test/unit/python/test_gufi_config.py export GUFI_PYTHON_TEST_COVERAGE="coverage run -a" - (${{ github.workspace }}/build/test/regression/gufi_find.sh || true) > /dev/null 2>&1 - (${{ github.workspace }}/build/test/regression/gufi_getfattr.sh || true) > /dev/null 2>&1 - (${{ github.workspace }}/build/test/regression/gufi_ls.sh || true) > /dev/null 2>&1 - (${{ github.workspace }}/build/test/regression/gufi_stat.sh || true) > /dev/null 2>&1 - (${{ github.workspace }}/build/test/regression/gufi_stats.sh || true) > /dev/null 2>&1 + (${{ github.workspace }}/build/test/regression/gufi_find.sh || true) > /dev/null 2>&1 + (${{ github.workspace }}/build/test/regression/gufi_getfattr.sh || true) > /dev/null 2>&1 + (${{ github.workspace }}/build/test/regression/gufi_ls.sh || true) > /dev/null 2>&1 + (${{ github.workspace }}/build/test/regression/gufi_stat.sh || true) > /dev/null 2>&1 + (${{ github.workspace }}/build/test/regression/gufi_stats.sh || true) > /dev/null 2>&1 + (${{ github.workspace }}/build/test/regression/longitudinal_snapshot.sh || true) > /dev/null 2>&1 coverage xml --omit="${{ github.workspace }}/build/contrib/hashes.py,${{ github.workspace }}/build/contrib/performance/performance_pkg/tests/*,${{ github.workspace }}/build/test/*,${{ github.workspace }}/build/test/regression/*,${{ github.workspace }}/build/test/unit/python/*" rm .coverage # If not removed, codecov will attempt to run coverage xml which will not account for our omissions diff --git a/contrib/longitudinal_snapshot.py b/contrib/longitudinal_snapshot.py index 7d58f3860..503b7af25 100755 --- a/contrib/longitudinal_snapshot.py +++ b/contrib/longitudinal_snapshot.py @@ -121,11 +121,11 @@ def parse_args(argv, now): return parser.parse_args(argv[1:]) # pylint: disable=too-many-locals, too-many-statements -def run(argv): +def run(argv, config_path): timestamp = int(time.time()) args = parse_args(argv, timestamp) - config = gufi_config.Server(gufi_config.PATH) + config = gufi_config.Server(config_path) log2_size_bucket_count = math.ceil(math.log(args.max_size, 2)) log2_name_len_bucket_count = math.ceil(math.log(args.max_name_len, 2)) @@ -307,4 +307,4 @@ def run(argv): return 0 if __name__ == '__main__': - run(sys.argv) + sys.exit(run(sys.argv, gufi_config.PATH)) diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index c55d4c876..c918078cb 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -107,6 +107,7 @@ set(EXAMPLES set(OTHERS bash_completion + longitudinal_snapshot split_trace treediff ) diff --git a/test/regression/gufi_tool.py.in b/test/regression/gufi_tool.py.in index 4bfee0025..356b048a9 100755 --- a/test/regression/gufi_tool.py.in +++ b/test/regression/gufi_tool.py.in @@ -68,17 +68,24 @@ import sys # test config file name CONFIG_PATH = os.path.join('@CMAKE_CURRENT_BINARY_DIR@', 'config.test') +SCRIPTS = os.path.join('@CMAKE_BINARY_DIR@', 'scripts') + +TOOLS = { + 'find' : os.path.join(SCRIPTS, 'gufi_find'), + 'getfattr' : os.path.join(SCRIPTS, 'gufi_getfattr'), + 'ls' : os.path.join(SCRIPTS, 'gufi_ls'), + 'stat' : os.path.join(SCRIPTS, 'gufi_stat'), + 'stats' : os.path.join(SCRIPTS, 'gufi_stats'), + 'longitudinal_snapshot' : os.path.join('@CMAKE_BINARY_DIR@', 'contrib', 'longitudinal_snapshot.py'), +} + # import a tool by path (default: the scripts directory) # needed for duplicate filenames or files without the .py extension -def import_tool(tool, filename=None, - path=os.path.join('@CMAKE_BINARY_DIR@', 'scripts')): - if filename is None: - filename = tool - +def import_tool(tool, filename): if sys.version_info.major < 3: # pylint: disable=no-else-return import imp # pylint: disable=deprecated-module,import-outside-toplevel - return imp.load_source(tool, os.path.join(path, filename)) + return imp.load_source(tool, filename) # https://csatlas.com/python-import-file-module/ elif sys.version_info.major >= 3: # pylint: disable=import-outside-toplevel @@ -86,12 +93,8 @@ def import_tool(tool, filename=None, import importlib.util from pathlib import Path - # Get path to mymodule - script_dir = Path(__file__).parent - mymodule_path = str(script_dir.joinpath(path, filename)) - # Import mymodule - loader = importlib.machinery.SourceFileLoader(tool, mymodule_path) + loader = importlib.machinery.SourceFileLoader(tool, filename) spec = importlib.util.spec_from_loader(tool, loader) script = importlib.util.module_from_spec(spec) loader.exec_module(script) @@ -101,16 +104,9 @@ def import_tool(tool, filename=None, if __name__ == '__main__': parser = argparse.ArgumentParser(description='GUFI Testing Tool Selector', add_help=False) - parser.add_argument('tool', - choices=[ - 'find', - 'getfattr', - 'ls', - 'stat', - 'stats', - ]) + parser.add_argument('tool', choices=TOOLS.keys()) args, tool_args = parser.parse_known_args() - gufi_tool = import_tool('gufi_{0}'.format(args.tool)) + gufi_tool = import_tool(args.tool, TOOLS[args.tool]) sys.exit(gufi_tool.run([args.tool] + tool_args, CONFIG_PATH)) diff --git a/test/regression/longitudinal_snapshot.expected b/test/regression/longitudinal_snapshot.expected new file mode 100644 index 000000000..fc9ca257f --- /dev/null +++ b/test/regression/longitudinal_snapshot.expected @@ -0,0 +1,192 @@ +$ longitudinal_snapshot --help + [--max_name_len pos_int] [--notes text] + outname + +GUFI Longitudinal Snapshot Generator + +positional arguments: + outname output db file name + +options: + -h, --help show this help message and exit + --reftime seconds reference point for age (since UNIX epoch) + --max_size pos_int the maximum expected size + --max_name_len pos_int + the maximum expected length of a name/linkname + --notes text freeform text of any extra information to add to the + snapshot + +$ gufi_treesummary_all prefix + +$ longitudinal_snapshot longitudinal_snapshot.db --notes 'test notes' + +$ sqlite3 longitudinal_snapshot.db ".tables" +metadata snapshot treesummary + +$ sqlite3 longitudinal_snapshot.db "PRAGMA TABLE_INFO(metadata);" +0|timestamp|INT|0||0 +1|src|TEXT|0||0 +2|notes|TEXT|0||0 + +$ sqlite3 longitudinal_snapshot.db "PRAGMA TABLE_INFO(snapshot);" +0|name|TEXT|0||0 +1|inode|TEXT|0||0 +2|mode|INT64|0||0 +3|nlink|INT64|0||0 +4|uid|INT64|0||0 +5|gid|INT64|0||0 +6|blksize|INT64|0||0 +7|blocks|INT64|0||0 +8|atime|INT64|0||0 +9|mtime|INT64|0||0 +10|ctime|INT64|0||0 +11|depth|INT64|0||0 +12|filesystem_type|BLOB|0||0 +13|pinode|TEXT|0||0 +14|totfiles|INT64|0||0 +15|totlinks|INT64|0||0 +16|totsubdirs|INT64|0||0 +17|uid_min|INT64|0||0 +18|uid_max|INT64|0||0 +19|uid_hist|INT64|0||0 +20|uid_num_unique|INT64|0||0 +21|gid_min|INT64|0||0 +22|gid_max|INT64|0||0 +23|gid_hist|TEXT|0||0 +24|gid_num_unique|INT64|0||0 +25|size_min|INT64|0||0 +26|size_max|INT64|0||0 +27|size_mean|DOUBLE|0||0 +28|size_median|DOUBLE|0||0 +29|size_mode|TEXT|0||0 +30|size_stdev|DOUBLE|0||0 +31|size_sum|INT64|0||0 +32|size_hist|TEXT|0||0 +33|permissions_hist|TEXT|0||0 +34|ctime_min|INT64|0||0 +35|ctime_max|INT64|0||0 +36|ctime_mean|DOUBLE|0||0 +37|ctime_median|DOUBLE|0||0 +38|ctime_mode|TEXT|0||0 +39|ctime_stdev|DOUBLE|0||0 +40|ctime_hist|TEXT|0||0 +41|atime_min|INT64|0||0 +42|atime_max|INT64|0||0 +43|atime_mean|DOUBLE|0||0 +44|atime_median|DOUBLE|0||0 +45|atime_mode|TEXT|0||0 +46|atime_stdev|DOUBLE|0||0 +47|atime_hist|TEXT|0||0 +48|mtime_min|INT64|0||0 +49|mtime_max|INT64|0||0 +50|mtime_mean|DOUBLE|0||0 +51|mtime_median|DOUBLE|0||0 +52|mtime_mode|TEXT|0||0 +53|mtime_stdev|DOUBLE|0||0 +54|mtime_hist|TEXT|0||0 +55|crtime_min|INT64|0||0 +56|crtime_max|INT64|0||0 +57|crtime_mean|DOUBLE|0||0 +58|crtime_median|DOUBLE|0||0 +59|crtime_mode|TEXT|0||0 +60|crtime_stdev|DOUBLE|0||0 +61|crtime_hist|TEXT|0||0 +62|name_min|INT64|0||0 +63|name_max|INT64|0||0 +64|name_mean|DOUBLE|0||0 +65|name_median|DOUBLE|0||0 +66|name_mode|TEXT|0||0 +67|name_stdev|DOUBLE|0||0 +68|name_hist|TEXT|0||0 +69|linkname_min|INT64|0||0 +70|linkname_max|INT64|0||0 +71|linkname_mean|DOUBLE|0||0 +72|linkname_median|DOUBLE|0||0 +73|linkname_mode|TEXT|0||0 +74|linkname_stdev|DOUBLE|0||0 +75|linkname_hist|TEXT|0||0 +76|xattr_name_min|INT64|0||0 +77|xattr_name_max|INT64|0||0 +78|xattr_name_mean|DOUBLE|0||0 +79|xattr_name_median|DOUBLE|0||0 +80|xattr_name_mode|TEXT|0||0 +81|xattr_name_stdev|DOUBLE|0||0 +82|xattr_name_hist|TEXT|0||0 +83|xattr_value_min|INT64|0||0 +84|xattr_value_max|INT64|0||0 +85|xattr_value_mean|DOUBLE|0||0 +86|xattr_value_median|DOUBLE|0||0 +87|xattr_value_mode|TEXT|0||0 +88|xattr_value_stdev|DOUBLE|0||0 +89|xattr_value_hist|TEXT|0||0 +90|extensions_hist|TEXT|0||0 + +$ sqlite3 longitudinal_snapshot.db "PRAGMA TABLE_INFO(treesummary);" +0|inode|TEXT|0||0 +1|totsubdirs|INT64|0||0 +2|maxsubdirfiles|INT64|0||0 +3|maxsubdirlinks|INT64|0||0 +4|maxsubdirsize|INT64|0||0 +5|totfiles|INT64|0||0 +6|totlinks|INT64|0||0 +7|minuid|INT64|0||0 +8|maxuid|INT64|0||0 +9|mingid|INT64|0||0 +10|maxgid|INT64|0||0 +11|minsize|INT64|0||0 +12|maxsize|INT64|0||0 +13|totzero|INT64|0||0 +14|totltk|INT64|0||0 +15|totmtk|INT64|0||0 +16|totltm|INT64|0||0 +17|totmtm|INT64|0||0 +18|totmtg|INT64|0||0 +19|totmtt|INT64|0||0 +20|totsize|INT64|0||0 +21|minctime|INT64|0||0 +22|maxctime|INT64|0||0 +23|minmtime|INT64|0||0 +24|maxmtime|INT64|0||0 +25|minatime|INT64|0||0 +26|maxatime|INT64|0||0 +27|minblocks|INT64|0||0 +28|maxblocks|INT64|0||0 +29|totxattr|INT64|0||0 +30|depth|INT64|0||0 +31|mincrtime|INT64|0||0 +32|maxcrtime|INT64|0||0 +33|minossint1|INT64|0||0 +34|maxossint1|INT64|0||0 +35|totossint1|INT64|0||0 +36|minossint2|INT64|0||0 +37|maxossint2|INT64|0||0 +38|totossint2|INT64|0||0 +39|minossint3|INT64|0||0 +40|maxossint3|INT64|0||0 +41|totossint3|INT64|0||0 +42|minossint4|INT64|0||0 +43|maxossint4|INT64|0||0 +44|totossint4|INT64|0||0 +45|rectype|INT64|0||0 +46|uid|INT64|0||0 +47|gid|INT64|0||0 + +$ sqlite3 longitudinal_snapshot.db "SELECT COUNT(*) FROM metadata;" +1 + +$ sqlite3 longitudinal_snapshot.db "SELECT COUNT(*) FROM snapshot;" +6 + +$ sqlite3 longitudinal_snapshot.db "SELECT COUNT(*) FROM treesummary;" +5 + +$ sqlite3 longitudinal_snapshot.db "SELECT notes FROM metadata;" +test notes + +$ sqlite3 longitudinal_snapshot.db "SELECT s.name, s.size_sum, t.totsize, s.totfiles, t.totfiles FROM snapshot AS s, treesummary AS t WHERE s.inode == t.inode;" +directory|6|11|3|4 +leaf_directory|21|21|2|2 +prefix|1049622|1049668|5|12 +subdirectory|5|5|1|1 +unusual#? directory ,|14|14|1|1 + diff --git a/test/regression/longitudinal_snapshot.sh.in b/test/regression/longitudinal_snapshot.sh.in new file mode 100755 index 000000000..a9b90aa7a --- /dev/null +++ b/test/regression/longitudinal_snapshot.sh.in @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# This file is part of GUFI, which is part of MarFS, which is released +# under the BSD license. +# +# +# Copyright (c) 2017, Los Alamos National Security (LANS), LLC +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# From Los Alamos National Security, LLC: +# LA-CC-15-039 +# +# Copyright (c) 2017, Los Alamos National Security, LLC All rights reserved. +# Copyright 2017. Los Alamos National Security, LLC. This software was produced +# under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National +# Laboratory (LANL), which is operated by Los Alamos National Security, LLC for +# the U.S. Department of Energy. The U.S. Government has rights to use, +# reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR LOS +# ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR +# ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is +# modified to produce derivative works, such modified software should be +# clearly marked, so as not to confuse it with the version available from +# LANL. +# +# THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +# OF SUCH DAMAGE. + + + +set -e +source @CMAKE_CURRENT_BINARY_DIR@/setup.sh 1 + +OUTPUT="longitudinal_snapshot.out" + +SNAPSHOT="longitudinal_snapshot.db" + +cleanup() { + rm -rf "${SNAPSHOT}" +} + +cleanup_exit() { + cleanup + setup_cleanup +} + +trap cleanup_exit EXIT + +cleanup + +( +run_no_sort "${LONGITUDINAL_SNAPSHOT} --help" | replace_argparse +run_no_sort "${GUFI_TREESUMMARY_ALL} ${INDEXROOT}" | sed '/^Started .*$/d' + +run_no_sort "${LONGITUDINAL_SNAPSHOT} ${SNAPSHOT} --notes 'test notes'" + +run_no_sort "${SQLITE3} ${SNAPSHOT} \".tables\"" + +run_no_sort "${SQLITE3} ${SNAPSHOT} \"PRAGMA TABLE_INFO(metadata);\"" +run_no_sort "${SQLITE3} ${SNAPSHOT} \"PRAGMA TABLE_INFO(snapshot);\"" +run_no_sort "${SQLITE3} ${SNAPSHOT} \"PRAGMA TABLE_INFO(treesummary);\"" + +run_no_sort "${SQLITE3} ${SNAPSHOT} \"SELECT COUNT(*) FROM metadata;\"" +run_no_sort "${SQLITE3} ${SNAPSHOT} \"SELECT COUNT(*) FROM snapshot;\"" +run_no_sort "${SQLITE3} ${SNAPSHOT} \"SELECT COUNT(*) FROM treesummary;\"" + +run_no_sort "${SQLITE3} ${SNAPSHOT} \"SELECT notes FROM metadata;\"" + +run_sort "${SQLITE3} ${SNAPSHOT} \"SELECT s.name, s.size_sum, t.totsize, s.totfiles, t.totfiles FROM snapshot AS s, treesummary AS t WHERE s.inode == t.inode;\"" +) | tee "${OUTPUT}" + +@DIFF@ @CMAKE_CURRENT_BINARY_DIR@/longitudinal_snapshot.expected "${OUTPUT}" +rm "${OUTPUT}" diff --git a/test/regression/setup.sh.in b/test/regression/setup.sh.in index c228ed3cc..f3a96eb1e 100755 --- a/test/regression/setup.sh.in +++ b/test/regression/setup.sh.in @@ -165,6 +165,7 @@ GUFI_TRACE2INDEX="@CMAKE_BINARY_DIR@/src/gufi_trace2index" GUFI_TREESUMMARY="@CMAKE_BINARY_DIR@/src/gufi_treesummary" GUFI_TREESUMMARY_ALL="@CMAKE_BINARY_DIR@/src/gufi_treesummary_all" GUFI_UNROLLUP="@CMAKE_BINARY_DIR@/src/gufi_unrollup" +LONGITUDINAL_SNAPSHOT="${GUFI_TOOL} longitudinal_snapshot" OLDBIGFILES="@CMAKE_BINARY_DIR@/examples/oldbigfiles" QUERYDBS="@CMAKE_BINARY_DIR@/src/querydbs" SPLIT_TRACE="@CMAKE_BINARY_DIR@/contrib/split_trace" @@ -203,6 +204,7 @@ replace() { s/${GUFI_TREESUMMARY//\//\\/}/gufi_treesummary/g; s/${GUFI_TREESUMMARY_ALL//\//\\/}/gufi_treesummary_all/g; s/${GUFI_UNROLLUP//\//\\/}/gufi_unrollup/g; + s/${LONGITUDINAL_SNAPSHOT//\//\\/}/longitudinal_snapshot/g; s/${OLDBIGFILES//\//\\/}/oldbigfiles/g; s/${QUERYDBS//\//\\/}/querydbs/g; s/${SPLIT_TRACE//\//\\/}/split_trace/g;