test script cleanup

|& -> 2>&1 |
remove realpath from generatetree
stat format flag
remove whitespace from wc -l output in gufi_dir2trace
change getfattr to os_getfattr
arparse help and description formatters

CMakeLists.txt

@@ -245,6 +245,7 @@ if(APPLE)
   find_exec_dependency(XATTR xattr)
 else()
   find_exec_dependency(SETFATTR setfattr)
+  find_exec_dependency(GETFATTR getfattr)
 endif()
 
 # Need the PkgConfig module for pkg_search_module

scripts/gufi_common.py

@@ -129,6 +129,12 @@
 UID     = 'uid'
 GID     = 'gid'
 
+# ArgumentParser formatter subclass
+# https://stackoverflow.com/a/67364090/341683
+class ArgParseFormatter(argparse.RawTextHelpFormatter):
+    def __init__(self, prog):
+        super().__init__(prog, width=80)
+
 # ###############################################
 # useful functions for using in ArgumentParser.add_argument(type=function_name)
 # to make sure input fits certain criteria

scripts/gufi_config.py

@@ -196,7 +196,8 @@ def port(self):
 
 def run(args):
     # simple config validator
-    parser = argparse.ArgumentParser(description='GUFI Configuration Tester')
+    parser = argparse.ArgumentParser(description='GUFI Configuration Tester',
+                                     formatter_class=gufi_common.ArgParseFormatter)
     parser.add_argument('type', choices=['server', 'client'])
     parser.add_argument('path')
     args = parser.parse_args(args)

scripts/gufi_find

@@ -519,7 +519,8 @@ def help(parser): # pylint: disable=redefined-builtin
 
 def build_expression_parser():
     # parse the arguments
-    parser = argparse.ArgumentParser('gufi_find', description='GUFI version of find', add_help=False)
+    parser = argparse.ArgumentParser('gufi_find', description='GUFI version of find', add_help=False,
+                                     formatter_class=gufi_common.ArgParseFormatter)
 
     # override help to not use -h
     parser.add_argument('-help', '--help',

scripts/gufi_getfattr

@@ -164,6 +164,12 @@ def getfattr(args, dirname, nondirname):
 
     return query_args + [dirname]
 
+# https://stackoverflow.com/a/67364090/341683
+class RawHelpFormatter(gufi_common.ArgParseFormatter,
+                       argparse.RawDescriptionHelpFormatter):
+    def __init__(self, prog):
+        super(RawHelpFormatter, self).__init__(prog)
+
 def run(argv, config_path):
     # pylint: disable=invalid-name
 
@@ -174,7 +180,7 @@ def run(argv, config_path):
     parser = argparse.ArgumentParser(
         'gufi_getfattr',
         description='GUFI version of getfattr\n\nNote: --no-dereference/-h is always set',
-        add_help=False, formatter_class=argparse.RawDescriptionHelpFormatter)
+        add_help=False, formatter_class=RawHelpFormatter)
 
     # override help to not use -h
     parser.add_argument('--help',

scripts/gufi_ls

@@ -219,7 +219,8 @@ def run(argv, config_path):
     config = gufi_config.Server(config_path)
 
     # parse the arguments
-    parser = argparse.ArgumentParser('gufi_ls', description='GUFI version of ls', add_help=False)
+    parser = argparse.ArgumentParser('gufi_ls', description='GUFI version of ls', add_help=False,
+                                     formatter_class=gufi_common.ArgParseFormatter)
     # override help to not use -h
     parser.add_argument('--help',
                         action='help',

scripts/gufi_stat

@@ -74,7 +74,8 @@ PATH = os.path.realpath(__file__)
 
 def parse_args():
     # parse the arguments
-    parser = argparse.ArgumentParser('gufi_stat', description='GUFI version of stat', add_help=False)
+    parser = argparse.ArgumentParser('gufi_stat', description='GUFI version of stat', add_help=False,
+                                     formatter_class=gufi_common.ArgParseFormatter)
 
     # parser.add_argument('-L', '--dereference'
     #                     action='set_true',

scripts/gufi_stats

@@ -1125,7 +1125,8 @@ def run(argv, config_path):
     config = gufi_config.Server(config_path)
 
     # parse the arguments
-    parser = argparse.ArgumentParser('gufi_stats', description='GUFI statistics', add_help=False)
+    parser = argparse.ArgumentParser('gufi_stats', description='GUFI statistics', add_help=False,
+                                     formatter_class=gufi_common.ArgParseFormatter)
 
     # override help to not use -h
     parser.add_argument('--help',

scripts/querydbs

@@ -69,7 +69,8 @@ import gufi_common
 import gufi_config
 
 def parse_args(argv):
-    parser = argparse.ArgumentParser('querydbs', description='Query GUFI result dbs')
+    parser = argparse.ArgumentParser('querydbs', description='Query GUFI result dbs',
+                                     formatter_class=gufi_common.ArgParseFormatter)
 
     parser.add_argument('table_name',
                         help='name of table in database file to attach; also used for view name: \'v<table_name>\'')

test/regression/CMakeLists.txt

@@ -63,7 +63,7 @@
 # regression test specific files
 set(CORE
   generatetree.sh
-  os_setfattr.sh
+  os_xattr.sh
   setup.sh
 )
 

test/regression/external_databases.sh.in

@@ -276,7 +276,7 @@ rollup() {
     do_tests
     rollup
     do_tests
-) |& tee "${OUTPUT}"
+) 2>&1 | tee "${OUTPUT}"
 
 @DIFF@ @CMAKE_CURRENT_BINARY_DIR@/external_databases.expected "${OUTPUT}"
 rm "${OUTPUT}"

test/regression/generatetree.sh.in

@@ -96,9 +96,9 @@ then
     return 1
 fi
 
-DIR="$(realpath $1)"
+DIR="$1"
 
-source @CMAKE_CURRENT_BINARY_DIR@/os_setfattr.sh
+source @CMAKE_CURRENT_BINARY_DIR@/os_xattr.sh
 
 rm -rf "${DIR}"
 mkdir -p "${DIR}"
@@ -186,4 +186,5 @@ mkdir "unusual#? directory ,"
 )
 touch -amt 197001010000.16 "unusual#? directory ,"
 
+cd ..
 touch -amt 197001010000.17 "${DIR}"

test/regression/gufi_dir2trace.sh.in

@@ -95,15 +95,16 @@ echo "/external/db" > "${SRCDIR}/external.gufi"
 run_no_sort "${GUFI_DIR2TRACE} -d \"${DELIM}\" -n ${THREADS} -x \"${SRCDIR}\" \"${TRACE}\""
 
 # count output files
-found=$(find "${TRACES[@]}" | wc -l)
+# shellcheck disable=SC2016
+found=$(find "${TRACES[@]}" | wc -l | @AWK@ '{ print $1 }')
 echo "Expecting ${THREADS} trace files. Found ${found}."
 
 # count lines
 # shellcheck disable=SC2016
-trace_lines=$(cat "${TRACES[@]}" | wc -l | @AWK@ '{print $1 - 1}')
+trace_lines=$(cat "${TRACES[@]}" | wc -l | @AWK@ '{ print $1 - 1} ')
 contents=$(find "${SRCDIR}")
 # shellcheck disable=SC2016
-content_lines=$(echo "${contents}" | wc -l | @AWK@ '{print $1}')
+content_lines=$(echo "${contents}" | wc -l | @AWK@ '{ print $1 }')
 echo "Expecting ${content_lines} lines. Got ${trace_lines}."
 
 # count separators

test/regression/gufi_find.expected

@@ -706,7 +706,7 @@ $ gufi_find -printf "%% %a %b %c %d %k %g %u %D %f %G %h %i %m %M %n %p %P %s %t
 % 1024 0 1024 2 0 0 0 - 1KB 0 prefix/1KB 0 664 -rw-rw-r-- 1 prefix/1KB prefix/1KB 1024 1024 0 f
 % 1048576 0 1048576 2 0 0 0 - 1MB 0 prefix/1MB 0 664 -rw-rw-r-- 1 prefix/1MB prefix/1MB 1048576 1048576 0 f
 
-$ gufi_find -empty path_after_expression |& grep "RuntimeError:"
+$ gufi_find -empty path_after_expression 2>&1 | grep "RuntimeError:"
 RuntimeError: gufi_find: unknown predicate `path_after_expression'
 
 $ gufi_find -atime abc
@@ -723,7 +723,7 @@ usage: gufi_find [-help] [-maxdepth levels] [-mindepth levels] [--version]
                  [--aggregate-name name] [--skip-file filename] [--verbose]
 gufi_find: error: argument -atime: abc is not a valid numeric argument
 
-$ gufi_find -unknown-predicate |& grep "RuntimeError:"
+$ gufi_find -unknown-predicate 2>&1 | grep "RuntimeError:"
 RuntimeError: gufi_find: unknown predicate `-unknown-predicate'
 
 $ gufi_find --skip "skip"

test/regression/gufi_find.sh.in

@@ -181,13 +181,13 @@ run_no_sort "${GUFI_QUERY} -a -w -E \"UPDATE entries SET inode = 0, uid = 0, gid
 run_no_sort "${GUFI_FIND} -printf \"%% %a %b %c %d %k %g %u %D %f %G %h %i %m %M %n %p %P %s %t %U %y\" -type f" | @AWK@ '{ if ((NR > 1) && (NF > 8)) { $3=0; $6=0; $7=0; $8=0; } print }' | sort -n -k 2; echo
 
 # path must come before expressions
-run_no_sort "${GUFI_FIND} -empty path_after_expression |& @GREP@ \"RuntimeError:\""
+run_no_sort "${GUFI_FIND} -empty path_after_expression 2>&1 | @GREP@ \"RuntimeError:\""
 
 # bad numerical input
 run_no_sort "${GUFI_FIND} -atime abc"
 
 # unknown predicate
-run_no_sort "${GUFI_FIND} -unknown-predicate |& @GREP@ \"RuntimeError:\""
+run_no_sort "${GUFI_FIND} -unknown-predicate 2>&1 | @GREP@ \"RuntimeError:\""
 
 # skip
 run_sort "${GUFI_FIND} --skip \"${SKIP}\""

test/regression/gufi_getfattr.expected

@@ -1,9 +1,8 @@
 $ gufi_getfattr --help
 usage: gufi_getfattr [--help] [--version] [--name NAME] [--dump]
-                     [--match PATTERN] [--only-values] [--recursive]
-                     [--delim c] [--in-memory-name name]
-                     [--aggregate-name name] [--skip-file filename]
-                     [--verbose]
+                     [--match PATTERN] [--only-values] [--recursive] [--delim c]
+                     [--in-memory-name name] [--aggregate-name name]
+                     [--skip-file filename] [--verbose]
                      path [path ...]
 
 GUFI version of getfattr
@@ -17,23 +16,14 @@ options:
   --help                show this help message and exit
   --version, -v         show program's version number and exit
   --name NAME, -n NAME  Dump the values of the named extended attribute
-  --dump, -d            Dump the values of all extended attributes associated
-                        with pathname.
+  --dump, -d            Dump the values of all extended attributes associated with pathname.
   --match PATTERN, -m PATTERN
-                        Only include attributes with names matching the
-                        regular expression pattern. The default value for
-                        pattern is "^user\.", which includes all the
-                        attributes in the user namespace. Specify "-" for
-                        including all attributes". Refer to attr(5) for a more
-                        detailed discussion of namespaces.
-  --only-values         Dump out the raw extended attribute value(s) without
-                        encoding them.
-  --recursive, -R       List the atttributes of all files and directories
-                        recursively.
+                        Only include attributes with names matching the regular expression pattern. The default value for pattern is "^user\.", which includes all the attributes in the user namespace. Specify "-" for including all attributes". Refer to attr(5) for a more detailed discussion of namespaces.
+  --only-values         Dump out the raw extended attribute value(s) without encoding them.
+  --recursive, -R       List the atttributes of all files and directories recursively.
   --delim c             delimiter separating output columns
   --in-memory-name name
-                        Name of in-memory database when aggregation is
-                        performed
+                        Name of in-memory database when aggregation is performed
   --aggregate-name name
                         Name of final database when aggregation is performed
   --skip-file filename  Name of file containing directory basenames to skip

test/regression/gufi_index2dir.expected

@@ -17,7 +17,6 @@ $ gufi_index2dir -n 2 -x "prefix" "reconstructed"
 $ diff <(find "prefix" | sort) <(find reconstructed/* | sed "s/reconstructed\///g" | sort)
 
 # Diff xattrs
-$ diff <(find "prefix" -exec getfattr -m ".*" -d {} \; | sort) <(find reconstructed/* -exec getfattr -m ".*" -d {} \; | sed "s/reconstructed\///g" | sort)
 
 $ gufi_index2dir -n 2 "prefix" "reconstructed/prefix/old_file"
 "reconstructed/prefix/old_file" Already exists!

test/regression/gufi_index2dir.sh.in

@@ -63,6 +63,7 @@
 
 set -e
 source @CMAKE_CURRENT_BINARY_DIR@/setup.sh 1
+source @CMAKE_CURRENT_BINARY_DIR@/os_xattr.sh
 
 OUTPUT="gufi_index2dir.out"
 DST="reconstructed"
@@ -90,7 +91,26 @@ echo "# Diff Trees"
 run_no_sort "diff <(find \"${SRCDIR}\" | sort) <(find ${DST}/* | sed \"s/${DST//\//\\/}\\///g\" | sort)"
 
 echo "# Diff xattrs"
-run_no_sort "diff <(find \"${SRCDIR}\" -exec getfattr -m \".*\" -d {} \; | sort) <(find ${DST}/* -exec getfattr -m \".*\" -d {} \; | sed \"s/${DST//\//\\/}\\///g\" | sort)"
+src=$(find "${SRCDIR}" -print0 | while IFS= read -r -d '' path
+      do
+          os_getfattr "${path}"
+      done | sort)
+dst=$(find "${DST}" -mindepth 1 -print0 | while IFS= read -r -d '' path
+      do
+          os_getfattr "${path}" | sed "s/${DST//\//\\/}\\///g"
+      done | sort)
+if [[ "${#src}" -eq 0 ]]
+then
+    echo "Did not find any xattrs in ${SRCDIR}"
+    exit 1
+fi
+if [[ "${#dst}" -eq 0 ]]
+then
+    echo "Did not find any xattrs in ${DST}"
+    exit 1
+fi
+diff <(echo "${src}") <(echo "${dst}")
+echo
 
 # bad inputs
 

test/regression/gufi_ls.expected

@@ -33,8 +33,7 @@ options:
   -t                    sort by modification time, newest first
   --delim c             delimiter separating output columns
   --in-memory-name name
-                        Name of in-memory database when aggregation is
-                        performed
+                        Name of in-memory database when aggregation is performed
   --aggregate-name name
                         Name of final database when aggregation is performed
   --skip-file filename  Name of file containing directory basenames to skip

test/regression/gufi_ls.sh.in

@@ -155,7 +155,7 @@ run_sort "${GUFI_LS} --skip \"${SKIP}\" ${BASENAME}"
 
 # bad skip
 run_no_sort "${GUFI_LS} --skip \"${SKIP}.bad\" ${BASENAME}"
-) |& tee "${OUTPUT}"
+) 2>&1 | tee "${OUTPUT}"
 
 @DIFF@ @CMAKE_CURRENT_BINARY_DIR@/gufi_ls.expected "${OUTPUT}"
 rm "${OUTPUT}"

test/regression/gufi_query.expected

@@ -383,16 +383,10 @@ prefix/unusual#? directory ,/unusual, name?# 0
 # Set timestamps of one db.db file to epoch, query the index with -m flag, and expect no changes
 $ touch -amt 197001010000.00 "prefix/db.db"
 
-Expect:
-    atime: 0
-    mtime: 0
-
+Expect: atime: 0, mtime: 0
 $ gufi_query -d " " -m -S "SELECT name FROM vrsummary;" -E "SELECT name FROM vrpentries;" "prefix" > /dev/null
 
-Got:
-    atime: 0
-    mtime: 0
-
+Got:    atime: 0, mtime: 0
 # Symlink at Level 1
 $ ln -svn "directory/subdirectory" "prefix/symlink"
 'prefix/symlink' -> 'directory/subdirectory'

test/regression/gufi_query.sh.in

@@ -203,9 +203,9 @@ run_sort "${GUFI_QUERY} -a -d \" \" -S \"SELECT rpath(sname, sroll), inode FROM
 
 echo "# Set timestamps of one db.db file to epoch, query the index with -m flag, and expect no changes"
 run_no_sort "touch -amt 197001010000.00 \"${INDEXROOT}/db.db\""
-stat --printf "Expect:\n    atime: %X\n    mtime: %Y\n\n" "${INDEXROOT}/db.db"
+stat "${STAT_FORMAT_FLAG}" "Expect: atime: %X, mtime: %Y" "${INDEXROOT}/db.db"
 run_sort "${GUFI_QUERY} -d \" \" -m -S \"SELECT name FROM vrsummary;\" -E \"SELECT name FROM vrpentries;\" \"${INDEXROOT}\" > /dev/null"
-stat --printf "Got:\n    atime: %X\n    mtime: %Y\n\n" "${INDEXROOT}/db.db"
+stat "${STAT_FORMAT_FLAG}" "Got:    atime: %X, mtime: %Y" "${INDEXROOT}/db.db"
 
 echo "# Symlink at Level 1"
 run_no_sort "ln -svn \"directory/subdirectory\" \"${INDEXROOT}/symlink\""
@@ -222,7 +222,7 @@ rm -f "${INDEXROOT}/directory/symlink"
 echo "# Missing database file"
 rm "${INDEXROOT}/db.db"
 run_sort "${GUFI_QUERY} -d \" \" -m -S \"SELECT name FROM vrsummary;\" -E \"SELECT name FROM vrpentries;\" \"${INDEXROOT}\" > /dev/null"
-) |& tee "${OUTPUT}"
+) 2>&1 | tee "${OUTPUT}"
 
 echo "# Print terse debug timestamps (not part of expected results)"
 run_sort "${GUFI_QUERY} -d \" \" -j -S \"SELECT name FROM vrsummary;\" -E \"SELECT name FROM vrpentries;\" \"${INDEXROOT}\""

test/regression/gufi_sqlite3.sh.in

@@ -126,7 +126,7 @@ PATH="@CMAKE_BINARY_DIR@/src:${PATH}" run_no_sort "echo -n \"CREATE VIRTUAL TABL
 echo "# bad SQL"
 run_no_sort "(echo \"CREATE TABLE;\") |" \
             "${GUFI_SQLITE3}"
-) |& tee "${OUTPUT}"
+) 2>&1 | tee "${OUTPUT}"
 
 @DIFF@ @CMAKE_CURRENT_BINARY_DIR@/gufi_sqlite3.expected "${OUTPUT}"
 rm "${OUTPUT}"

test/regression/gufi_stat.expected

@@ -9,8 +9,7 @@ positional arguments:
 
 options:
   -c FORMAT, --format FORMAT
-                        use the specified FORMAT instead of the default;
-                        output a newline after each use of FORMAT
+                        use the specified FORMAT instead of the default; output a newline after each use of FORMAT
   -t, --terse           print the information in terse form
   --help                display this help and exit
   --version             show program's version number and exit

test/regression/gufi_stats.expected

@@ -13,24 +13,14 @@ positional arguments:
 options:
   --help                show this help message and exit
   --version, -v         show program's version number and exit
-  --recursive, -r       run command recursively (depth, filesize, filecount,
-                        linkcount, dircount, leaf-dirs, leaf-depth, leaf-
-                        files, leaf-links, extensions, filesize-log2-bins,
-                        filesize-log1024-bins, dirfilecount-log2-bins,
-                        dirfilecount-log1024-bins)
-  --cumulative, -c      return cumulative values (total-filesize, total-
-                        filecount, total-linkcount, total-dircount, total-
-                        leaf-files, total-leaf-links, files-per-level, links-
-                        per-level, dirs-per-level, filesize-log2-bins,
-                        filesize-log1024-bins, dirfilecount-log2-bins,
-                        dirfilecount-log1024-bins)
+  --recursive, -r       run command recursively (depth, filesize, filecount, linkcount, dircount, leaf-dirs, leaf-depth, leaf-files, leaf-links, extensions, filesize-log2-bins, filesize-log1024-bins, dirfilecount-log2-bins, dirfilecount-log1024-bins)
+  --cumulative, -c      return cumulative values (total-filesize, total-filecount, total-linkcount, total-dircount, total-leaf-files, total-leaf-links, files-per-level, links-per-level, dirs-per-level, filesize-log2-bins, filesize-log1024-bins, dirfilecount-log2-bins, dirfilecount-log1024-bins)
   --order order         sort output (if applicable)
   --num-results n       first n results
   --uid u, --user u     restrict to user
   --delim c             delimiter separating output columns
   --in-memory-name name
-                        Name of in-memory database when aggregation is
-                        performed
+                        Name of in-memory database when aggregation is performed
   --aggregate-name name
                         Name of final database when aggregation is performed
   --skip-file filename  Name of file containing directory basenames to skip