-
Notifications
You must be signed in to change notification settings - Fork 66
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
test_dsync: add test_expectfail.sh and test_existence.sh and stubs
Add the following tests: test_expectfail.sh: add tests for missing source or destination parent test_existence.sh: verify set of files synced is correct Add stubs for the following tests: test_metadata.sh: verify metadata of synced files is correct test_data.sh: verify data in synced files is correct Signed-off-by: Olaf Faaland <faaland1@llnl.gov>
- Loading branch information
Showing
4 changed files
with
322 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| #!/bin/bash | ||
|
|
||
| ############################################################################## | ||
| # Description: | ||
| # | ||
| # Verify dsync handles data | ||
| # - file data is identical after a copy | ||
| # - file with differing data is copied if --contents arg is used | ||
| # - file with differing data is not copied if --contents arg is not used and metadata match | ||
| # | ||
| # Notes: | ||
| # - does not test whether data copes are spread across nodes/tasks evenly | ||
| # | ||
| ############################################################################## | ||
|
|
||
| # Turn on verbose output | ||
| #set -x | ||
|
|
||
| DSYNC_TEST_BIN=${DSYNC_TEST_BIN:-${1}} | ||
| DSYNC_SRC_DIR=${DSYNC_SRC_DIR:-${2}} | ||
| DSYNC_DEST_DIR=${DSYNC_DEST_DIR:-${3}} | ||
| DSYNC_TMP_FILE=${DSYNC_TMP_FILE:-${4}} | ||
|
|
||
| echo "Using dsync binary at: $DSYNC_TEST_BIN" | ||
| echo "Using src directory at: $DSYNC_SRC_DIR" | ||
| echo "Using dest directory at: $DSYNC_DEST_DIR" | ||
| echo "Using directory tree: $DSYNC_TMP_FILE" | ||
|
|
||
| # tests not yet implemented | ||
|
|
||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,186 @@ | ||
| #!/bin/bash | ||
|
|
||
| . utility/set_funcs.sh | ||
|
|
||
| ############################################################################## | ||
| # Description: | ||
| # | ||
| # Verify dsync handles presence or absence | ||
| # - empty source, but non-empty destination | ||
| # - non-empty source, but empty destination | ||
| # - directories on source, but not on destination, are copied | ||
| # - files on source, but not on destination, are copied | ||
| # - directories on destination, but not source, are undisturbed without --delete | ||
| # - files on destination, but not source, are undisturbed without --delete | ||
| # - directories on destination, but not source, are removed with --delete | ||
| # - files on destination, but not source, are removed with --delete | ||
| # | ||
| # Notes: | ||
| # | ||
| ############################################################################## | ||
|
|
||
| # Turn on verbose output | ||
| #set -x | ||
|
|
||
| DSYNC_TEST_BIN=${DSYNC_TEST_BIN:-${1}} | ||
| DSYNC_SRC_BASE=${DSYNC_SRC_BASE:-${2}} | ||
| DSYNC_DEST_BASE=${DSYNC_DEST_BASE:-${3}} | ||
| DSYNC_TREE_NAME=${DSYNC_TREE_NAME:-${4}} | ||
|
|
||
| DSYNC_TREE_DATA=/usr/include/c++ | ||
|
|
||
| mpirun=$(which mpirun 2>/dev/null) | ||
| mpirun_opts="" | ||
| if [[ -n $mpirun ]]; then | ||
| procs=$(( $(nproc ) / 8 )) | ||
| if [[ $procs -gt 16 ]]; then | ||
| procs=16 | ||
| fi | ||
| mpirun_opts="-c $procs" | ||
|
|
||
| echo "Using mpirun: $mpirun $mpirun_opts" | ||
| fi | ||
|
|
||
| echo "Using dsync binary at: $DSYNC_TEST_BIN" | ||
| echo "Using src parent directory at: $DSYNC_SRC_BASE" | ||
| echo "Using dest parent directory at: $DSYNC_DEST_BASE" | ||
| echo "Using test data from: $DSYNC_TREE_DATA" | ||
|
|
||
| DSYNC_SRC_DIR=$(mktemp --directory ${DSYNC_SRC_BASE}/${DSYNC_TREE_NAME}.XXXXX) | ||
| DSYNC_DEST_DIR=$(mktemp --directory ${DSYNC_DEST_BASE}/${DSYNC_TREE_NAME}.XXXXX) | ||
|
|
||
| function fs_type() | ||
| { | ||
| fname=$1 | ||
| df -T ${fname} | awk '$1 != "Filesystem" {print $2}' | ||
| } | ||
|
|
||
| function list_all_files() | ||
| { | ||
| find $1 -printf '%P\n' | sort | grep -v '^$' | ||
| } | ||
|
|
||
| function sync_and_verify() | ||
| { | ||
| local srcdir=$1 | ||
| local destdir=$2 | ||
| local name=$3 | ||
| local expectation=$4 | ||
|
|
||
| local result=0 | ||
| local dest_type="" | ||
|
|
||
| src_list=$(mktemp /tmp/sync_and_verify.src.XXXXX) | ||
| list_all_files $srcdir > $src_list | ||
|
|
||
| dest_list=$(mktemp /tmp/sync_and_verify.dest.XXXXX) | ||
|
|
||
| # dest may not exist, but its parent must exist | ||
| if [[ -d $destdir ]]; then | ||
| dest_type=$(fs_type $destdir) | ||
| list_all_files $destdir > $dest_list | ||
| else | ||
| parent=$(dirname $destdir) | ||
| if [[ ! -d $parent ]]; then | ||
| echo "sync_and_verify: destdir $destdir and its parent do not exist" | ||
| exit 1 | ||
| fi | ||
| dest_type=$(fs_type $parent) | ||
| fi | ||
|
|
||
| quiet_opt="--quiet" | ||
| delete_opt="" | ||
| if [[ $name = "delete" ]]; then | ||
| delete_opt="--delete" | ||
| fi | ||
|
|
||
| if [[ -n $mpirun ]]; then | ||
| $mpirun $mpirun_opts $DSYNC_TEST_BIN $quiet_opt $delete_opt $srcdir $destdir | ||
| else | ||
| $DSYNC_TEST_BIN $quiet_opt $delete_opt $srcdir $destdir | ||
| fi | ||
| rc=$? | ||
|
|
||
| if [[ $rc -ne 0 ]]; then | ||
| echo "dsync failed with rc $rc" | ||
| result=1 | ||
| fi | ||
|
|
||
| if [[ $result -eq 0 ]]; then | ||
| after_list=$(mktemp /tmp/sync_and_verify.after.XXXXX) | ||
| list_all_files $destdir > $after_list | ||
|
|
||
| expected_list=$(mktemp /tmp/sync_and_verify.expected.XXXXX) | ||
|
|
||
| case $expectation in | ||
| "union") | ||
| union $src_list $dest_list > $expected_list | ||
| ;; | ||
| "src_exactly") | ||
| cat $src_list > $expected_list | ||
| ;; | ||
| esac | ||
|
|
||
| sets_equal $after_list $expected_list | ||
| result=$? | ||
| fi | ||
|
|
||
| if [ "$result" -eq 0 ]; then | ||
| echo "PASSED verify of option $name for $destdir type $dest_type" | ||
| else | ||
| echo "FAILED verify of option $name for $destdir type $dest_type" | ||
| echo ======================= | ||
| echo "before: src_list" | ||
| cat $src_list | ||
| echo | ||
| echo "before: dest_list" | ||
| cat $dest_list | ||
| echo | ||
| echo "after: after_list:" | ||
| cat $after_list | ||
| echo | ||
| echo "expected:" | ||
| cat $expected_list | ||
| echo ======================= | ||
| fi | ||
|
|
||
| rm $src_list $dest_list $after_list $expected_list | ||
|
|
||
| return $result | ||
| } | ||
|
|
||
| # empty source, but non-empty destination | ||
| # directories on destination, but not source, are undisturbed without --delete | ||
| # files on destination, but not source, are undisturbed without --delete | ||
| rm -fr $DSYNC_SRC_DIR/stuff | ||
| rm -fr $DSYNC_DEST_DIR/stuff | ||
| mkdir $DSYNC_SRC_DIR/stuff | ||
| mkdir $DSYNC_DEST_DIR/stuff | ||
| cp -a $DSYNC_TREE_DATA $DSYNC_DEST_DIR/stuff | ||
| sync_and_verify $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff empty_source union | ||
|
|
||
| # non-empty source, but empty destination | ||
| # files on source, but not on destination, are copied | ||
| # directories on source, but not on destination, are copied | ||
| rm -fr $DSYNC_SRC_DIR/stuff | ||
| rm -fr $DSYNC_DEST_DIR/stuff | ||
| mkdir $DSYNC_SRC_DIR/stuff | ||
| cp -a $DSYNC_TREE_DATA $DSYNC_SRC_DIR/stuff | ||
| sync_and_verify $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff empty_destination union | ||
|
|
||
| # directories on destination, but not source, are removed with --delete | ||
| # files on destination, but not source, are removed with --delete | ||
| rm -fr $DSYNC_SRC_DIR/stuff | ||
| rm -fr $DSYNC_DEST_DIR/stuff | ||
| mkdir $DSYNC_SRC_DIR/stuff | ||
| mkdir $DSYNC_DEST_DIR/stuff | ||
| cp -a $DSYNC_TREE_DATA $DSYNC_SRC_DIR/stuff | ||
| mkdir $DSYNC_DEST_DIR/stuff/destdir | ||
| touch $DSYNC_DEST_DIR/stuff/destfile | ||
| sync_and_verify $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff delete src_exactly | ||
|
|
||
| # clean up | ||
| rm -fr $DSYNC_SRC_DIR/stuff | ||
| rm -fr $DSYNC_DEST_DIR/stuff | ||
|
|
||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| #!/bin/bash | ||
|
|
||
| . utility/set_funcs.sh | ||
|
|
||
| ############################################################################## | ||
| # Description: | ||
| # | ||
| # Verify dsync fails when it should | ||
| # - source does not exist | ||
| # - parent of dest does not exist | ||
| # | ||
| # Notes: | ||
| # | ||
| ############################################################################## | ||
|
|
||
| # Turn on verbose output | ||
| #set -x | ||
|
|
||
| DSYNC_TEST_BIN=${DSYNC_TEST_BIN:-${1}} | ||
| DSYNC_SRC_BASE=${DSYNC_SRC_BASE:-${2}} | ||
| DSYNC_DEST_BASE=${DSYNC_DEST_BASE:-${3}} | ||
| DSYNC_TREE_NAME=${DSYNC_TREE_NAME:-${4}} | ||
|
|
||
| echo "Using dsync binary at: $DSYNC_TEST_BIN" | ||
| echo "Using src parent directory at: $DSYNC_SRC_BASE" | ||
| echo "Using dest parent directory at: $DSYNC_DEST_BASE" | ||
|
|
||
| DSYNC_SRC_DIR=$(mktemp --directory ${DSYNC_SRC_BASE}/${DSYNC_TREE_NAME}.XXXXX) | ||
| DSYNC_DEST_DIR=$(mktemp --directory ${DSYNC_DEST_BASE}/${DSYNC_TREE_NAME}.XXXXX) | ||
|
|
||
| function sync_and_verify() | ||
| { | ||
| local srcdir=$1 | ||
| local destdir=$2 | ||
| local name=$3 | ||
| local expectation=$4 | ||
|
|
||
| local result=0 | ||
| local dest_type="unknown" | ||
|
|
||
| quiet_opt="--quiet" | ||
|
|
||
| $DSYNC_TEST_BIN $quiet_opt $delete_opt $srcdir $destdir | ||
| rc=$? | ||
| echo "dsync failed with rc $rc" | ||
|
|
||
| if [[ "$rc" -eq 0 ]]; then | ||
| echo "FAILED verify of option $name for $destdir type $dest_type" | ||
| result=1 | ||
|
|
||
| else | ||
| echo "PASSED verify of option $name for $destdir type $dest_type" | ||
| result=0 | ||
| fi | ||
|
|
||
| return $result | ||
| } | ||
|
|
||
| # source does not exist | ||
| rm -fr $DSYNC_SRC_DIR/stuff | ||
| rm -fr $DSYNC_DEST_DIR/stuff | ||
| mkdir $DSYNC_DEST_DIR/stuff | ||
| sync_and_verify $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff source_missing expect_dsync_fail | ||
|
|
||
| # parent of dest does not exist | ||
| rm -fr $DSYNC_SRC_DIR/stuff | ||
| rm -fr $DSYNC_DEST_DIR/stuff | ||
| mkdir $DSYNC_SRC_DIR/stuff | ||
| sync_and_verify $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff/childdir dest_parent_missing expect_dsync_fail | ||
|
|
||
| # clean up | ||
| rm -fr $DSYNC_SRC_DIR/stuff | ||
| rm -fr $DSYNC_DEST_DIR/stuff | ||
|
|
||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| #!/bin/bash | ||
|
|
||
| ############################################################################## | ||
| # Description: | ||
| # | ||
| # Verify dsync handles metdata | ||
| # - file with differing metadata is copied | ||
| # - file metadata is identical after a copy | ||
| # - directory metadata is identical after a sync | ||
| # | ||
| # Notes: | ||
| # | ||
| ############################################################################## | ||
|
|
||
| # Turn on verbose output | ||
| #set -x | ||
|
|
||
| DSYNC_TEST_BIN=${DSYNC_TEST_BIN:-${1}} | ||
| DSYNC_SRC_DIR=${DSYNC_SRC_DIR:-${2}} | ||
| DSYNC_DEST_DIR=${DSYNC_DEST_DIR:-${3}} | ||
| DSYNC_TMP_FILE=${DSYNC_TMP_FILE:-${4}} | ||
|
|
||
| echo "Using dsync binary at: $DSYNC_TEST_BIN" | ||
| echo "Using src directory at: $DSYNC_SRC_DIR" | ||
| echo "Using dest directory at: $DSYNC_DEST_DIR" | ||
| echo "Using directory tree: $DSYNC_TMP_FILE" | ||
|
|
||
| # tests not yet implemented | ||
|
|
||
| exit 0 |