From 3f7b9e3c5a7fcaf7bd32a1417c5f538d35d5063b Mon Sep 17 00:00:00 2001 From: mstmdev Date: Tue, 31 Oct 2023 11:31:25 +0800 Subject: [PATCH 1/4] Fix push MinIO and SFTP files once are ineffective --- integration/integration_minio_test.go | 1 + integration/integration_sftp_test.go | 1 + .../conf/run-gofs-minio-push-client-once.yaml | 4 + .../conf/run-gofs-sftp-push-client-once.yaml | 3 + .../test/test-gofs-minio-push-once.yaml | 111 +++++++++++++++++ .../test/test-gofs-sftp-push-once.yaml | 113 ++++++++++++++++++ sync/driver_push_client_sync.go | 8 +- sync/minio_push_client_sync.go | 5 +- sync/option.go | 4 + sync/sftp_push_client_sync.go | 5 +- 10 files changed, 250 insertions(+), 5 deletions(-) create mode 100644 integration/testdata/conf/run-gofs-minio-push-client-once.yaml create mode 100644 integration/testdata/conf/run-gofs-sftp-push-client-once.yaml create mode 100644 integration/testdata/test/test-gofs-minio-push-once.yaml create mode 100644 integration/testdata/test/test-gofs-sftp-push-once.yaml diff --git a/integration/integration_minio_test.go b/integration/integration_minio_test.go index ebd324be..02eef816 100644 --- a/integration/integration_minio_test.go +++ b/integration/integration_minio_test.go @@ -17,6 +17,7 @@ func TestIntegration_MinIO(t *testing.T) { {"gofs MinIO pull", "", "run-gofs-minio-pull-client.yaml", "test-gofs-minio-pull.yaml"}, {"gofs MinIO push partial", "", "run-gofs-minio-push-client-partial.yaml", "test-gofs-minio-push-partial.yaml"}, {"gofs MinIO pull partial", "", "run-gofs-minio-pull-client-partial.yaml", "test-gofs-minio-pull-partial.yaml"}, + {"gofs MinIO push once", "", "run-gofs-minio-push-client-once.yaml", "test-gofs-minio-push-once.yaml"}, } for _, tc := range testCases { diff --git a/integration/integration_sftp_test.go b/integration/integration_sftp_test.go index 90fc05b7..768aa247 100644 --- a/integration/integration_sftp_test.go +++ b/integration/integration_sftp_test.go @@ -15,6 +15,7 @@ func TestIntegration_SFTP(t *testing.T) { }{ {"gofs SFTP push", "", "run-gofs-sftp-push-client.yaml", "test-gofs-sftp-push.yaml"}, {"gofs SFTP pull", "", "run-gofs-sftp-pull-client.yaml", "test-gofs-sftp-pull.yaml"}, + {"gofs SFTP push once", "", "run-gofs-sftp-push-client-once.yaml", "test-gofs-sftp-push-once.yaml"}, } for _, tc := range testCases { diff --git a/integration/testdata/conf/run-gofs-minio-push-client-once.yaml b/integration/testdata/conf/run-gofs-minio-push-client-once.yaml new file mode 100644 index 00000000..36de9b29 --- /dev/null +++ b/integration/testdata/conf/run-gofs-minio-push-client-once.yaml @@ -0,0 +1,4 @@ +source: ./minio-push-client-once/source +dest: minio://127.0.0.1:9000?secure=false&local_sync_disabled=false&path=./minio-push-client-once/dest&remote_path=minio-bucket +log_dir: ./minio-push-client-once-logs/ +users: minio_user|minio_pwd \ No newline at end of file diff --git a/integration/testdata/conf/run-gofs-sftp-push-client-once.yaml b/integration/testdata/conf/run-gofs-sftp-push-client-once.yaml new file mode 100644 index 00000000..ee1e793c --- /dev/null +++ b/integration/testdata/conf/run-gofs-sftp-push-client-once.yaml @@ -0,0 +1,3 @@ +source: ./sftp-push-client-once/source +dest: sftp://127.0.0.1:22?local_sync_disabled=false&path=./sftp-push-client-once/dest&remote_path=/sftp-workspace&ssh_user=sftp_user&ssh_pass=sftp_pwd +log_dir: ./sftp-push-client-once-logs/ \ No newline at end of file diff --git a/integration/testdata/test/test-gofs-minio-push-once.yaml b/integration/testdata/test/test-gofs-minio-push-once.yaml new file mode 100644 index 00000000..832e5098 --- /dev/null +++ b/integration/testdata/test/test-gofs-minio-push-once.yaml @@ -0,0 +1,111 @@ +name: test for gofs MinIO push once +init: + - mkdir: + source: ./minio-push-client-once/source + - mkdir: + source: ./minio-push-client-once/dest + - cp: + source: ./integration_test.go + dest: ./minio-push-client-once/source/integration_test.go.bak1 + - cp: + source: ./integration_test.go + dest: ./minio-push-client-once/source/integration_test.go.bak2 + - sleep: 2s + - rm: + source: ./minio-push-client-once/source/integration_test.go.bak2 + - touch: + source: ./minio-push-client-once/source/hello + - echo: + source: ./minio-push-client-once/source/hello + input: Hello World + append: false + - echo: + source: ./minio-push-client-once/source/hello + input: Bye Bye + append: true + no-newline: false + - echo: + source: ./minio-push-client-once/source/empty1 + no-newline: true + - echo: + source: ./minio-push-client-once/source/empty2 + no-newline: true + - symlink: + link: ./minio-push-client-once/source/hello.symlink + dest: ./minio-push-client-once/source/hello + ignore-error: false + - symlink: + link: ./minio-push-client-once/source/hello.symlink.bak + dest: ./minio-push-client-once/source/hello + ignore-error: false + - symlink: + link: ./minio-push-client-once/source/not_exist.symlink + dest: ./minio-push-client-once/source/not_exist + ignore-error: false + - sleep: 2s + - rm: + source: ./minio-push-client-once/source/hello.symlink.bak +actions: + - sleep: 10s + - is-equal: + source: ./minio-push-client-once/source/integration_test.go.bak1 + dest: ./minio-data-mount/integration_test.go.bak1 + expect: true + must-non-empty: true + - is-equal: + source: ./minio-push-client-once/source/integration_test.go.bak1 + dest: ./minio-data-mount/integration_test.go.bak1 + expect: true + must-non-empty: true + algorithm: sha1 + - is-equal: + source: ./minio-push-client-once/source/integration_test.go.bak1 + dest: ./minio-data-mount/empty1 + expect: false + - is-equal: + source: ./minio-push-client-once/source/empty1 + dest: ./minio-data-mount/empty2 + expect: true + must-non-empty: false + - is-empty: + source: ./minio-data-mount/integration_test.go.bak1 + expect: false + - is-exist: + source: ./minio-data-mount/integration_test.go.bak1 + expect: true + - is-exist: + source: ./minio-data-mount/integration_test.go.bak2 + expect: false + - is-dir: + source: ./minio-data-mount + expect: true + - is-dir: + source: ./minio-data-mount/integration_test.go.bak1 + expect: false + - hash: + algorithm: md5 + source: ./minio-data-mount/hello + expect: 856719c57653fa86008d49db895a5752 + - hash: + algorithm: sha1 + source: ./minio-data-mount/hello + expect: f343874b5df87e887d85df2e790df33584463162 + - rm: + source: ./minio-push-client-once/source/integration_test.go.bak1 + - sleep: 10s + - run: ../scripts/minio/remount-minio.sh + - is-exist: + source: ./minio-data-mount/integration_test.go.bak1 + expect: false + - is-empty: + source: ./minio-data-mount/hello.symlink + expect: false + - is-empty: + source: ./minio-data-mount/not_exist.symlink + expect: false + - is-exist: + source: ./minio-data-mount/hello.symlink.bak + expect: false +clear: + - rm: + source: ./minio-push-client-once \ No newline at end of file diff --git a/integration/testdata/test/test-gofs-sftp-push-once.yaml b/integration/testdata/test/test-gofs-sftp-push-once.yaml new file mode 100644 index 00000000..74218d06 --- /dev/null +++ b/integration/testdata/test/test-gofs-sftp-push-once.yaml @@ -0,0 +1,113 @@ +name: test for gofs SFTP push once +init: + - mkdir: + source: ./sftp-push-client-once/source + - mkdir: + source: ./sftp-push-client-once/dest + - cp: + source: ./integration_test.go + dest: ./sftp-push-client-once/source/integration_test.go.bak1 + - cp: + source: ./integration_test.go + dest: ./sftp-push-client-once/source/integration_test.go.bak2 + - sleep: 2s + - rm: + source: ./sftp-push-client-once/source/integration_test.go.bak2 + - touch: + source: ./sftp-push-client-once/source/hello + - echo: + source: ./sftp-push-client-once/source/hello + input: Hello World + append: false + - echo: + source: ./sftp-push-client-once/source/hello + input: Bye Bye + append: true + no-newline: false + - echo: + source: ./sftp-push-client-once/source/empty1 + no-newline: true + - echo: + source: ./sftp-push-client-once/source/empty2 + no-newline: true + - symlink: + link: ./sftp-push-client-once/source/hello.symlink + dest: ./sftp-push-client-once/source/hello + ignore-error: false + - symlink: + link: ./sftp-push-client-once/source/hello.symlink.bak + dest: ./sftp-push-client-once/source/hello + ignore-error: false + - symlink: + link: ./sftp-push-client-once/source/not_exist.symlink + dest: ./sftp-push-client-once/source/not_exist + ignore-error: false + - sleep: 2s + - rm: + source: ./sftp-push-client-once/source/hello.symlink.bak +actions: + - sleep: 10s + - is-equal: + source: ./sftp-push-client-once/source/integration_test.go.bak1 + dest: /sftp-workspace/integration_test.go.bak1 + expect: true + must-non-empty: true + - is-equal: + source: ./sftp-push-client-once/source/integration_test.go.bak1 + dest: /sftp-workspace/integration_test.go.bak1 + expect: true + must-non-empty: true + algorithm: sha1 + - is-equal: + source: ./sftp-push-client-once/source/integration_test.go.bak1 + dest: /sftp-workspace/empty1 + expect: false + - is-equal: + source: ./sftp-push-client-once/source/empty1 + dest: /sftp-workspace/empty2 + expect: true + must-non-empty: false + - is-empty: + source: /sftp-workspace/integration_test.go.bak1 + expect: false + - is-exist: + source: /sftp-workspace/integration_test.go.bak1 + expect: true + - is-exist: + source: /sftp-workspace/integration_test.go.bak2 + expect: false + - is-dir: + source: /sftp-workspace + expect: true + - is-dir: + source: /sftp-workspace/integration_test.go.bak1 + expect: false + - hash: + algorithm: md5 + source: /sftp-workspace/hello + expect: 856719c57653fa86008d49db895a5752 + - hash: + algorithm: sha1 + source: /sftp-workspace/hello + expect: f343874b5df87e887d85df2e790df33584463162 + - sleep: 2s + - rm: + source: ./sftp-push-client-once/source/integration_test.go.bak1 + - sleep: 10s + - is-exist: + source: /sftp-workspace/integration_test.go.bak1 + expect: false + - is-symlink: + link: /sftp-workspace/hello.symlink + expect: true + ignore-error: false + - is-symlink: + link: /sftp-workspace/not_exist.symlink + expect: true + ignore-error: false + - is-exist: + source: /sftp-workspace/hello.symlink.bak + expect: false +clear: + - rm: + source: ./sftp-push-client-once \ No newline at end of file diff --git a/sync/driver_push_client_sync.go b/sync/driver_push_client_sync.go index f5d4d3b4..78e472d0 100644 --- a/sync/driver_push_client_sync.go +++ b/sync/driver_push_client_sync.go @@ -27,9 +27,11 @@ func newDriverPushClientSync(ds diskSync, basePath string) driverPushClientSync } } -func (s *driverPushClientSync) start() error { - if err := s.initFileInfo(); err != nil { - return err +func (s *driverPushClientSync) start(isSync bool) error { + if !isSync { + if err := s.initFileInfo(); err != nil { + return err + } } return s.driver.Connect() } diff --git a/sync/minio_push_client_sync.go b/sync/minio_push_client_sync.go index d6946978..41d4819f 100644 --- a/sync/minio_push_client_sync.go +++ b/sync/minio_push_client_sync.go @@ -23,6 +23,8 @@ func NewMinIOPushClientSync(opt Option) (Sync, error) { maxTranRate := opt.MaxTranRate r := opt.Retry logger := opt.Logger + syncOnce := opt.SyncOnce + syncCron := opt.SyncCron if chunkSize <= 0 { return nil, errInvalidChunkSize @@ -47,7 +49,8 @@ func NewMinIOPushClientSync(opt Option) (Sync, error) { s.driver = minio.NewMinIODriver(s.endpoint, s.bucketName, s.secure, s.currentUser.UserName(), s.currentUser.Password(), true, r, maxTranRate, logger) - err = s.start() + isSync := syncOnce || len(syncCron) > 0 + err = s.start(isSync) if err != nil { return nil, err } diff --git a/sync/option.go b/sync/option.go index 4708aaac..e11e0458 100644 --- a/sync/option.go +++ b/sync/option.go @@ -39,6 +39,8 @@ type Option struct { Reporter report.Reporter TaskConf string Logger *logger.Logger + SyncOnce bool + SyncCron string } // NewSyncOption create an instance of the Option, store all the sync component options @@ -70,6 +72,8 @@ func NewSyncOption(config conf.Config, users []*auth.User, r retry.Retry, pi ign Reporter: reporter, TaskConf: config.TaskConf, Logger: logger, + SyncOnce: config.SyncOnce, + SyncCron: config.SyncCron, } return opt } diff --git a/sync/sftp_push_client_sync.go b/sync/sftp_push_client_sync.go index 473eef42..64e562ed 100644 --- a/sync/sftp_push_client_sync.go +++ b/sync/sftp_push_client_sync.go @@ -18,6 +18,8 @@ func NewSftpPushClientSync(opt Option) (Sync, error) { maxTranRate := opt.MaxTranRate r := opt.Retry logger := opt.Logger + syncOnce := opt.SyncOnce + syncCron := opt.SyncCron if chunkSize <= 0 { return nil, errInvalidChunkSize @@ -35,7 +37,8 @@ func NewSftpPushClientSync(opt Option) (Sync, error) { s.driver = sftp.NewSFTPDriver(s.remoteAddr, dest.SSHConfig(), true, r, maxTranRate, logger) - err = s.start() + isSync := syncOnce || len(syncCron) > 0 + err = s.start(isSync) if err != nil { return nil, err } From be8d3fcf63a2d82be7e83b90c9922d8012f449bd Mon Sep 17 00:00:00 2001 From: mstmdev Date: Tue, 31 Oct 2023 11:47:07 +0800 Subject: [PATCH 2/4] Add sync_once=true --- integration/testdata/conf/run-gofs-minio-push-client-once.yaml | 3 ++- integration/testdata/conf/run-gofs-sftp-push-client-once.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/integration/testdata/conf/run-gofs-minio-push-client-once.yaml b/integration/testdata/conf/run-gofs-minio-push-client-once.yaml index 36de9b29..ddd7d1e1 100644 --- a/integration/testdata/conf/run-gofs-minio-push-client-once.yaml +++ b/integration/testdata/conf/run-gofs-minio-push-client-once.yaml @@ -1,4 +1,5 @@ source: ./minio-push-client-once/source dest: minio://127.0.0.1:9000?secure=false&local_sync_disabled=false&path=./minio-push-client-once/dest&remote_path=minio-bucket log_dir: ./minio-push-client-once-logs/ -users: minio_user|minio_pwd \ No newline at end of file +users: minio_user|minio_pwd +sync_once: true \ No newline at end of file diff --git a/integration/testdata/conf/run-gofs-sftp-push-client-once.yaml b/integration/testdata/conf/run-gofs-sftp-push-client-once.yaml index ee1e793c..c2abb833 100644 --- a/integration/testdata/conf/run-gofs-sftp-push-client-once.yaml +++ b/integration/testdata/conf/run-gofs-sftp-push-client-once.yaml @@ -1,3 +1,4 @@ source: ./sftp-push-client-once/source dest: sftp://127.0.0.1:22?local_sync_disabled=false&path=./sftp-push-client-once/dest&remote_path=/sftp-workspace&ssh_user=sftp_user&ssh_pass=sftp_pwd -log_dir: ./sftp-push-client-once-logs/ \ No newline at end of file +log_dir: ./sftp-push-client-once-logs/ +sync_once: true \ No newline at end of file From 06bb29f50e80dd576bc67833074867e4c8b3320e Mon Sep 17 00:00:00 2001 From: mstmdev Date: Tue, 31 Oct 2023 12:03:03 +0800 Subject: [PATCH 3/4] Fix tests --- integration/testdata/test/test-gofs-minio-push-once.yaml | 6 ------ integration/testdata/test/test-gofs-sftp-push-once.yaml | 7 ------- 2 files changed, 13 deletions(-) diff --git a/integration/testdata/test/test-gofs-minio-push-once.yaml b/integration/testdata/test/test-gofs-minio-push-once.yaml index 832e5098..189feb7a 100644 --- a/integration/testdata/test/test-gofs-minio-push-once.yaml +++ b/integration/testdata/test/test-gofs-minio-push-once.yaml @@ -90,13 +90,7 @@ actions: algorithm: sha1 source: ./minio-data-mount/hello expect: f343874b5df87e887d85df2e790df33584463162 - - rm: - source: ./minio-push-client-once/source/integration_test.go.bak1 - - sleep: 10s - run: ../scripts/minio/remount-minio.sh - - is-exist: - source: ./minio-data-mount/integration_test.go.bak1 - expect: false - is-empty: source: ./minio-data-mount/hello.symlink expect: false diff --git a/integration/testdata/test/test-gofs-sftp-push-once.yaml b/integration/testdata/test/test-gofs-sftp-push-once.yaml index 74218d06..885bd647 100644 --- a/integration/testdata/test/test-gofs-sftp-push-once.yaml +++ b/integration/testdata/test/test-gofs-sftp-push-once.yaml @@ -90,13 +90,6 @@ actions: algorithm: sha1 source: /sftp-workspace/hello expect: f343874b5df87e887d85df2e790df33584463162 - - sleep: 2s - - rm: - source: ./sftp-push-client-once/source/integration_test.go.bak1 - - sleep: 10s - - is-exist: - source: /sftp-workspace/integration_test.go.bak1 - expect: false - is-symlink: link: /sftp-workspace/hello.symlink expect: true From bb6620c47c7d00708397f17ae150852158b99dd4 Mon Sep 17 00:00:00 2001 From: mstmdev Date: Tue, 31 Oct 2023 12:12:08 +0800 Subject: [PATCH 4/4] Update test-gofs-minio-push-once.yaml --- integration/testdata/test/test-gofs-minio-push-once.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/testdata/test/test-gofs-minio-push-once.yaml b/integration/testdata/test/test-gofs-minio-push-once.yaml index 189feb7a..04a66a58 100644 --- a/integration/testdata/test/test-gofs-minio-push-once.yaml +++ b/integration/testdata/test/test-gofs-minio-push-once.yaml @@ -90,7 +90,6 @@ actions: algorithm: sha1 source: ./minio-data-mount/hello expect: f343874b5df87e887d85df2e790df33584463162 - - run: ../scripts/minio/remount-minio.sh - is-empty: source: ./minio-data-mount/hello.symlink expect: false