From 83a3d54340bd6afa971ff60f6c300681bb61dc82 Mon Sep 17 00:00:00 2001 From: BenB196 Date: Wed, 9 Oct 2019 19:52:38 -0400 Subject: [PATCH 1/3] Update crashplan-ffs-go-pkg to v0.0.7 This adds support for the new FFS fields. Updated related files. Fixes issue #41 --- docs/default_index_template.json | 9 +++++++++ docs/index_mapping.json | 9 +++++++++ elasticsearch/elasticsearch.go | 9 +++++++++ go.mod | 2 +- go.sum | 2 ++ vendor/modules.txt | 2 +- 6 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/default_index_template.json b/docs/default_index_template.json index c57e9a5..fc12a05 100644 --- a/docs/default_index_template.json +++ b/docs/default_index_template.json @@ -148,6 +148,15 @@ "removableMediaBusType": { "type": "keyword" }, + "removableMediaMediaName": { + "type": "keyword" + }, + "removableMediaVolumeName": { + "type": "keyword" + }, + "removableMediaPartitionId": { + "type": "keyword" + }, "syncDestination": { "type": "keyword" }, diff --git a/docs/index_mapping.json b/docs/index_mapping.json index 7f99891..1bd9354 100644 --- a/docs/index_mapping.json +++ b/docs/index_mapping.json @@ -125,6 +125,15 @@ "removableMediaBusType": { "type": "keyword" }, + "removableMediaMediaName": { + "type": "keyword" + }, + "removableMediaVolumeName": { + "type": "keyword" + }, + "removableMediaPartitionId": { + "type": "keyword" + }, "syncDestination": { "type": "keyword" }, diff --git a/elasticsearch/elasticsearch.go b/elasticsearch/elasticsearch.go index 8018122..7655db8 100644 --- a/elasticsearch/elasticsearch.go +++ b/elasticsearch/elasticsearch.go @@ -196,6 +196,15 @@ func BuildIndexPattern(elasticConfig config.Elasticsearch) string { " \"removableMediaBusType\": {" + " \"type\": \"keyword\"" + " }," + + " \"removableMediaMediaName\": {" + + " \"type\": \"keyword\"" + + " }," + + " \"removableMediaVolumeName\": {" + + " \"type\": \"keyword\"" + + " }," + + " \"removableMediaPartitionId\": {" + + " \"type\": \"keyword\"" + + " }," + " \"syncDestination\": {" + " \"type\": \"keyword\"" + " }," + diff --git a/go.mod b/go.mod index 87e97ad..6d84541 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/BenB196/crashplan-ffs-puller go 1.12 require ( - github.com/BenB196/crashplan-ffs-go-pkg v0.0.6 + github.com/BenB196/crashplan-ffs-go-pkg v0.0.7 github.com/BenB196/ip-api-go-pkg v0.0.4 github.com/google/go-cmp v0.3.1 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index fc04ffb..493da7f 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/BenB196/crashplan-ffs-go-pkg v0.0.5 h1:ioDSdeFzIc7351h95qwTbL6tXkinDS github.com/BenB196/crashplan-ffs-go-pkg v0.0.5/go.mod h1:oBwZ5/KfaNokjvI2Hn2e6c5801ir11FOCq4lG5PcniE= github.com/BenB196/crashplan-ffs-go-pkg v0.0.6 h1:kZzkUpz2epVIgJJZSEsBYdEW2OhB2QfPLvtW7peP4WA= github.com/BenB196/crashplan-ffs-go-pkg v0.0.6/go.mod h1:oBwZ5/KfaNokjvI2Hn2e6c5801ir11FOCq4lG5PcniE= +github.com/BenB196/crashplan-ffs-go-pkg v0.0.7 h1:gTd0tnrptsJ6krc04SC4v5ZEy0ys0jkBUe4QvsUyJZk= +github.com/BenB196/crashplan-ffs-go-pkg v0.0.7/go.mod h1:oBwZ5/KfaNokjvI2Hn2e6c5801ir11FOCq4lG5PcniE= github.com/BenB196/ip-api-go-pkg v0.0.3 h1:FWmM7FkhT1N55jd4jPW7W9LVOQrG89DLlrgwMb5fosw= github.com/BenB196/ip-api-go-pkg v0.0.3/go.mod h1:ccPdkBNnzf/uvuk7qXgEO06TCS/qILNJQP/KETQG4jU= github.com/BenB196/ip-api-go-pkg v0.0.4 h1:BbWELxooG6l2gaXQ4i4gm6NsyikdbhalF+TKg08gDPQ= diff --git a/vendor/modules.txt b/vendor/modules.txt index f3a19e8..fd77f8f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# github.com/BenB196/crashplan-ffs-go-pkg v0.0.6 +# github.com/BenB196/crashplan-ffs-go-pkg v0.0.7 github.com/BenB196/crashplan-ffs-go-pkg # github.com/BenB196/ip-api-go-pkg v0.0.4 github.com/BenB196/ip-api-go-pkg From b2a34e1f145e8f0618ec76ad0b4cdf5baa87a1d9 Mon Sep 17 00:00:00 2001 From: BenB196 Date: Wed, 9 Oct 2019 19:55:24 -0400 Subject: [PATCH 2/3] Adds retry attempts for Protocol errors. Fixes issue #39 --- ffsEvent/ffsEvent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffsEvent/ffsEvent.go b/ffsEvent/ffsEvent.go index 7090296..6cdd154 100644 --- a/ffsEvent/ffsEvent.go +++ b/ffsEvent/ffsEvent.go @@ -235,7 +235,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg if err != nil { log.Println("error getting file events for ffs query: " + query.Name) //check if recoverable errors are thrown - if strings.Contains(err.Error(),"Error with gathering file events POST: 500 Internal Server Error") || (strings.Contains(err.Error(),"stream error: stream ID") && strings.Contains(err.Error(),"INTERNAL_ERROR")) || strings.Contains(err.Error(),"read: connection reset by peer") || strings.Contains(err.Error(),"POST: 400 Bad Request") { + if strings.Contains(err.Error(),"Error with gathering file events POST: 500 Internal Server Error") || (strings.Contains(err.Error(),"stream error: stream ID") && (strings.Contains(err.Error(),"INTERNAL_ERROR") || strings.Contains(err.Error(),"PROTOCOL_ERROR"))) || strings.Contains(err.Error(),"read: connection reset by peer") || strings.Contains(err.Error(),"POST: 400 Bad Request") { //allow for 10 retries before killing to save resource overload. log.Println("Attempting to recover from error: " + err.Error() + ". Retry number: " + strconv.Itoa(retryCount)) if retryCount <= 10 { From f2aae8542db5a23abdc91b2f028dc5a32fd31ab8 Mon Sep 17 00:00:00 2001 From: BenB196 Date: Wed, 9 Oct 2019 20:05:34 -0400 Subject: [PATCH 3/3] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8294c18..7693c96 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.2 \ No newline at end of file +0.1.3 \ No newline at end of file