From 0a2058cc0fc1caa9d2ed80e2d80c30f055a31383 Mon Sep 17 00:00:00 2001 From: Jan Cajthaml Date: Sun, 24 Mar 2019 11:02:35 +0100 Subject: [PATCH] no leading zero in query string for fx-other --- .../cnb-rates-batch/config/environment.go | 16 +++++----- services/cnb-rates-batch/daemon/batch.go | 31 ++++++++----------- services/cnb-rates-import/utils/paths.go | 2 +- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/services/cnb-rates-batch/config/environment.go b/services/cnb-rates-batch/config/environment.go index a2ff5b3b..7730f789 100644 --- a/services/cnb-rates-batch/config/environment.go +++ b/services/cnb-rates-batch/config/environment.go @@ -34,8 +34,6 @@ func loadConfFromEnv() Configuration { metricsOutput := getEnvString("CNB_RATES_METRICS_OUTPUT", "") metricsRefreshRate := getEnvDuration("CNB_RATES_METRICS_REFRESHRATE", time.Second) - // FIXME freeze start time (now) to config - if rootStorage == "" { log.Fatal("missing required parameter to run") } @@ -47,23 +45,27 @@ func loadConfFromEnv() Configuration { } if os.MkdirAll(rootStorage+"/"+utils.FXMainDailyCacheDirectory(), os.ModePerm) != nil { - log.Fatal("unable to assert main-fx daily cache directory") + log.Fatal("unable to assert fx-main daily cache directory") } if os.MkdirAll(rootStorage+"/"+utils.FXMainMonthlyCacheDirectory(), os.ModePerm) != nil { - log.Fatal("unable to assert main-fx monthly cache directory") + log.Fatal("unable to assert fx-main monthly cache directory") } if os.MkdirAll(rootStorage+"/"+utils.FXMainYearlyCacheDirectory(), os.ModePerm) != nil { - log.Fatal("unable to assert main-fx yearly cache directory") + log.Fatal("unable to assert fx-main yearly cache directory") + } + + if os.MkdirAll(rootStorage+"/"+utils.FXOtherDailyCacheDirectory(), os.ModePerm) != nil { + log.Fatal("unable to assert fx-other daily cache directory") } if os.MkdirAll(rootStorage+"/"+utils.FXOtherMonthlyCacheDirectory(), os.ModePerm) != nil { - log.Fatal("unable to assert other-fx yearly cache directory") + log.Fatal("unable to assert fx-other monthly cache directory") } if os.MkdirAll(rootStorage+"/"+utils.FXOtherYearlyCacheDirectory(), os.ModePerm) != nil { - log.Fatal("unable to assert other-fx yearly cache directory") + log.Fatal("unable to assert fx-other yearly cache directory") } return Configuration{ diff --git a/services/cnb-rates-batch/daemon/batch.go b/services/cnb-rates-batch/daemon/batch.go index 4e7ad16e..9c2b0057 100644 --- a/services/cnb-rates-batch/daemon/batch.go +++ b/services/cnb-rates-batch/daemon/batch.go @@ -84,7 +84,7 @@ func (batch Batch) ProcessNewMonthly() { func (batch Batch) ProcessNewFXMain(wg *sync.WaitGroup) error { defer wg.Done() - log.Info("Processing new main-fx rates") + log.Info("Processing new fx-main rates") days, err := utils.GetFXMainUnprocessedFiles(batch.storage) if err != nil { @@ -98,7 +98,7 @@ func (batch Batch) ProcessNewFXMain(wg *sync.WaitGroup) error { return nil } - log.Infof("Processing new main-fx for %s", day) + log.Infof("Processing new fx-main for %s", day) reader, err := batch.storage.GetFileReader(cachePath + day) if err != nil { @@ -108,23 +108,21 @@ func (batch Batch) ProcessNewFXMain(wg *sync.WaitGroup) error { result, err := utils.ParseCSV(day, reader) if err != nil { - log.Warnf("error parse main-fx CSV data for day %s, %+v\n", day, err) + log.Warnf("error parse fx-main CSV data for day %s, %+v\n", day, err) continue } bytes, err := result.MarshalJSON() if err != nil { - log.Warnf("error marshall main-fx data for day %s, %+v\n", day, err) + log.Warnf("error marshall fx-main data for day %s, %+v\n", day, err) continue } err = batch.storage.WriteFile(utils.FXMainDailyCachePath(result.Date), bytes) if err != nil { - log.Warnf("error write cache fail main-fx data for day %s, %+v\n", day, err) + log.Warnf("error write cache fail fx-main data for day %s, %+v\n", day, err) continue } - - log.Infof("parsed main-fx CSV data for day %s\n", day) } return nil @@ -133,7 +131,7 @@ func (batch Batch) ProcessNewFXMain(wg *sync.WaitGroup) error { func (batch Batch) ProcessNewFXOther(wg *sync.WaitGroup) error { defer wg.Done() - log.Info("Processing new other-fx rates") + log.Info("Processing new fx-other rates") days, err := utils.GetFXOtherUnprocessedFiles(batch.storage) if err != nil { @@ -141,39 +139,36 @@ func (batch Batch) ProcessNewFXOther(wg *sync.WaitGroup) error { } cachePath := utils.FXOtherOfflineDirectory() + "/" - for _, day := range days { if batch.ctx.Err() != nil { return nil } - log.Infof("Processing new other-fx for %s", day) + log.Infof("Processing new fx-other for %s", day) reader, err := batch.storage.GetFileReader(cachePath + day) if err != nil { - log.Warnf("error parse other-fx CSV data for day %s, %+v\n", day, err) + log.Warnf("error parse fx-other CSV data for day %s, %+v\n", day, err) continue } result, err := utils.ParseCSV(day, reader) if err != nil { - log.Warnf("error parse other-fx CSV data for day %s, %+v\n", day, err) + log.Warnf("error parse fx-other CSV data for day %s, %+v\n", day, err) continue } bytes, err := result.MarshalJSON() if err != nil { - log.Warnf("error marshall other-fx data for mo %s, %+v\n", day, err) + log.Warnf("error marshall fx-other data for mo %s, %+v\n", day, err) continue } err = batch.storage.WriteFile(utils.FXOtherDailyCachePath(result.Date), bytes) if err != nil { - log.Warnf("error write cache fail other-fx data for day %s, %+v\n", day, err) + log.Warnf("error write cache fail fx-other data for day %s, %+v\n", day, err) continue } - - log.Infof("parsed other-fx CSV data for day %s\n", day) } return nil @@ -183,10 +178,10 @@ func (batch Batch) ProcessNewFX() { var wg sync.WaitGroup wg.Add(1) - batch.ProcessNewFXMain(&wg) + go batch.ProcessNewFXMain(&wg) wg.Add(1) - batch.ProcessNewFXOther(&wg) + go batch.ProcessNewFXOther(&wg) wg.Wait() } diff --git a/services/cnb-rates-import/utils/paths.go b/services/cnb-rates-import/utils/paths.go index 36a06b46..c5e02097 100644 --- a/services/cnb-rates-import/utils/paths.go +++ b/services/cnb-rates-import/utils/paths.go @@ -21,7 +21,7 @@ func GetUrlForDateMainFx(date time.Time) string { } func GetUrlForDateOtherFx(date time.Time) string { - return "/en/financial_markets/foreign_exchange_market/other_currencies_fx_rates/fx_rates.txt?month=" + date.Format("01") + "&year=" + date.Format("2006") + return "/en/financial_markets/foreign_exchange_market/other_currencies_fx_rates/fx_rates.txt?month=" + date.Format("1") + "&year=" + date.Format("2006") } func FXMainOfflineDirectory() string {