Skip to content

Commit

Permalink
Merge pull request #23 from jancajthaml-openbank/bufxig/fx-other-quer…
Browse files Browse the repository at this point in the history
…ystring

no leading zero in query string for fx-other
  • Loading branch information
jancajthaml authored Mar 24, 2019
2 parents 3040f1b + ca31e65 commit 92ad698
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 27 deletions.
14 changes: 6 additions & 8 deletions services/cnb-rates-batch/config/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@ func loadConfFromEnv() Configuration {
metricsOutput := getEnvString("CNB_RATES_METRICS_OUTPUT", "")
metricsRefreshRate := getEnvDuration("CNB_RATES_METRICS_REFRESHRATE", time.Second)

// FIXME freeze start time (now) to config

if rootStorage == "" {
log.Fatal("missing required parameter to run")
}
Expand All @@ -47,27 +45,27 @@ func loadConfFromEnv() Configuration {
}

if os.MkdirAll(rootStorage+"/"+utils.FXMainDailyCacheDirectory(), os.ModePerm) != nil {
log.Fatal("unable to assert main-fx daily cache directory")
log.Fatal("unable to assert fx-main daily cache directory")
}

if os.MkdirAll(rootStorage+"/"+utils.FXMainMonthlyCacheDirectory(), os.ModePerm) != nil {
log.Fatal("unable to assert main-fx monthly cache directory")
log.Fatal("unable to assert fx-main monthly cache directory")
}

if os.MkdirAll(rootStorage+"/"+utils.FXMainYearlyCacheDirectory(), os.ModePerm) != nil {
log.Fatal("unable to assert main-fx yearly cache directory")
log.Fatal("unable to assert fx-main yearly cache directory")
}

if os.MkdirAll(rootStorage+"/"+utils.FXOtherDailyCacheDirectory(), os.ModePerm) != nil {
log.Fatal("unable to assert other-fx daily cache directory")
log.Fatal("unable to assert fx-other daily cache directory")
}

if os.MkdirAll(rootStorage+"/"+utils.FXOtherMonthlyCacheDirectory(), os.ModePerm) != nil {
log.Fatal("unable to assert other-fx monthly cache directory")
log.Fatal("unable to assert fx-other monthly cache directory")
}

if os.MkdirAll(rootStorage+"/"+utils.FXOtherYearlyCacheDirectory(), os.ModePerm) != nil {
log.Fatal("unable to assert other-fx yearly cache directory")
log.Fatal("unable to assert fx-other yearly cache directory")
}

return Configuration{
Expand Down
31 changes: 13 additions & 18 deletions services/cnb-rates-batch/daemon/batch.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func (batch Batch) ProcessNewMonthly() {

func (batch Batch) ProcessNewFXMain(wg *sync.WaitGroup) error {
defer wg.Done()
log.Info("Processing new main-fx rates")
log.Info("Processing new fx-main rates")

days, err := utils.GetFXMainUnprocessedFiles(batch.storage)
if err != nil {
Expand All @@ -98,7 +98,7 @@ func (batch Batch) ProcessNewFXMain(wg *sync.WaitGroup) error {
return nil
}

log.Infof("Processing new main-fx for %s", day)
log.Infof("Processing new fx-main for %s", day)

reader, err := batch.storage.GetFileReader(cachePath + day)
if err != nil {
Expand All @@ -108,23 +108,21 @@ func (batch Batch) ProcessNewFXMain(wg *sync.WaitGroup) error {

result, err := utils.ParseCSV(day, reader)
if err != nil {
log.Warnf("error parse main-fx CSV data for day %s, %+v\n", day, err)
log.Warnf("error parse fx-main CSV data for day %s, %+v\n", day, err)
continue
}

bytes, err := result.MarshalJSON()
if err != nil {
log.Warnf("error marshall main-fx data for day %s, %+v\n", day, err)
log.Warnf("error marshall fx-main data for day %s, %+v\n", day, err)
continue
}

err = batch.storage.WriteFile(utils.FXMainDailyCachePath(result.Date), bytes)
if err != nil {
log.Warnf("error write cache fail main-fx data for day %s, %+v\n", day, err)
log.Warnf("error write cache fail fx-main data for day %s, %+v\n", day, err)
continue
}

log.Infof("parsed main-fx CSV data for day %s\n", day)
}

return nil
Expand All @@ -133,47 +131,44 @@ func (batch Batch) ProcessNewFXMain(wg *sync.WaitGroup) error {
func (batch Batch) ProcessNewFXOther(wg *sync.WaitGroup) error {
defer wg.Done()

log.Info("Processing new other-fx rates")
log.Info("Processing new fx-other rates")

days, err := utils.GetFXOtherUnprocessedFiles(batch.storage)
if err != nil {
return err
}

cachePath := utils.FXOtherOfflineDirectory() + "/"

for _, day := range days {
if batch.ctx.Err() != nil {
return nil
}

log.Infof("Processing new other-fx for %s", day)
log.Infof("Processing new fx-other for %s", day)

reader, err := batch.storage.GetFileReader(cachePath + day)
if err != nil {
log.Warnf("error parse other-fx CSV data for day %s, %+v\n", day, err)
log.Warnf("error parse fx-other CSV data for day %s, %+v\n", day, err)
continue
}

result, err := utils.ParseCSV(day, reader)
if err != nil {
log.Warnf("error parse other-fx CSV data for day %s, %+v\n", day, err)
log.Warnf("error parse fx-other CSV data for day %s, %+v\n", day, err)
continue
}

bytes, err := result.MarshalJSON()
if err != nil {
log.Warnf("error marshall other-fx data for mo %s, %+v\n", day, err)
log.Warnf("error marshall fx-other data for mo %s, %+v\n", day, err)
continue
}

err = batch.storage.WriteFile(utils.FXOtherDailyCachePath(result.Date), bytes)
if err != nil {
log.Warnf("error write cache fail other-fx data for day %s, %+v\n", day, err)
log.Warnf("error write cache fail fx-other data for day %s, %+v\n", day, err)
continue
}

log.Infof("parsed other-fx CSV data for day %s\n", day)
}

return nil
Expand All @@ -183,10 +178,10 @@ func (batch Batch) ProcessNewFX() {
var wg sync.WaitGroup

wg.Add(1)
batch.ProcessNewFXMain(&wg)
go batch.ProcessNewFXMain(&wg)

wg.Add(1)
batch.ProcessNewFXOther(&wg)
go batch.ProcessNewFXOther(&wg)

wg.Wait()
}
Expand Down
2 changes: 1 addition & 1 deletion services/cnb-rates-import/utils/paths.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func GetUrlForDateMainFx(date time.Time) string {
}

func GetUrlForDateOtherFx(date time.Time) string {
return "/en/financial_markets/foreign_exchange_market/other_currencies_fx_rates/fx_rates.txt?month=" + date.Format("01") + "&year=" + date.Format("2006")
return "/en/financial_markets/foreign_exchange_market/other_currencies_fx_rates/fx_rates.txt?month=" + date.Format("1") + "&year=" + date.Format("2006")
}

func FXMainOfflineDirectory() string {
Expand Down

0 comments on commit 92ad698

Please sign in to comment.