From 5393f658b643324c434af92385b0a5c533c26fbd Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 08:53:59 +0200 Subject: [PATCH 01/43] Configuration for metrics. --- pkg/metrics/config/config.go | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 pkg/metrics/config/config.go diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go new file mode 100644 index 0000000000..195037973a --- /dev/null +++ b/pkg/metrics/config/config.go @@ -0,0 +1,37 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +// Config holds the config options that need to be passed down to the metrics reader +type Config struct { + Prefix string `mapstructure:"prefix"` + MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` + MetricsDataLocation string `mapstructure:"metrics_data_location"` +} + +// Init sets sane defaults +func (c *Config) Init() { + if c.Prefix == "" { + c.Prefix = "metrics" + } + if c.MetricsDataDriverType == "json" { + // default values + c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json" + } +} From 77839392065b4e769deacf2fe435e4c9202e2fc1 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 08:55:00 +0200 Subject: [PATCH 02/43] Dummy (test) driver implementation. --- pkg/metrics/reader/driver/dummy/dummy.go | 53 ++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 pkg/metrics/reader/driver/dummy/dummy.go diff --git a/pkg/metrics/reader/driver/dummy/dummy.go b/pkg/metrics/reader/driver/dummy/dummy.go new file mode 100644 index 0000000000..15d90a2ed7 --- /dev/null +++ b/pkg/metrics/reader/driver/dummy/dummy.go @@ -0,0 +1,53 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package dummy + +import ( + "math/rand" + "reva/pkg/metrics/config" +) + +// New returns a new DummyDriver object. +func New(config *config.Config) (*DummyDriver, error) { + driver := &DummyDriver{ + config: config, + } + + return driver, nil +} + +// DummyDriver the DummyDriver struct +type DummyDriver struct { + config *config.Config +} + +// GetNumUsers returns the number of site users, it's a dummy number +func (d *DummyDriver) GetNumUsers() int64 { + return int64(rand.Intn(30000)) +} + +// GetNumGroups returns the number of site groups, it's a dummy number +func (d *DummyDriver) GetNumGroups() int64 { + return int64(rand.Intn(200)) +} + +// GetAmountStorage returns the amount of site storage used, it's a dummy amount +func (d *DummyDriver) GetAmountStorage() int64 { + return int64(rand.Intn(70000000000)) +} From e7f2c0cdd33007ea76495e37a5453a196e4aaf57 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 08:55:37 +0200 Subject: [PATCH 03/43] JSON metrics data file driver implementation. --- pkg/metrics/reader/driver/json/json.go | 91 ++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 pkg/metrics/reader/driver/json/json.go diff --git a/pkg/metrics/reader/driver/json/json.go b/pkg/metrics/reader/driver/json/json.go new file mode 100644 index 0000000000..e9481e9877 --- /dev/null +++ b/pkg/metrics/reader/driver/json/json.go @@ -0,0 +1,91 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package json + +import ( + "encoding/json" + "errors" + "io/ioutil" + "reva/pkg/metrics/config" +) + +// New returns a new MetricsJSONDriver object. +// It reads the data file from the specified config.MetricsDataLocation upon initializing. +// It does not reload the data file for each metric. +func New(config *config.Config) (*MetricsJSONDriver, error) { + // the json driver reads the data metrics file upon initializing + metricsData, err := readJSON(config) + if err != nil { + return nil, err + } + + driver := &MetricsJSONDriver{ + config: config, + data: metricsData, + } + + return driver, nil +} + +func readJSON(config *config.Config) (*data, error) { + if config.MetricsDataLocation == "" { + err := errors.New("Unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?") + return nil, err + } + + file, err := ioutil.ReadFile(config.MetricsDataLocation) + if err != nil { + return nil, err + } + + data := &data{} + err = json.Unmarshal(file, data) + if err != nil { + return nil, err + } + + return data, nil +} + +type data struct { + NumUsers int64 `json:"cs3_org_sciencemesh_site_total_num_users"` + NumGroups int64 `json:"cs3_org_sciencemesh_site_total_num_groups"` + AmountStorage int64 `json:"cs3_org_sciencemesh_site_total_amount_storage"` +} + +// MetricsJSONDriver the JsonDriver struct that also holds the data +type MetricsJSONDriver struct { + config *config.Config + data *data +} + +// GetNumUsers returns the number of site users +func (d *MetricsJSONDriver) GetNumUsers() int64 { + return int64(d.data.NumUsers) +} + +// GetNumGroups returns the number of site groups +func (d *MetricsJSONDriver) GetNumGroups() int64 { + return int64(d.data.NumGroups) +} + +// GetAmountStorage returns the amount of site storage used +func (d *MetricsJSONDriver) GetAmountStorage() int64 { + return int64(d.data.AmountStorage) +} From 8c256419522b96ed09c5827259737bef2376231d Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 08:59:21 +0200 Subject: [PATCH 04/43] Metrics now uses data driver to access the actual metrics data. --- pkg/metrics/metrics.go | 157 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 153 insertions(+), 4 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 6aa00e15f2..fcece55146 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -19,21 +19,170 @@ package metrics import ( + "context" + "errors" + "reva/pkg/metrics/config" + "reva/pkg/metrics/reader/driver/dummy" + "reva/pkg/metrics/reader/driver/json" + + "github.com/rs/zerolog/log" + "go.opencensus.io/stats" "go.opencensus.io/stats/view" ) -// Reader is the interface that defines how the metrics will be read. +// New returns a Metrics object +func New(c *config.Config) (*Metrics, error) { + m := &Metrics{ + dataDriverType: "", + dataLocation: "", + dataDriver: nil, + config: c, + NumUsersMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless), + NumGroupsMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless), + AmountStorageMeasure: stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitBytes), + } + + // register the desired measures' views + if err := view.Register( + m.getNumUsersView(), + m.getNumGroupsView(), + m.getAmountStorageView(), + ); err != nil { + log.Error().Err(err).Msg("error registering the driver's views with opencensus exporter") + return nil, err + } + + return m, nil +} + +// Metrics the metrics struct +type Metrics struct { + dataDriverType string + dataLocation string + dataDriver Reader // the metrics data driver is an implemention of Reader + config *config.Config + NumUsersMeasure *stats.Int64Measure + NumGroupsMeasure *stats.Int64Measure + AmountStorageMeasure *stats.Int64Measure +} + +// RecordMetrics records the latest metrics from the metrics data source as OpenCensus stats views. +func (m *Metrics) RecordMetrics() error { + if err := initDataDriver(m); err != nil { + log.Error().Err(err).Msg("Could not set a driver") + return err + } + // record all latest metrics + m.recordNumUsers() + m.recordNumGroups() + m.recordAmountStorage() + + return nil +} + +// initDataDriver initializes a data driver and sets it to be the Metrics.dataDriver +func initDataDriver(m *Metrics) error { + // find out what driver to use + if m.config.MetricsDataDriverType == "" { + err := errors.New("Unable to initialize a metrics data driver, has a driver type (metrics_data_driver_type) been configured?") + return err + } + m.dataLocation = m.config.MetricsDataLocation + + // create/init a driver depending on driver type + if m.config.MetricsDataDriverType == "json" { + // Because the json metrics data file is only read on json driver creation + // a json driver must be re-created to make sure we have the current/latest metrics data. + // Other drivers may need creation only once. + jsonDriver, err := json.New(m.config) + if err != nil { + log.Error().Err(err) + return err + } + m.dataDriver = jsonDriver + log.Debug().Msgf("Metrics uses json driver") + } + if m.config.MetricsDataDriverType == "dummy" && m.dataDriver == nil { + // the dummy driver does not need to be initialized every time + dummyDriver, err := dummy.New(m.config) + if err != nil { + log.Error().Err(err) + return err + } + m.dataDriver = dummyDriver + log.Debug().Msgf("Metrics uses dummy driver") + } + // no known driver configured, return error + if m.dataDriver == nil { + err := errors.New("Unable to initialize a metrics data driver. Has a correct driver type (one of: json, dummy) been configured?") + return err + } + + return nil +} + +// recordNumUsers records the latest number of site users figure +func (m *Metrics) recordNumUsers() { + ctx := context.Background() + stats.Record(ctx, m.NumUsersMeasure.M(m.dataDriver.GetNumUsers())) +} + +func (m *Metrics) getNumUsersView() *view.View { + return &view.View{ + Name: m.NumUsersMeasure.Name(), + Description: m.NumUsersMeasure.Description(), + Measure: m.NumUsersMeasure, + Aggregation: view.LastValue(), + } +} + +// recordNumGroups records the latest number of site groups figure +func (m *Metrics) recordNumGroups() { + ctx := context.Background() + stats.Record(ctx, m.NumGroupsMeasure.M(m.dataDriver.GetNumGroups())) +} + +func (m *Metrics) getNumGroupsView() *view.View { + return &view.View{ + Name: m.NumGroupsMeasure.Name(), + Description: m.NumGroupsMeasure.Description(), + Measure: m.NumGroupsMeasure, + Aggregation: view.LastValue(), + } +} + +// recordAmountStorage records the latest amount storage figure +func (m *Metrics) recordAmountStorage() { + ctx := context.Background() + stats.Record(ctx, m.AmountStorageMeasure.M(m.dataDriver.GetAmountStorage())) +} + +func (m *Metrics) getAmountStorageView() *view.View { + return &view.View{ + Name: m.AmountStorageMeasure.Name(), + Description: m.AmountStorageMeasure.Description(), + Measure: m.AmountStorageMeasure, + Aggregation: view.LastValue(), + } +} + +// Reader is the interface that defines the metrics to read. +// Any metrics data driver must implement this interface. +// Each function should return the current/latest available metrics figure relevant to that function. type Reader interface { // GetNumUsersView returns an OpenCensus stats view which records the // number of users registered in the mesh provider. - GetNumUsersView() *view.View + // Metric name: cs3_org_sciencemesh_site_total_num_users + GetNumUsers() int64 // GetNumGroupsView returns an OpenCensus stats view which records the // number of user groups registered in the mesh provider. - GetNumGroupsView() *view.View + // Metric name: cs3_org_sciencemesh_site_total_num_groups + GetNumGroups() int64 // GetAmountStorageView returns an OpenCensus stats view which records the // amount of storage in the system. - GetAmountStorageView() *view.View + // Metric name: cs3_org_sciencemesh_site_total_amount_storage + GetAmountStorage() int64 } From 0935163ba11f30d1e9c2e0b597ff28ed8a166299 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 10:31:53 +0200 Subject: [PATCH 05/43] deleted --- pkg/metrics/reader/dummy/dummy.go | 131 ---------------------------- pkg/metrics/reader/script/script.go | 78 ----------------- 2 files changed, 209 deletions(-) delete mode 100644 pkg/metrics/reader/dummy/dummy.go delete mode 100644 pkg/metrics/reader/script/script.go diff --git a/pkg/metrics/reader/dummy/dummy.go b/pkg/metrics/reader/dummy/dummy.go deleted file mode 100644 index d12947c0fc..0000000000 --- a/pkg/metrics/reader/dummy/dummy.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package dummy - -import ( - "context" - "math/rand" - "os" - "time" - - "github.com/cs3org/reva/pkg/logger" - "github.com/cs3org/reva/pkg/metrics" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" -) - -func init() { - - log := logger.New().With().Int("pid", os.Getpid()).Logger() - - m := &Metrics{ - NumUsersMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless), - NumGroupsMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless), - AmountStorageMeasure: stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitBytes), - } - - // Verify that the struct implements the metrics.Reader interface - _, ok := interface{}(m).(metrics.Reader) - if !ok { - log.Error().Msg("the driver does not implement the metrics.Reader interface") - return - } - - // register the desired measures' views - if err := view.Register( - m.GetNumUsersView(), - m.GetNumGroupsView(), - m.GetAmountStorageView(), - ); err != nil { - log.Error().Err(err).Msg("error registering views with opencensus exporter") - return - } - - // call the actual metric provider functions for the latest metrics every 4th second - go func() { - rand.Seed(time.Now().UnixNano()) - for { - m.getNumUsers() - m.getNumGroups() - m.getAmountStorage() - time.Sleep(4 * time.Second) - } - }() -} - -// Metrics returns randomly generated values for the defined metrics. -type Metrics struct { - numUsersCounter int64 - amountStorageCounter int64 - - NumUsersMeasure *stats.Int64Measure - NumGroupsMeasure *stats.Int64Measure - AmountStorageMeasure *stats.Int64Measure -} - -// getNumberUsers links to the underlying number of site users provider -func (m *Metrics) getNumUsers() { - ctx := context.Background() - m.numUsersCounter += int64(rand.Intn(100)) - stats.Record(ctx, m.NumUsersMeasure.M(m.numUsersCounter)) -} - -// GetNumUsersView returns the number of site users measure view -func (m *Metrics) GetNumUsersView() *view.View { - return &view.View{ - Name: m.NumUsersMeasure.Name(), - Description: m.NumUsersMeasure.Description(), - Measure: m.NumUsersMeasure, - Aggregation: view.LastValue(), - } -} - -// getNumberGroups links to the underlying number of site groups provider -func (m *Metrics) getNumGroups() { - ctx := context.Background() - var numGroupsCounter = int64(rand.Intn(100)) - stats.Record(ctx, m.NumGroupsMeasure.M(numGroupsCounter)) -} - -// GetNumGroupsView returns the number of site groups measure view -func (m *Metrics) GetNumGroupsView() *view.View { - return &view.View{ - Name: m.NumGroupsMeasure.Name(), - Description: m.NumGroupsMeasure.Description(), - Measure: m.NumGroupsMeasure, - Aggregation: view.LastValue(), - } -} - -// getAmountStorage links to the underlying amount of storage provider -func (m *Metrics) getAmountStorage() { - ctx := context.Background() - m.amountStorageCounter += int64(rand.Intn(12865000)) - stats.Record(ctx, m.AmountStorageMeasure.M(m.amountStorageCounter)) -} - -// GetAmountStorageView returns the amount of site storage measure view -func (m *Metrics) GetAmountStorageView() *view.View { - return &view.View{ - Name: m.AmountStorageMeasure.Name(), - Description: m.AmountStorageMeasure.Description(), - Measure: m.AmountStorageMeasure, - Aggregation: view.LastValue(), - } -} diff --git a/pkg/metrics/reader/script/script.go b/pkg/metrics/reader/script/script.go deleted file mode 100644 index 79617d5c00..0000000000 --- a/pkg/metrics/reader/script/script.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package script - -import ( - "os" - - "github.com/cs3org/reva/pkg/logger" - "github.com/cs3org/reva/pkg/metrics" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" -) - -func init() { - - log := logger.New().With().Int("pid", os.Getpid()).Logger() - - m := &Metrics{ - NumUsersMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless), - NumGroupsMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless), - AmountStorageMeasure: stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitBytes), - } - - // Verify that the struct implements the metrics.Reader interface - _, ok := interface{}(m).(metrics.Reader) - if !ok { - log.Error().Msg("the driver does not implement the metrics.Reader interface") - return - } - - // register the desired measures' views - if err := view.Register( - m.GetNumUsersView(), - m.GetNumGroupsView(), - m.GetAmountStorageView(), - ); err != nil { - log.Error().Err(err).Msg("error registering views with opencensus exporter") - return - } -} - -// Metrics returns randomly generated values for the defined metrics. -type Metrics struct { - NumUsersMeasure *stats.Int64Measure - NumGroupsMeasure *stats.Int64Measure - AmountStorageMeasure *stats.Int64Measure -} - -// GetNumUsersView returns the number of site users measure view -func (m *Metrics) GetNumUsersView() *view.View { - return nil -} - -// GetNumGroupsView returns the number of site groups measure view -func (m *Metrics) GetNumGroupsView() *view.View { - return nil -} - -// GetAmountStorageView returns the amount of site storage measure view -func (m *Metrics) GetAmountStorageView() *view.View { - return nil -} From db1fc17aef8bf4be93335db2e117b16fc5de4dff Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 10:34:33 +0200 Subject: [PATCH 06/43] New configuration for metrics. --- examples/metrics/metrics.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/metrics/metrics.toml b/examples/metrics/metrics.toml index caa41975f7..d5203f3b4c 100644 --- a/examples/metrics/metrics.toml +++ b/examples/metrics/metrics.toml @@ -2,6 +2,9 @@ jwt_secret = "Pive-Fumkiu4" [http.services.prometheus] +# metrics_data_driver_type, one of: dummy, json +metrics_data_driver_type = "dummy" +metrics_data_location = "" [http] address = "0.0.0.0:5550" \ No newline at end of file From c28411c927641824cae71d37b6eeb024dea64be6 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 15:29:31 +0200 Subject: [PATCH 07/43] Changed package name, dummy driver struct name. --- pkg/metrics/metrics.go | 4 +- pkg/metrics/reader/driver/dummy.go | 53 +++++++++++++++++ pkg/metrics/reader/driver/json.go | 91 ++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 pkg/metrics/reader/driver/dummy.go create mode 100644 pkg/metrics/reader/driver/json.go diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index fcece55146..4cddd768f1 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -22,8 +22,8 @@ import ( "context" "errors" "reva/pkg/metrics/config" - "reva/pkg/metrics/reader/driver/dummy" - "reva/pkg/metrics/reader/driver/json" + "reva/pkg/metrics/driver/dummy" + "reva/pkg/metrics/driver/json" "github.com/rs/zerolog/log" "go.opencensus.io/stats" diff --git a/pkg/metrics/reader/driver/dummy.go b/pkg/metrics/reader/driver/dummy.go new file mode 100644 index 0000000000..f454688c47 --- /dev/null +++ b/pkg/metrics/reader/driver/dummy.go @@ -0,0 +1,53 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package driver + +import ( + "math/rand" + "reva/pkg/metrics/config" +) + +// New returns a new DummyDriver object. +func New(config *config.Config) (*DummyDriver, error) { + driver := &DummyDriver{ + config: config, + } + + return driver, nil +} + +// DummyDriver the DummyDriver struct +type DummyDriver struct { + config *config.Config +} + +// GetNumUsers returns the number of site users, it's a dummy number +func (d *DummyDriver) GetNumUsers() int64 { + return int64(rand.Intn(30000)) +} + +// GetNumGroups returns the number of site groups, it's a dummy number +func (d *DummyDriver) GetNumGroups() int64 { + return int64(rand.Intn(200)) +} + +// GetAmountStorage returns the amount of site storage used, it's a dummy amount +func (d *DummyDriver) GetAmountStorage() int64 { + return int64(rand.Intn(70000000000)) +} diff --git a/pkg/metrics/reader/driver/json.go b/pkg/metrics/reader/driver/json.go new file mode 100644 index 0000000000..b0f7a60f0d --- /dev/null +++ b/pkg/metrics/reader/driver/json.go @@ -0,0 +1,91 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package driver + +import ( + "encoding/json" + "errors" + "io/ioutil" + "reva/pkg/metrics/config" +) + +// New returns a new MetricsJSONDriver object. +// It reads the data file from the specified config.MetricsDataLocation upon initializing. +// It does not reload the data file for each metric. +func New(config *config.Config) (*MetricsJSONDriver, error) { + // the json driver reads the data metrics file upon initializing + metricsData, err := readJSON(config) + if err != nil { + return nil, err + } + + driver := &MetricsJSONDriver{ + config: config, + data: metricsData, + } + + return driver, nil +} + +func readJSON(config *config.Config) (*data, error) { + if config.MetricsDataLocation == "" { + err := errors.New("Unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?") + return nil, err + } + + file, err := ioutil.ReadFile(config.MetricsDataLocation) + if err != nil { + return nil, err + } + + data := &data{} + err = json.Unmarshal(file, data) + if err != nil { + return nil, err + } + + return data, nil +} + +type data struct { + NumUsers int64 `json:"cs3_org_sciencemesh_site_total_num_users"` + NumGroups int64 `json:"cs3_org_sciencemesh_site_total_num_groups"` + AmountStorage int64 `json:"cs3_org_sciencemesh_site_total_amount_storage"` +} + +// MetricsJSONDriver the JsonDriver struct that also holds the data +type MetricsJSONDriver struct { + config *config.Config + data *data +} + +// GetNumUsers returns the number of site users +func (d *MetricsJSONDriver) GetNumUsers() int64 { + return int64(d.data.NumUsers) +} + +// GetNumGroups returns the number of site groups +func (d *MetricsJSONDriver) GetNumGroups() int64 { + return int64(d.data.NumGroups) +} + +// GetAmountStorage returns the amount of site storage used +func (d *MetricsJSONDriver) GetAmountStorage() int64 { + return int64(d.data.AmountStorage) +} From dac5a4fd33185b80b3884e7ab11b81f7177092c2 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 15:30:31 +0200 Subject: [PATCH 08/43] Changed package name. --- pkg/metrics/reader/driver/dummy/dummy.go | 53 -------------- pkg/metrics/reader/driver/json/json.go | 91 ------------------------ 2 files changed, 144 deletions(-) delete mode 100644 pkg/metrics/reader/driver/dummy/dummy.go delete mode 100644 pkg/metrics/reader/driver/json/json.go diff --git a/pkg/metrics/reader/driver/dummy/dummy.go b/pkg/metrics/reader/driver/dummy/dummy.go deleted file mode 100644 index 15d90a2ed7..0000000000 --- a/pkg/metrics/reader/driver/dummy/dummy.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package dummy - -import ( - "math/rand" - "reva/pkg/metrics/config" -) - -// New returns a new DummyDriver object. -func New(config *config.Config) (*DummyDriver, error) { - driver := &DummyDriver{ - config: config, - } - - return driver, nil -} - -// DummyDriver the DummyDriver struct -type DummyDriver struct { - config *config.Config -} - -// GetNumUsers returns the number of site users, it's a dummy number -func (d *DummyDriver) GetNumUsers() int64 { - return int64(rand.Intn(30000)) -} - -// GetNumGroups returns the number of site groups, it's a dummy number -func (d *DummyDriver) GetNumGroups() int64 { - return int64(rand.Intn(200)) -} - -// GetAmountStorage returns the amount of site storage used, it's a dummy amount -func (d *DummyDriver) GetAmountStorage() int64 { - return int64(rand.Intn(70000000000)) -} diff --git a/pkg/metrics/reader/driver/json/json.go b/pkg/metrics/reader/driver/json/json.go deleted file mode 100644 index e9481e9877..0000000000 --- a/pkg/metrics/reader/driver/json/json.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package json - -import ( - "encoding/json" - "errors" - "io/ioutil" - "reva/pkg/metrics/config" -) - -// New returns a new MetricsJSONDriver object. -// It reads the data file from the specified config.MetricsDataLocation upon initializing. -// It does not reload the data file for each metric. -func New(config *config.Config) (*MetricsJSONDriver, error) { - // the json driver reads the data metrics file upon initializing - metricsData, err := readJSON(config) - if err != nil { - return nil, err - } - - driver := &MetricsJSONDriver{ - config: config, - data: metricsData, - } - - return driver, nil -} - -func readJSON(config *config.Config) (*data, error) { - if config.MetricsDataLocation == "" { - err := errors.New("Unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?") - return nil, err - } - - file, err := ioutil.ReadFile(config.MetricsDataLocation) - if err != nil { - return nil, err - } - - data := &data{} - err = json.Unmarshal(file, data) - if err != nil { - return nil, err - } - - return data, nil -} - -type data struct { - NumUsers int64 `json:"cs3_org_sciencemesh_site_total_num_users"` - NumGroups int64 `json:"cs3_org_sciencemesh_site_total_num_groups"` - AmountStorage int64 `json:"cs3_org_sciencemesh_site_total_amount_storage"` -} - -// MetricsJSONDriver the JsonDriver struct that also holds the data -type MetricsJSONDriver struct { - config *config.Config - data *data -} - -// GetNumUsers returns the number of site users -func (d *MetricsJSONDriver) GetNumUsers() int64 { - return int64(d.data.NumUsers) -} - -// GetNumGroups returns the number of site groups -func (d *MetricsJSONDriver) GetNumGroups() int64 { - return int64(d.data.NumGroups) -} - -// GetAmountStorage returns the amount of site storage used -func (d *MetricsJSONDriver) GetAmountStorage() int64 { - return int64(d.data.AmountStorage) -} From 256b4af6eab42147779a0c7a0ba264c3a2b22943 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 15:31:19 +0200 Subject: [PATCH 09/43] Changed package name. --- pkg/metrics/driver/dummy/dummy.go | 53 ++++++++++++++++++ pkg/metrics/driver/json/json.go | 91 +++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 pkg/metrics/driver/dummy/dummy.go create mode 100644 pkg/metrics/driver/json/json.go diff --git a/pkg/metrics/driver/dummy/dummy.go b/pkg/metrics/driver/dummy/dummy.go new file mode 100644 index 0000000000..ddd5b2424e --- /dev/null +++ b/pkg/metrics/driver/dummy/dummy.go @@ -0,0 +1,53 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package dummy + +import ( + "math/rand" + "reva/pkg/metrics/config" +) + +// New returns a new DummyDataDriver object. +func New(config *config.Config) (*DummyDataDriver, error) { + driver := &DummyDataDriver{ + config: config, + } + + return driver, nil +} + +// DummyDataDriver the DummyDataDriver struct +type DummyDataDriver struct { + config *config.Config +} + +// GetNumUsers returns the number of site users, it's a dummy number +func (d *DummyDataDriver) GetNumUsers() int64 { + return int64(rand.Intn(30000)) +} + +// GetNumGroups returns the number of site groups, it's a dummy number +func (d *DummyDataDriver) GetNumGroups() int64 { + return int64(rand.Intn(200)) +} + +// GetAmountStorage returns the amount of site storage used, it's a dummy amount +func (d *DummyDataDriver) GetAmountStorage() int64 { + return int64(rand.Intn(70000000000)) +} diff --git a/pkg/metrics/driver/json/json.go b/pkg/metrics/driver/json/json.go new file mode 100644 index 0000000000..e9481e9877 --- /dev/null +++ b/pkg/metrics/driver/json/json.go @@ -0,0 +1,91 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package json + +import ( + "encoding/json" + "errors" + "io/ioutil" + "reva/pkg/metrics/config" +) + +// New returns a new MetricsJSONDriver object. +// It reads the data file from the specified config.MetricsDataLocation upon initializing. +// It does not reload the data file for each metric. +func New(config *config.Config) (*MetricsJSONDriver, error) { + // the json driver reads the data metrics file upon initializing + metricsData, err := readJSON(config) + if err != nil { + return nil, err + } + + driver := &MetricsJSONDriver{ + config: config, + data: metricsData, + } + + return driver, nil +} + +func readJSON(config *config.Config) (*data, error) { + if config.MetricsDataLocation == "" { + err := errors.New("Unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?") + return nil, err + } + + file, err := ioutil.ReadFile(config.MetricsDataLocation) + if err != nil { + return nil, err + } + + data := &data{} + err = json.Unmarshal(file, data) + if err != nil { + return nil, err + } + + return data, nil +} + +type data struct { + NumUsers int64 `json:"cs3_org_sciencemesh_site_total_num_users"` + NumGroups int64 `json:"cs3_org_sciencemesh_site_total_num_groups"` + AmountStorage int64 `json:"cs3_org_sciencemesh_site_total_amount_storage"` +} + +// MetricsJSONDriver the JsonDriver struct that also holds the data +type MetricsJSONDriver struct { + config *config.Config + data *data +} + +// GetNumUsers returns the number of site users +func (d *MetricsJSONDriver) GetNumUsers() int64 { + return int64(d.data.NumUsers) +} + +// GetNumGroups returns the number of site groups +func (d *MetricsJSONDriver) GetNumGroups() int64 { + return int64(d.data.NumGroups) +} + +// GetAmountStorage returns the amount of site storage used +func (d *MetricsJSONDriver) GetAmountStorage() int64 { + return int64(d.data.AmountStorage) +} From 0eaecaf026d9fea8fa4d3d2c7f5a8d52e3c1d9e5 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 15:41:51 +0200 Subject: [PATCH 10/43] Struct name changed to please hound bot. --- pkg/metrics/driver/dummy/dummy.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/metrics/driver/dummy/dummy.go b/pkg/metrics/driver/dummy/dummy.go index ddd5b2424e..9e4ec95fe9 100644 --- a/pkg/metrics/driver/dummy/dummy.go +++ b/pkg/metrics/driver/dummy/dummy.go @@ -23,31 +23,31 @@ import ( "reva/pkg/metrics/config" ) -// New returns a new DummyDataDriver object. -func New(config *config.Config) (*DummyDataDriver, error) { - driver := &DummyDataDriver{ +// New returns a new MetricsDummyDriver object. +func New(config *config.Config) (*MetricsDummyDriver, error) { + driver := &MetricsDummyDriver{ config: config, } return driver, nil } -// DummyDataDriver the DummyDataDriver struct -type DummyDataDriver struct { +// MetricsDummyDriver the MetricsDummyDriver struct +type MetricsDummyDriver struct { config *config.Config } // GetNumUsers returns the number of site users, it's a dummy number -func (d *DummyDataDriver) GetNumUsers() int64 { +func (d *MetricsDummyDriver) GetNumUsers() int64 { return int64(rand.Intn(30000)) } // GetNumGroups returns the number of site groups, it's a dummy number -func (d *DummyDataDriver) GetNumGroups() int64 { +func (d *MetricsDummyDriver) GetNumGroups() int64 { return int64(rand.Intn(200)) } // GetAmountStorage returns the amount of site storage used, it's a dummy amount -func (d *DummyDataDriver) GetAmountStorage() int64 { +func (d *MetricsDummyDriver) GetAmountStorage() int64 { return int64(rand.Intn(70000000000)) } From 51c6b1463dbdd042c2f1aea1f6952cb7663ad759 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 17:58:33 +0200 Subject: [PATCH 11/43] Moved. --- pkg/metrics/reader/driver/dummy.go | 53 ----------------- pkg/metrics/reader/driver/json.go | 91 ------------------------------ 2 files changed, 144 deletions(-) delete mode 100644 pkg/metrics/reader/driver/dummy.go delete mode 100644 pkg/metrics/reader/driver/json.go diff --git a/pkg/metrics/reader/driver/dummy.go b/pkg/metrics/reader/driver/dummy.go deleted file mode 100644 index f454688c47..0000000000 --- a/pkg/metrics/reader/driver/dummy.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package driver - -import ( - "math/rand" - "reva/pkg/metrics/config" -) - -// New returns a new DummyDriver object. -func New(config *config.Config) (*DummyDriver, error) { - driver := &DummyDriver{ - config: config, - } - - return driver, nil -} - -// DummyDriver the DummyDriver struct -type DummyDriver struct { - config *config.Config -} - -// GetNumUsers returns the number of site users, it's a dummy number -func (d *DummyDriver) GetNumUsers() int64 { - return int64(rand.Intn(30000)) -} - -// GetNumGroups returns the number of site groups, it's a dummy number -func (d *DummyDriver) GetNumGroups() int64 { - return int64(rand.Intn(200)) -} - -// GetAmountStorage returns the amount of site storage used, it's a dummy amount -func (d *DummyDriver) GetAmountStorage() int64 { - return int64(rand.Intn(70000000000)) -} diff --git a/pkg/metrics/reader/driver/json.go b/pkg/metrics/reader/driver/json.go deleted file mode 100644 index b0f7a60f0d..0000000000 --- a/pkg/metrics/reader/driver/json.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package driver - -import ( - "encoding/json" - "errors" - "io/ioutil" - "reva/pkg/metrics/config" -) - -// New returns a new MetricsJSONDriver object. -// It reads the data file from the specified config.MetricsDataLocation upon initializing. -// It does not reload the data file for each metric. -func New(config *config.Config) (*MetricsJSONDriver, error) { - // the json driver reads the data metrics file upon initializing - metricsData, err := readJSON(config) - if err != nil { - return nil, err - } - - driver := &MetricsJSONDriver{ - config: config, - data: metricsData, - } - - return driver, nil -} - -func readJSON(config *config.Config) (*data, error) { - if config.MetricsDataLocation == "" { - err := errors.New("Unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?") - return nil, err - } - - file, err := ioutil.ReadFile(config.MetricsDataLocation) - if err != nil { - return nil, err - } - - data := &data{} - err = json.Unmarshal(file, data) - if err != nil { - return nil, err - } - - return data, nil -} - -type data struct { - NumUsers int64 `json:"cs3_org_sciencemesh_site_total_num_users"` - NumGroups int64 `json:"cs3_org_sciencemesh_site_total_num_groups"` - AmountStorage int64 `json:"cs3_org_sciencemesh_site_total_amount_storage"` -} - -// MetricsJSONDriver the JsonDriver struct that also holds the data -type MetricsJSONDriver struct { - config *config.Config - data *data -} - -// GetNumUsers returns the number of site users -func (d *MetricsJSONDriver) GetNumUsers() int64 { - return int64(d.data.NumUsers) -} - -// GetNumGroups returns the number of site groups -func (d *MetricsJSONDriver) GetNumGroups() int64 { - return int64(d.data.NumGroups) -} - -// GetAmountStorage returns the amount of site storage used -func (d *MetricsJSONDriver) GetAmountStorage() int64 { - return int64(d.data.AmountStorage) -} From 81a59e0e2911c06bf87650e7e371670be210d288 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 17:59:19 +0200 Subject: [PATCH 12/43] Rebase on SysInfo metrics --- internal/http/services/prometheus/prometheus.go | 2 -- pkg/metrics/driver/dummy/dummy.go | 3 ++- pkg/metrics/driver/json/json.go | 3 ++- pkg/metrics/metrics.go | 7 ++++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index 7330fcb111..3f4ff6cdd2 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -28,8 +28,6 @@ import ( "go.opencensus.io/stats/view" "github.com/cs3org/reva/pkg/rhttp/global" - // Initializes goroutines which periodically update stats - _ "github.com/cs3org/reva/pkg/metrics/reader/dummy" ) func init() { diff --git a/pkg/metrics/driver/dummy/dummy.go b/pkg/metrics/driver/dummy/dummy.go index 9e4ec95fe9..010201610d 100644 --- a/pkg/metrics/driver/dummy/dummy.go +++ b/pkg/metrics/driver/dummy/dummy.go @@ -20,7 +20,8 @@ package dummy import ( "math/rand" - "reva/pkg/metrics/config" + + "github.com/cs3org/reva/pkg/metrics/config" ) // New returns a new MetricsDummyDriver object. diff --git a/pkg/metrics/driver/json/json.go b/pkg/metrics/driver/json/json.go index e9481e9877..a7d79eb670 100644 --- a/pkg/metrics/driver/json/json.go +++ b/pkg/metrics/driver/json/json.go @@ -22,7 +22,8 @@ import ( "encoding/json" "errors" "io/ioutil" - "reva/pkg/metrics/config" + + "github.com/cs3org/reva/pkg/metrics/config" ) // New returns a new MetricsJSONDriver object. diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 4cddd768f1..af865f38a9 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -21,9 +21,10 @@ package metrics import ( "context" "errors" - "reva/pkg/metrics/config" - "reva/pkg/metrics/driver/dummy" - "reva/pkg/metrics/driver/json" + + "github.com/cs3org/reva/pkg/metrics/config" + "github.com/cs3org/reva/pkg/metrics/driver/dummy" + "github.com/cs3org/reva/pkg/metrics/driver/json" "github.com/rs/zerolog/log" "go.opencensus.io/stats" From 7bb316a13cb9d936b4caf8f8a7f625bb6df7c5e5 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 14 Jul 2020 18:21:14 +0200 Subject: [PATCH 13/43] Fixed unnecessary conversion. --- pkg/metrics/driver/json/json.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/metrics/driver/json/json.go b/pkg/metrics/driver/json/json.go index a7d79eb670..29b880dfab 100644 --- a/pkg/metrics/driver/json/json.go +++ b/pkg/metrics/driver/json/json.go @@ -78,15 +78,15 @@ type MetricsJSONDriver struct { // GetNumUsers returns the number of site users func (d *MetricsJSONDriver) GetNumUsers() int64 { - return int64(d.data.NumUsers) + return d.data.NumUsers } // GetNumGroups returns the number of site groups func (d *MetricsJSONDriver) GetNumGroups() int64 { - return int64(d.data.NumGroups) + return d.data.NumGroups } // GetAmountStorage returns the amount of site storage used func (d *MetricsJSONDriver) GetAmountStorage() int64 { - return int64(d.data.AmountStorage) + return d.data.AmountStorage } From db498a927301c3ce2e00440c72ecf75248d43a08 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Wed, 15 Jul 2020 09:43:47 +0200 Subject: [PATCH 14/43] Changelog added. --- changelog/unreleased/read-metrics-from-file.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/read-metrics-from-file.md diff --git a/changelog/unreleased/read-metrics-from-file.md b/changelog/unreleased/read-metrics-from-file.md new file mode 100644 index 0000000000..d6aabb1c7c --- /dev/null +++ b/changelog/unreleased/read-metrics-from-file.md @@ -0,0 +1,5 @@ +Enhancement: Metrics module can be configured to retrieve metrics data from file. + +- Export site metrics in Prometheus #698 + +https://github.com/cs3org/reva/pull/973 From 55ce5a95220c03101d155853a3938a7d11a4f66e Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Wed, 15 Jul 2020 16:45:05 +0200 Subject: [PATCH 15/43] Fixed: custom json file location not used. --- pkg/metrics/config/config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go index 195037973a..0cef040983 100644 --- a/pkg/metrics/config/config.go +++ b/pkg/metrics/config/config.go @@ -32,6 +32,8 @@ func (c *Config) Init() { } if c.MetricsDataDriverType == "json" { // default values - c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json" + if c.MetricsDataLocation == "" { + c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json" + } } } From c3b02425fed74f0fe57422de97df6bd745690a28 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 25 Aug 2020 09:16:54 +0200 Subject: [PATCH 16/43] Use sharedconfig instead. --- pkg/metrics/config/config.go | 39 ------------------------------------ 1 file changed, 39 deletions(-) delete mode 100644 pkg/metrics/config/config.go diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go deleted file mode 100644 index 0cef040983..0000000000 --- a/pkg/metrics/config/config.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package config - -// Config holds the config options that need to be passed down to the metrics reader -type Config struct { - Prefix string `mapstructure:"prefix"` - MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` - MetricsDataLocation string `mapstructure:"metrics_data_location"` -} - -// Init sets sane defaults -func (c *Config) Init() { - if c.Prefix == "" { - c.Prefix = "metrics" - } - if c.MetricsDataDriverType == "json" { - // default values - if c.MetricsDataLocation == "" { - c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json" - } - } -} From 3616aecd435163cc690cd189609adc1952cae941 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 25 Aug 2020 09:17:51 +0200 Subject: [PATCH 17/43] Does not need config. --- pkg/metrics/driver/dummy/dummy.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/metrics/driver/dummy/dummy.go b/pkg/metrics/driver/dummy/dummy.go index 010201610d..ccfa5f6db8 100644 --- a/pkg/metrics/driver/dummy/dummy.go +++ b/pkg/metrics/driver/dummy/dummy.go @@ -20,35 +20,29 @@ package dummy import ( "math/rand" - - "github.com/cs3org/reva/pkg/metrics/config" ) // New returns a new MetricsDummyDriver object. -func New(config *config.Config) (*MetricsDummyDriver, error) { - driver := &MetricsDummyDriver{ - config: config, - } - +func New() (*MetricsDummyDriver, error) { + driver := &MetricsDummyDriver{} return driver, nil } // MetricsDummyDriver the MetricsDummyDriver struct type MetricsDummyDriver struct { - config *config.Config } -// GetNumUsers returns the number of site users, it's a dummy number +// GetNumUsers returns the number of site users; it's a random number func (d *MetricsDummyDriver) GetNumUsers() int64 { return int64(rand.Intn(30000)) } -// GetNumGroups returns the number of site groups, it's a dummy number +// GetNumGroups returns the number of site groups; it's a random number func (d *MetricsDummyDriver) GetNumGroups() int64 { return int64(rand.Intn(200)) } -// GetAmountStorage returns the amount of site storage used, it's a dummy amount +// GetAmountStorage returns the amount of site storage used; it's a random amount func (d *MetricsDummyDriver) GetAmountStorage() int64 { return int64(rand.Intn(70000000000)) } From 3e503b9ef0a49f6a9e2fc8aa7718158c2038eff4 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 25 Aug 2020 09:18:49 +0200 Subject: [PATCH 18/43] Read file location from sharedconfig. --- pkg/metrics/driver/json/json.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/metrics/driver/json/json.go b/pkg/metrics/driver/json/json.go index 29b880dfab..3985c818f1 100644 --- a/pkg/metrics/driver/json/json.go +++ b/pkg/metrics/driver/json/json.go @@ -23,34 +23,35 @@ import ( "errors" "io/ioutil" - "github.com/cs3org/reva/pkg/metrics/config" + "github.com/cs3org/reva/pkg/sharedconf" ) // New returns a new MetricsJSONDriver object. // It reads the data file from the specified config.MetricsDataLocation upon initializing. // It does not reload the data file for each metric. -func New(config *config.Config) (*MetricsJSONDriver, error) { +func New() (*MetricsJSONDriver, error) { // the json driver reads the data metrics file upon initializing - metricsData, err := readJSON(config) + metricsData, err := readJSON() if err != nil { return nil, err } driver := &MetricsJSONDriver{ - config: config, - data: metricsData, + data: metricsData, } return driver, nil } -func readJSON(config *config.Config) (*data, error) { - if config.MetricsDataLocation == "" { +func readJSON() (*data, error) { + metricsDataLocation := sharedconf.GetMetricsDataLocation() + + if metricsDataLocation == "" { err := errors.New("Unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?") return nil, err } - file, err := ioutil.ReadFile(config.MetricsDataLocation) + file, err := ioutil.ReadFile(metricsDataLocation) if err != nil { return nil, err } @@ -72,8 +73,7 @@ type data struct { // MetricsJSONDriver the JsonDriver struct that also holds the data type MetricsJSONDriver struct { - config *config.Config - data *data + data *data } // GetNumUsers returns the number of site users From cf0eac5b1a06d5c14016d578ab42930ce3f4529e Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 25 Aug 2020 09:20:26 +0200 Subject: [PATCH 19/43] Add metrics data driver type, location. --- pkg/sharedconf/sharedconf.go | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/pkg/sharedconf/sharedconf.go b/pkg/sharedconf/sharedconf.go index 453f8137e5..43d8559cb5 100644 --- a/pkg/sharedconf/sharedconf.go +++ b/pkg/sharedconf/sharedconf.go @@ -28,9 +28,11 @@ import ( var sharedConf = &conf{} type conf struct { - JWTSecret string `mapstructure:"jwt_secret"` - GatewaySVC string `mapstructure:"gatewaysvc"` - DataGateway string `mapstructure:"datagateway"` + JWTSecret string `mapstructure:"jwt_secret"` + GatewaySVC string `mapstructure:"gatewaysvc"` + DataGateway string `mapstructure:"datagateway"` + MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` + MetricsDataLocation string `mapstructure:"metrics_data_location"` } // Decode decodes the configuration. @@ -60,6 +62,14 @@ func Decode(v interface{}) error { sharedConf.JWTSecret = "changemeplease" } + if sharedConf.MetricsDataDriverType == "" { + sharedConf.MetricsDataDriverType = "dummy" + } + + if sharedConf.MetricsDataLocation == "" { + sharedConf.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json" + } + return nil } @@ -86,3 +96,13 @@ func GetDataGateway(val string) string { } return val } + +// GetMetricsDataDriverType returns the package level metrics data driver +func GetMetricsDataDriverType() string { + return sharedConf.MetricsDataDriverType +} + +// GetMetricsDataLocation returns the package level metrics data location +func GetMetricsDataLocation() string { + return sharedConf.MetricsDataLocation +} From 2f86b9bda02634fef53dae2ce3f3d57de2c126f1 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 25 Aug 2020 09:22:48 +0200 Subject: [PATCH 20/43] Self initialize and periodic metrics data recording. Use sharedconfig. --- pkg/metrics/metrics.go | 50 ++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index af865f38a9..be4059babb 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -18,11 +18,18 @@ package metrics +/* +Metrics registers OpenCensus data views of the metrics. +The Reader interface defines the available metrics which are implemented in the drivers that read the metrics data. +Metrics initializes the correct driver as specified in the configuration. +*/ import ( "context" "errors" + "time" + + "github.com/cs3org/reva/pkg/sharedconf" - "github.com/cs3org/reva/pkg/metrics/config" "github.com/cs3org/reva/pkg/metrics/driver/dummy" "github.com/cs3org/reva/pkg/metrics/driver/json" @@ -31,13 +38,9 @@ import ( "go.opencensus.io/stats/view" ) -// New returns a Metrics object -func New(c *config.Config) (*Metrics, error) { +func init() { m := &Metrics{ - dataDriverType: "", - dataLocation: "", dataDriver: nil, - config: c, NumUsersMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless), NumGroupsMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless), AmountStorageMeasure: stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitBytes), @@ -50,25 +53,27 @@ func New(c *config.Config) (*Metrics, error) { m.getAmountStorageView(), ); err != nil { log.Error().Err(err).Msg("error registering the driver's views with opencensus exporter") - return nil, err } - return m, nil + // periodic record metrics data call; every 5 seconds + go func() { + for { + m.recordMetrics() + <-time.After(time.Millisecond * time.Duration(5000)) + } + }() } // Metrics the metrics struct type Metrics struct { - dataDriverType string - dataLocation string dataDriver Reader // the metrics data driver is an implemention of Reader - config *config.Config NumUsersMeasure *stats.Int64Measure NumGroupsMeasure *stats.Int64Measure AmountStorageMeasure *stats.Int64Measure } // RecordMetrics records the latest metrics from the metrics data source as OpenCensus stats views. -func (m *Metrics) RecordMetrics() error { +func (m *Metrics) recordMetrics() error { if err := initDataDriver(m); err != nil { log.Error().Err(err).Msg("Could not set a driver") return err @@ -83,19 +88,12 @@ func (m *Metrics) RecordMetrics() error { // initDataDriver initializes a data driver and sets it to be the Metrics.dataDriver func initDataDriver(m *Metrics) error { - // find out what driver to use - if m.config.MetricsDataDriverType == "" { - err := errors.New("Unable to initialize a metrics data driver, has a driver type (metrics_data_driver_type) been configured?") - return err - } - m.dataLocation = m.config.MetricsDataLocation + driverType := sharedconf.GetMetricsDataDriverType() // create/init a driver depending on driver type - if m.config.MetricsDataDriverType == "json" { - // Because the json metrics data file is only read on json driver creation - // a json driver must be re-created to make sure we have the current/latest metrics data. - // Other drivers may need creation only once. - jsonDriver, err := json.New(m.config) + if driverType == "json" { + // create a json driver + jsonDriver, err := json.New() if err != nil { log.Error().Err(err) return err @@ -103,9 +101,9 @@ func initDataDriver(m *Metrics) error { m.dataDriver = jsonDriver log.Debug().Msgf("Metrics uses json driver") } - if m.config.MetricsDataDriverType == "dummy" && m.dataDriver == nil { - // the dummy driver does not need to be initialized every time - dummyDriver, err := dummy.New(m.config) + if driverType == "dummy" && m.dataDriver == nil { + // create the dummy driver + dummyDriver, err := dummy.New() if err != nil { log.Error().Err(err) return err From 5e60a12cb76fb69e8eae7a0030cb9961f6186ce9 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 25 Aug 2020 09:42:39 +0200 Subject: [PATCH 21/43] Metrics conf moved to shared block. --- examples/metrics/metrics.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/examples/metrics/metrics.toml b/examples/metrics/metrics.toml index d5203f3b4c..584dc3b33d 100644 --- a/examples/metrics/metrics.toml +++ b/examples/metrics/metrics.toml @@ -1,10 +1,8 @@ [shared] -jwt_secret = "Pive-Fumkiu4" +metrics_data_driver_type = "json" +metrics_data_location = "" [http.services.prometheus] -# metrics_data_driver_type, one of: dummy, json -metrics_data_driver_type = "dummy" -metrics_data_location = "" [http] -address = "0.0.0.0:5550" \ No newline at end of file +address = "0.0.0.0:5550" From 1059a7a7bd379fc3dac4e056ffb4490a1d2d65a6 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 25 Aug 2020 09:55:46 +0200 Subject: [PATCH 22/43] Use driver type dummy by default. --- examples/metrics/metrics.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/metrics/metrics.toml b/examples/metrics/metrics.toml index 584dc3b33d..5d19f41296 100644 --- a/examples/metrics/metrics.toml +++ b/examples/metrics/metrics.toml @@ -1,5 +1,7 @@ [shared] -metrics_data_driver_type = "json" +# one of dummy, json. +metrics_data_driver_type = "dummy" +# if left unspecified location /var/tmp/reva/metrics/metricsdata.json is used (for driver type json). metrics_data_location = "" [http.services.prometheus] From 9d1d53f8aef6962bf4ade92f274d2d18586cde87 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 25 Aug 2020 11:02:13 +0200 Subject: [PATCH 23/43] Fixed error check. --- pkg/metrics/metrics.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index be4059babb..5449e84d47 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -58,7 +58,9 @@ func init() { // periodic record metrics data call; every 5 seconds go func() { for { - m.recordMetrics() + if err := m.recordMetrics(); err != nil { + log.Error().Err(err).Msg("Metrics recording failed.") + } <-time.After(time.Millisecond * time.Duration(5000)) } }() From 29f220a986eba9bf1fb4d727acb9fd2b48f56252 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 25 Aug 2020 11:13:52 +0200 Subject: [PATCH 24/43] Fixed title. --- changelog/unreleased/read-metrics-from-file.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog/unreleased/read-metrics-from-file.md b/changelog/unreleased/read-metrics-from-file.md index d6aabb1c7c..05473b5ba7 100644 --- a/changelog/unreleased/read-metrics-from-file.md +++ b/changelog/unreleased/read-metrics-from-file.md @@ -1,5 +1,5 @@ -Enhancement: Metrics module can be configured to retrieve metrics data from file. +Enhancement: Metrics module can be configured to retrieve metrics data from file - Export site metrics in Prometheus #698 -https://github.com/cs3org/reva/pull/973 +https://github.com/cs3org/reva/pull/1118 From db62bc192342e240419562a763e445d3484bbf62 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 13:00:17 +0200 Subject: [PATCH 25/43] Reset sharedconf. Metrics does not use it. --- pkg/sharedconf/sharedconf.go | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/pkg/sharedconf/sharedconf.go b/pkg/sharedconf/sharedconf.go index 43d8559cb5..453f8137e5 100644 --- a/pkg/sharedconf/sharedconf.go +++ b/pkg/sharedconf/sharedconf.go @@ -28,11 +28,9 @@ import ( var sharedConf = &conf{} type conf struct { - JWTSecret string `mapstructure:"jwt_secret"` - GatewaySVC string `mapstructure:"gatewaysvc"` - DataGateway string `mapstructure:"datagateway"` - MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` - MetricsDataLocation string `mapstructure:"metrics_data_location"` + JWTSecret string `mapstructure:"jwt_secret"` + GatewaySVC string `mapstructure:"gatewaysvc"` + DataGateway string `mapstructure:"datagateway"` } // Decode decodes the configuration. @@ -62,14 +60,6 @@ func Decode(v interface{}) error { sharedConf.JWTSecret = "changemeplease" } - if sharedConf.MetricsDataDriverType == "" { - sharedConf.MetricsDataDriverType = "dummy" - } - - if sharedConf.MetricsDataLocation == "" { - sharedConf.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json" - } - return nil } @@ -96,13 +86,3 @@ func GetDataGateway(val string) string { } return val } - -// GetMetricsDataDriverType returns the package level metrics data driver -func GetMetricsDataDriverType() string { - return sharedConf.MetricsDataDriverType -} - -// GetMetricsDataLocation returns the package level metrics data location -func GetMetricsDataLocation() string { - return sharedConf.MetricsDataLocation -} From c271c33060848204f7132611b9561d2c2f6f3082 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 13:01:01 +0200 Subject: [PATCH 26/43] Add metrics drivers to loader. --- cmd/revad/runtime/loader.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index ffdbb3bedb..7cc01473ac 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -29,7 +29,8 @@ import ( _ "github.com/cs3org/reva/internal/http/services/loader" _ "github.com/cs3org/reva/pkg/auth/manager/loader" _ "github.com/cs3org/reva/pkg/auth/registry/loader" - _ "github.com/cs3org/reva/pkg/metrics" + _ "github.com/cs3org/reva/pkg/metrics/driver/dummy" + _ "github.com/cs3org/reva/pkg/metrics/driver/json" _ "github.com/cs3org/reva/pkg/ocm/invite/manager/loader" _ "github.com/cs3org/reva/pkg/ocm/provider/authorizer/loader" _ "github.com/cs3org/reva/pkg/ocm/share/manager/loader" From e3f47c5dec4407862ecda9e510c68339b5fbd104 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 13:01:30 +0200 Subject: [PATCH 27/43] Add metrics service to loader. --- internal/http/services/loader/loader.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/http/services/loader/loader.go b/internal/http/services/loader/loader.go index 06a4534a49..4c92606a6e 100644 --- a/internal/http/services/loader/loader.go +++ b/internal/http/services/loader/loader.go @@ -25,6 +25,7 @@ import ( _ "github.com/cs3org/reva/internal/http/services/helloworld" _ "github.com/cs3org/reva/internal/http/services/mentix" _ "github.com/cs3org/reva/internal/http/services/meshdirectory" + _ "github.com/cs3org/reva/internal/http/services/metrics" _ "github.com/cs3org/reva/internal/http/services/ocmd" _ "github.com/cs3org/reva/internal/http/services/oidcprovider" _ "github.com/cs3org/reva/internal/http/services/owncloud/ocdav" From 6191a449db88718fd4de8ccbcdcec4f3302ea9eb Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 13:02:49 +0200 Subject: [PATCH 28/43] Config for metrics. --- pkg/metrics/config/config.go | 43 ++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 pkg/metrics/config/config.go diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go new file mode 100644 index 0000000000..c77fdbbb58 --- /dev/null +++ b/pkg/metrics/config/config.go @@ -0,0 +1,43 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +// Config holds the config options that need to be passed down to the metrics reader(driver) +type Config struct { + Prefix string `mapstructure:"prefix"` + MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` + MetricsDataLocation string `mapstructure:"metrics_data_location"` + MetricsRecordInterval int `mapstructure:"metrics_record_interval"` +} + +// Init sets sane defaults +func (c *Config) Init() { + if c.Prefix == "" { + c.Prefix = "metrics" + } + if c.MetricsDataDriverType == "json" { + // default values + if c.MetricsDataLocation == "" { + c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json" + } + } + if c.MetricsRecordInterval == 0 { + c.MetricsRecordInterval = 5000 + } +} From e16eb4a25036ca2d47bdb50ec03bcc9a162d526c Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 13:03:32 +0200 Subject: [PATCH 29/43] Drivers register themselfes. --- pkg/metrics/driver/dummy/dummy.go | 18 ++++++-- pkg/metrics/driver/json/json.go | 70 +++++++++++++++++-------------- 2 files changed, 53 insertions(+), 35 deletions(-) diff --git a/pkg/metrics/driver/dummy/dummy.go b/pkg/metrics/driver/dummy/dummy.go index ccfa5f6db8..3d7b8a7950 100644 --- a/pkg/metrics/driver/dummy/dummy.go +++ b/pkg/metrics/driver/dummy/dummy.go @@ -20,18 +20,30 @@ package dummy import ( "math/rand" + + "github.com/cs3org/reva/pkg/metrics/config" + "github.com/cs3org/reva/pkg/metrics/driver/registry" ) -// New returns a new MetricsDummyDriver object. -func New() (*MetricsDummyDriver, error) { +func init() { driver := &MetricsDummyDriver{} - return driver, nil + registry.Register(driverName(), driver) +} + +func driverName() string { + return "dummy" } // MetricsDummyDriver the MetricsDummyDriver struct type MetricsDummyDriver struct { } +// Configure configures this driver +func (d *MetricsDummyDriver) Configure(c *config.Config) error { + // no configuration necessary + return nil +} + // GetNumUsers returns the number of site users; it's a random number func (d *MetricsDummyDriver) GetNumUsers() int64 { return int64(rand.Intn(30000)) diff --git a/pkg/metrics/driver/json/json.go b/pkg/metrics/driver/json/json.go index 3985c818f1..a7f23af54d 100644 --- a/pkg/metrics/driver/json/json.go +++ b/pkg/metrics/driver/json/json.go @@ -22,47 +22,41 @@ import ( "encoding/json" "errors" "io/ioutil" + "os" - "github.com/cs3org/reva/pkg/sharedconf" -) + "github.com/cs3org/reva/pkg/metrics/driver/registry" -// New returns a new MetricsJSONDriver object. -// It reads the data file from the specified config.MetricsDataLocation upon initializing. -// It does not reload the data file for each metric. -func New() (*MetricsJSONDriver, error) { - // the json driver reads the data metrics file upon initializing - metricsData, err := readJSON() - if err != nil { - return nil, err - } + "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/metrics/config" + "github.com/rs/zerolog" +) - driver := &MetricsJSONDriver{ - data: metricsData, - } +var log zerolog.Logger - return driver, nil +func init() { + log = logger.New().With().Int("pid", os.Getpid()).Logger() + driver := &MetricsJSONDriver{} + registry.Register(driverName(), driver) } -func readJSON() (*data, error) { - metricsDataLocation := sharedconf.GetMetricsDataLocation() +func driverName() string { + return "json" +} - if metricsDataLocation == "" { - err := errors.New("Unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?") - return nil, err - } +// readJSON always returns a data object but logs the error in case reading the json fails. +func readJSON(driver *MetricsJSONDriver) *data { + data := &data{} - file, err := ioutil.ReadFile(metricsDataLocation) + file, err := ioutil.ReadFile(driver.metricsDataLocation) if err != nil { - return nil, err + log.Error().Err(err).Str("location", driver.metricsDataLocation).Msg("Unable to read json file from location.") } - - data := &data{} err = json.Unmarshal(file, data) if err != nil { - return nil, err + log.Error().Err(err).Msg("Unable to unmarshall json file.") } - return data, nil + return data } type data struct { @@ -71,22 +65,34 @@ type data struct { AmountStorage int64 `json:"cs3_org_sciencemesh_site_total_amount_storage"` } -// MetricsJSONDriver the JsonDriver struct that also holds the data +// MetricsJSONDriver the JsonDriver struct type MetricsJSONDriver struct { - data *data + metricsDataLocation string +} + +// Configure configures this driver +func (d *MetricsJSONDriver) Configure(c *config.Config) error { + if c.MetricsDataLocation == "" { + err := errors.New("Unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?") + return err + } + + d.metricsDataLocation = c.MetricsDataLocation + + return nil } // GetNumUsers returns the number of site users func (d *MetricsJSONDriver) GetNumUsers() int64 { - return d.data.NumUsers + return readJSON(d).NumUsers } // GetNumGroups returns the number of site groups func (d *MetricsJSONDriver) GetNumGroups() int64 { - return d.data.NumGroups + return readJSON(d).NumGroups } // GetAmountStorage returns the amount of site storage used func (d *MetricsJSONDriver) GetAmountStorage() int64 { - return d.data.AmountStorage + return readJSON(d).AmountStorage } From 93180dcff528b1b8ca3d1a489c12df0b04084a10 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 13:03:58 +0200 Subject: [PATCH 30/43] New metrics service. --- internal/http/services/metrics/metrics.go | 92 +++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 internal/http/services/metrics/metrics.go diff --git a/internal/http/services/metrics/metrics.go b/internal/http/services/metrics/metrics.go new file mode 100644 index 0000000000..b18b222991 --- /dev/null +++ b/internal/http/services/metrics/metrics.go @@ -0,0 +1,92 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package metrics + +/* +This service initializes the metrics package according to the metrics configuration. +*/ +import ( + "net/http" + "os" + + "github.com/cs3org/reva/pkg/logger" + "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/metrics" + "github.com/cs3org/reva/pkg/metrics/config" + "github.com/cs3org/reva/pkg/rhttp/global" +) + +func init() { + global.Register(serviceName, New) +} + +const ( + serviceName = "metrics" +) + +// Close is called when this service is being stopped. +func (s *svc) Close() error { + return nil +} + +// Prefix returns the main endpoint of this service. +func (s *svc) Prefix() string { + return s.prefix +} + +// Unprotected returns all endpoints that can be queried without prior authorization. +func (s *svc) Unprotected() []string { + return []string{"/"} +} + +// Handler serves all HTTP requests. +func (s *svc) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log := logger.New().With().Int("pid", os.Getpid()).Logger() + if _, err := w.Write([]byte("This is the metrics service.\n")); err != nil { + log.Error().Err(err).Msg("error writing metrics response") + } + }) +} + +// New returns a new metrics service. +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { + // Prepare the configuration + conf := &config.Config{} + if err := mapstructure.Decode(m, conf); err != nil { + return nil, err + } + + conf.Init() + + // initialize metrics using the configuration + metrics.Init(conf) + + // Create the service + s := &svc{ + prefix: conf.Prefix, + } + return s, nil +} + +type svc struct { + prefix string +} From c418d2042577c37786f097b0d7d784efea38e75e Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 13:04:31 +0200 Subject: [PATCH 31/43] Driver register. --- pkg/metrics/driver/registry/registry.go | 59 +++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 pkg/metrics/driver/registry/registry.go diff --git a/pkg/metrics/driver/registry/registry.go b/pkg/metrics/driver/registry/registry.go new file mode 100644 index 0000000000..250b4f1f31 --- /dev/null +++ b/pkg/metrics/driver/registry/registry.go @@ -0,0 +1,59 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import ( + "os" + + "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/metrics/reader" + "github.com/rs/zerolog" +) + +var log zerolog.Logger + +var drivers map[string]reader.Reader // map key is driver type name + +func init() { + log = logger.New().With().Int("pid", os.Getpid()).Logger() +} + +// Register register a driver +func Register(driverName string, r reader.Reader) { + if drivers == nil { + drivers = make(map[string]reader.Reader) + } + _, found := drivers[driverName] + if found { + log.Debug().Str("driverName", driverName).Msg("Metrics driver already registered.") + } else { + drivers[driverName] = r + log.Debug().Str("driverName", driverName).Msg("Metrics driver registered.") + } +} + +// GetDriver returns the registered driver for the specified driver name, or nil if it is not registered +func GetDriver(driverName string) reader.Reader { + driver, found := drivers[driverName] + if found { + return driver + } + log.Debug().Str("driverName", driverName).Msg("Metrics driver not found in metrics driver registry.") + return nil +} From f9b310326d09f131e17a0ed51c416c7c19a91f5c Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 13:06:28 +0200 Subject: [PATCH 32/43] Use register to get configured driver. --- pkg/metrics/metrics.go | 139 +++++++++++++---------------------------- 1 file changed, 45 insertions(+), 94 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 5449e84d47..6460f44bde 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -20,55 +20,64 @@ package metrics /* Metrics registers OpenCensus data views of the metrics. -The Reader interface defines the available metrics which are implemented in the drivers that read the metrics data. -Metrics initializes the correct driver as specified in the configuration. +Metrics initializes the driver as specified in the configuration. */ import ( "context" - "errors" + "os" "time" - "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/metrics/config" + "github.com/cs3org/reva/pkg/metrics/driver/registry" + "github.com/cs3org/reva/pkg/metrics/reader" - "github.com/cs3org/reva/pkg/metrics/driver/dummy" - "github.com/cs3org/reva/pkg/metrics/driver/json" - - "github.com/rs/zerolog/log" + "github.com/cs3org/reva/pkg/logger" "go.opencensus.io/stats" "go.opencensus.io/stats/view" ) -func init() { - m := &Metrics{ - dataDriver: nil, - NumUsersMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless), - NumGroupsMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless), - AmountStorageMeasure: stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitBytes), - } +// Init intializes metrics according to the specified configuration +func Init(conf *config.Config) { + log := logger.New().With().Int("pid", os.Getpid()).Logger() - // register the desired measures' views - if err := view.Register( - m.getNumUsersView(), - m.getNumGroupsView(), - m.getAmountStorageView(), - ); err != nil { - log.Error().Err(err).Msg("error registering the driver's views with opencensus exporter") - } + driver := registry.GetDriver(conf.MetricsDataDriverType) - // periodic record metrics data call; every 5 seconds - go func() { - for { - if err := m.recordMetrics(); err != nil { - log.Error().Err(err).Msg("Metrics recording failed.") - } - <-time.After(time.Millisecond * time.Duration(5000)) + if driver != nil { + // configure the driver + driver.Configure(conf) + + m := &Metrics{ + dataDriver: driver, + NumUsersMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless), + NumGroupsMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless), + AmountStorageMeasure: stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitBytes), } - }() + + if err := view.Register( + m.getNumUsersView(), + m.getNumGroupsView(), + m.getAmountStorageView(), + ); err != nil { + log.Error().Msg("error registering the driver's views with opencensus exporter") + } + + // periodically record metrics data + go func() { + for { + if err := m.recordMetrics(); err != nil { + log.Error().Err(err).Msg("Metrics recording failed.") + } + <-time.After(time.Millisecond * time.Duration(conf.MetricsRecordInterval)) + } + }() + } else { + log.Info().Msg("No metrics are being recorded.") + } } // Metrics the metrics struct type Metrics struct { - dataDriver Reader // the metrics data driver is an implemention of Reader + dataDriver reader.Reader // the metrics data driver is an implemention of Reader NumUsersMeasure *stats.Int64Measure NumGroupsMeasure *stats.Int64Measure AmountStorageMeasure *stats.Int64Measure @@ -76,49 +85,12 @@ type Metrics struct { // RecordMetrics records the latest metrics from the metrics data source as OpenCensus stats views. func (m *Metrics) recordMetrics() error { - if err := initDataDriver(m); err != nil { - log.Error().Err(err).Msg("Could not set a driver") - return err - } // record all latest metrics - m.recordNumUsers() - m.recordNumGroups() - m.recordAmountStorage() - - return nil -} - -// initDataDriver initializes a data driver and sets it to be the Metrics.dataDriver -func initDataDriver(m *Metrics) error { - driverType := sharedconf.GetMetricsDataDriverType() - - // create/init a driver depending on driver type - if driverType == "json" { - // create a json driver - jsonDriver, err := json.New() - if err != nil { - log.Error().Err(err) - return err - } - m.dataDriver = jsonDriver - log.Debug().Msgf("Metrics uses json driver") + if m.dataDriver != nil { + m.recordNumUsers() + m.recordNumGroups() + m.recordAmountStorage() } - if driverType == "dummy" && m.dataDriver == nil { - // create the dummy driver - dummyDriver, err := dummy.New() - if err != nil { - log.Error().Err(err) - return err - } - m.dataDriver = dummyDriver - log.Debug().Msgf("Metrics uses dummy driver") - } - // no known driver configured, return error - if m.dataDriver == nil { - err := errors.New("Unable to initialize a metrics data driver. Has a correct driver type (one of: json, dummy) been configured?") - return err - } - return nil } @@ -166,24 +138,3 @@ func (m *Metrics) getAmountStorageView() *view.View { Aggregation: view.LastValue(), } } - -// Reader is the interface that defines the metrics to read. -// Any metrics data driver must implement this interface. -// Each function should return the current/latest available metrics figure relevant to that function. -type Reader interface { - - // GetNumUsersView returns an OpenCensus stats view which records the - // number of users registered in the mesh provider. - // Metric name: cs3_org_sciencemesh_site_total_num_users - GetNumUsers() int64 - - // GetNumGroupsView returns an OpenCensus stats view which records the - // number of user groups registered in the mesh provider. - // Metric name: cs3_org_sciencemesh_site_total_num_groups - GetNumGroups() int64 - - // GetAmountStorageView returns an OpenCensus stats view which records the - // amount of storage in the system. - // Metric name: cs3_org_sciencemesh_site_total_amount_storage - GetAmountStorage() int64 -} From e419e36831eec55825866a0b36be724f85c40f10 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 13:07:36 +0200 Subject: [PATCH 33/43] Moved to dedicated namespace. --- pkg/metrics/reader/reader.go | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 pkg/metrics/reader/reader.go diff --git a/pkg/metrics/reader/reader.go b/pkg/metrics/reader/reader.go new file mode 100644 index 0000000000..650e50bdd8 --- /dev/null +++ b/pkg/metrics/reader/reader.go @@ -0,0 +1,49 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package reader + +/* +Reader is the interface that defines the metrics to read. +Any metrics data driver must implement this interface. +Each metric function should return the current/latest available metrics figure relevant to that function. +*/ + +import "github.com/cs3org/reva/pkg/metrics/config" + +// Reader the Reader interface +type Reader interface { + + // Configure configures the reader according to the specified configuration + Configure(c *config.Config) error + + // GetNumUsersView returns an OpenCensus stats view which records the + // number of users registered in the mesh provider. + // Metric name: cs3_org_sciencemesh_site_total_num_users + GetNumUsers() int64 + + // GetNumGroupsView returns an OpenCensus stats view which records the + // number of user groups registered in the mesh provider. + // Metric name: cs3_org_sciencemesh_site_total_num_groups + GetNumGroups() int64 + + // GetAmountStorageView returns an OpenCensus stats view which records the + // amount of storage in the system. + // Metric name: cs3_org_sciencemesh_site_total_amount_storage + GetAmountStorage() int64 +} From e6a14f9965abea90bf72a9a7438a08e9f643e5f6 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 13:23:03 +0200 Subject: [PATCH 34/43] Changelog added. --- changelog/unreleased/read-metrics-from-file-2.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/read-metrics-from-file-2.md diff --git a/changelog/unreleased/read-metrics-from-file-2.md b/changelog/unreleased/read-metrics-from-file-2.md new file mode 100644 index 0000000000..05473b5ba7 --- /dev/null +++ b/changelog/unreleased/read-metrics-from-file-2.md @@ -0,0 +1,5 @@ +Enhancement: Metrics module can be configured to retrieve metrics data from file + +- Export site metrics in Prometheus #698 + +https://github.com/cs3org/reva/pull/1118 From ab30488a3e5ab1c05cf8148f493d5acfdf19538d Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 17:18:46 +0200 Subject: [PATCH 35/43] Renamed. --- changelog/unreleased/read-metrics-from-file.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 changelog/unreleased/read-metrics-from-file.md diff --git a/changelog/unreleased/read-metrics-from-file.md b/changelog/unreleased/read-metrics-from-file.md deleted file mode 100644 index 05473b5ba7..0000000000 --- a/changelog/unreleased/read-metrics-from-file.md +++ /dev/null @@ -1,5 +0,0 @@ -Enhancement: Metrics module can be configured to retrieve metrics data from file - -- Export site metrics in Prometheus #698 - -https://github.com/cs3org/reva/pull/1118 From 6aa7da84623420616c33c9668bff3e4e67096fcf Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 17:19:51 +0200 Subject: [PATCH 36/43] New parameter: metrics_record_interval --- examples/metrics/metrics.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/metrics/metrics.toml b/examples/metrics/metrics.toml index 5d19f41296..ce0f89c2be 100644 --- a/examples/metrics/metrics.toml +++ b/examples/metrics/metrics.toml @@ -3,6 +3,8 @@ metrics_data_driver_type = "dummy" # if left unspecified location /var/tmp/reva/metrics/metricsdata.json is used (for driver type json). metrics_data_location = "" +# metrics recording interval in milliseconds +metrics_record_interval = 5000 [http.services.prometheus] From 69d8a666609ef219ef822f9f195bdd9f1aef972c Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 17:20:37 +0200 Subject: [PATCH 37/43] Init returns error. --- internal/http/services/metrics/metrics.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/http/services/metrics/metrics.go b/internal/http/services/metrics/metrics.go index b18b222991..817f507bcc 100644 --- a/internal/http/services/metrics/metrics.go +++ b/internal/http/services/metrics/metrics.go @@ -78,7 +78,10 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) conf.Init() // initialize metrics using the configuration - metrics.Init(conf) + err := metrics.Init(conf) + if err != nil { + return nil, err + } // Create the service s := &svc{ From 66cd4352755585600b38e2acac8c4dfd82cdd17d Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 4 Sep 2020 17:21:21 +0200 Subject: [PATCH 38/43] Init returns error. --- pkg/metrics/metrics.go | 63 +++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 6460f44bde..586e4dd41f 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -37,42 +37,49 @@ import ( ) // Init intializes metrics according to the specified configuration -func Init(conf *config.Config) { +func Init(conf *config.Config) error { log := logger.New().With().Int("pid", os.Getpid()).Logger() driver := registry.GetDriver(conf.MetricsDataDriverType) - if driver != nil { - // configure the driver - driver.Configure(conf) + if driver == nil { + log.Info().Msg("No metrics are being recorded.") + // No error but just don't proceed with metrics + return nil + } - m := &Metrics{ - dataDriver: driver, - NumUsersMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless), - NumGroupsMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless), - AmountStorageMeasure: stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitBytes), - } + // configure the driver + err := driver.Configure(conf) + if err != nil { + return err + } - if err := view.Register( - m.getNumUsersView(), - m.getNumGroupsView(), - m.getAmountStorageView(), - ); err != nil { - log.Error().Msg("error registering the driver's views with opencensus exporter") - } + m := &Metrics{ + dataDriver: driver, + NumUsersMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless), + NumGroupsMeasure: stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless), + AmountStorageMeasure: stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitBytes), + } - // periodically record metrics data - go func() { - for { - if err := m.recordMetrics(); err != nil { - log.Error().Err(err).Msg("Metrics recording failed.") - } - <-time.After(time.Millisecond * time.Duration(conf.MetricsRecordInterval)) - } - }() - } else { - log.Info().Msg("No metrics are being recorded.") + if err := view.Register( + m.getNumUsersView(), + m.getNumGroupsView(), + m.getAmountStorageView(), + ); err != nil { + return err } + + // periodically record metrics data + go func() { + for { + if err := m.recordMetrics(); err != nil { + log.Error().Err(err).Msg("Metrics recording failed.") + } + <-time.After(time.Millisecond * time.Duration(conf.MetricsRecordInterval)) + } + }() + + return nil } // Metrics the metrics struct From c6bbcedfa423748a11988746692fadff9ce9d31e Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Wed, 9 Sep 2020 13:17:06 +0200 Subject: [PATCH 39/43] Fix missing service line. --- examples/metrics/metrics.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/metrics/metrics.toml b/examples/metrics/metrics.toml index ce0f89c2be..3dd0a39e73 100644 --- a/examples/metrics/metrics.toml +++ b/examples/metrics/metrics.toml @@ -1,4 +1,6 @@ [shared] + +[http.services.metrics] # one of dummy, json. metrics_data_driver_type = "dummy" # if left unspecified location /var/tmp/reva/metrics/metricsdata.json is used (for driver type json). From 1f8b376496a68e8df780e95d5352ae0adaa55475 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Wed, 9 Sep 2020 13:19:36 +0200 Subject: [PATCH 40/43] Load specific drivers in pkg loader. --- cmd/revad/runtime/loader.go | 3 +-- pkg/metrics/driver/loader/loader.go | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 pkg/metrics/driver/loader/loader.go diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index 7cc01473ac..21175fbde1 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -29,8 +29,7 @@ import ( _ "github.com/cs3org/reva/internal/http/services/loader" _ "github.com/cs3org/reva/pkg/auth/manager/loader" _ "github.com/cs3org/reva/pkg/auth/registry/loader" - _ "github.com/cs3org/reva/pkg/metrics/driver/dummy" - _ "github.com/cs3org/reva/pkg/metrics/driver/json" + _ "github.com/cs3org/reva/pkg/metrics/driver/loader" _ "github.com/cs3org/reva/pkg/ocm/invite/manager/loader" _ "github.com/cs3org/reva/pkg/ocm/provider/authorizer/loader" _ "github.com/cs3org/reva/pkg/ocm/share/manager/loader" diff --git a/pkg/metrics/driver/loader/loader.go b/pkg/metrics/driver/loader/loader.go new file mode 100644 index 0000000000..97adaf9278 --- /dev/null +++ b/pkg/metrics/driver/loader/loader.go @@ -0,0 +1,26 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package loader + +import ( + // Load metrics drivers. + _ "github.com/cs3org/reva/pkg/metrics/driver/dummy" + _ "github.com/cs3org/reva/pkg/metrics/driver/json" + // Add your own here +) From 375b58808b02b096228d1b8bc745fd5912ad5a2e Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Wed, 9 Sep 2020 14:53:34 +0200 Subject: [PATCH 41/43] Remove prefix. --- internal/http/services/metrics/metrics.go | 8 +++----- pkg/metrics/config/config.go | 4 ---- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/internal/http/services/metrics/metrics.go b/internal/http/services/metrics/metrics.go index 817f507bcc..31fd904a2b 100644 --- a/internal/http/services/metrics/metrics.go +++ b/internal/http/services/metrics/metrics.go @@ -49,12 +49,12 @@ func (s *svc) Close() error { // Prefix returns the main endpoint of this service. func (s *svc) Prefix() string { - return s.prefix + return "" } // Unprotected returns all endpoints that can be queried without prior authorization. func (s *svc) Unprotected() []string { - return []string{"/"} + return []string{} } // Handler serves all HTTP requests. @@ -84,9 +84,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) } // Create the service - s := &svc{ - prefix: conf.Prefix, - } + s := &svc{} return s, nil } diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go index c77fdbbb58..f4d6dbe3b2 100644 --- a/pkg/metrics/config/config.go +++ b/pkg/metrics/config/config.go @@ -20,7 +20,6 @@ package config // Config holds the config options that need to be passed down to the metrics reader(driver) type Config struct { - Prefix string `mapstructure:"prefix"` MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` MetricsDataLocation string `mapstructure:"metrics_data_location"` MetricsRecordInterval int `mapstructure:"metrics_record_interval"` @@ -28,9 +27,6 @@ type Config struct { // Init sets sane defaults func (c *Config) Init() { - if c.Prefix == "" { - c.Prefix = "metrics" - } if c.MetricsDataDriverType == "json" { // default values if c.MetricsDataLocation == "" { From 3cf7ae2bb23372a59414817262bc762cc426dcd8 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Wed, 9 Sep 2020 14:56:40 +0200 Subject: [PATCH 42/43] Remove prefix. --- internal/http/services/metrics/metrics.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/http/services/metrics/metrics.go b/internal/http/services/metrics/metrics.go index 31fd904a2b..3ac1680afb 100644 --- a/internal/http/services/metrics/metrics.go +++ b/internal/http/services/metrics/metrics.go @@ -89,5 +89,4 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) } type svc struct { - prefix string } From 473da686258e62381ca201e9903bcefec4a45521 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Wed, 9 Sep 2020 15:10:42 +0200 Subject: [PATCH 43/43] Clean up. --- pkg/metrics/driver/registry/registry.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/pkg/metrics/driver/registry/registry.go b/pkg/metrics/driver/registry/registry.go index 250b4f1f31..6c982619f6 100644 --- a/pkg/metrics/driver/registry/registry.go +++ b/pkg/metrics/driver/registry/registry.go @@ -19,33 +19,17 @@ package registry import ( - "os" - - "github.com/cs3org/reva/pkg/logger" "github.com/cs3org/reva/pkg/metrics/reader" - "github.com/rs/zerolog" ) -var log zerolog.Logger - var drivers map[string]reader.Reader // map key is driver type name -func init() { - log = logger.New().With().Int("pid", os.Getpid()).Logger() -} - // Register register a driver func Register(driverName string, r reader.Reader) { if drivers == nil { drivers = make(map[string]reader.Reader) } - _, found := drivers[driverName] - if found { - log.Debug().Str("driverName", driverName).Msg("Metrics driver already registered.") - } else { - drivers[driverName] = r - log.Debug().Str("driverName", driverName).Msg("Metrics driver registered.") - } + drivers[driverName] = r } // GetDriver returns the registered driver for the specified driver name, or nil if it is not registered @@ -54,6 +38,5 @@ func GetDriver(driverName string) reader.Reader { if found { return driver } - log.Debug().Str("driverName", driverName).Msg("Metrics driver not found in metrics driver registry.") return nil }