Skip to content

Commit

Permalink
🌱 cron: expose the stackdriver prefix as a config variable so it can …
Browse files Browse the repository at this point in the history 
…be changed. (ossf#2446)

* Expose the stackdriver prefix as a config variable so it can be changed.

Signed-off-by: Caleb Brown <calebbrown@google.com>

* fix linter warning

Signed-off-by: Caleb Brown <calebbrown@google.com>

Signed-off-by: Caleb Brown <calebbrown@google.com>
Co-authored-by: Spencer Schrock <sschrock@google.com>
Signed-off-by: latortuga71 <christopheralonso1@gmail.com>
  • Loading branch information
@calebbrown @spencerschrock @latortuga71
2 people authored and latortuga71 committed Nov 18, 2022
1 parent 239b3b2 commit 49e9418
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 55 deletions.
60 changes: 34 additions & 26 deletions cron/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,19 +42,20 @@ const (
configUsage string = "Location of config file. Required"
inputBucketParams string = "input-bucket"

projectID string = "SCORECARD_PROJECT_ID"
requestTopicURL string = "SCORECARD_REQUEST_TOPIC_URL"
requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL"
bigqueryDataset string = "SCORECARD_BIGQUERY_DATASET"
completionThreshold string = "SCORECARD_COMPLETION_THRESHOLD"
shardSize string = "SCORECARD_SHARD_SIZE"
webhookURL string = "SCORECARD_WEBHOOK_URL"
metricExporter string = "SCORECARD_METRIC_EXPORTER"
bigqueryTable string = "SCORECARD_BIGQUERY_TABLE"
resultDataBucketURL string = "SCORECARD_DATA_BUCKET_URL"
apiResultsBucketURL string = "SCORECARD_API_RESULTS_BUCKET_URL"
inputBucketURL string = "SCORECARD_INPUT_BUCKET_URL"
inputBucketPrefix string = "SCORECARD_INPUT_BUCKET_PREFIX"
projectID string = "SCORECARD_PROJECT_ID"
requestTopicURL string = "SCORECARD_REQUEST_TOPIC_URL"
requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL"
bigqueryDataset string = "SCORECARD_BIGQUERY_DATASET"
completionThreshold string = "SCORECARD_COMPLETION_THRESHOLD"
shardSize string = "SCORECARD_SHARD_SIZE"
webhookURL string = "SCORECARD_WEBHOOK_URL"
metricExporter string = "SCORECARD_METRIC_EXPORTER"
metricStackdriverPrefix string = "SCORECARD_METRIC_STACKDRIVER_PREFIX"
bigqueryTable string = "SCORECARD_BIGQUERY_TABLE"
resultDataBucketURL string = "SCORECARD_DATA_BUCKET_URL"
apiResultsBucketURL string = "SCORECARD_API_RESULTS_BUCKET_URL"
inputBucketURL string = "SCORECARD_INPUT_BUCKET_URL"
inputBucketPrefix string = "SCORECARD_INPUT_BUCKET_PREFIX"
)

var (
Expand All @@ -71,19 +72,20 @@ var (

//nolint:govet
type config struct {
ProjectID string `yaml:"project-id"`
ResultDataBucketURL string `yaml:"result-data-bucket-url"`
RequestTopicURL string `yaml:"request-topic-url"`
RequestSubscriptionURL string `yaml:"request-subscription-url"`
BigQueryDataset string `yaml:"bigquery-dataset"`
BigQueryTable string `yaml:"bigquery-table"`
CompletionThreshold float32 `yaml:"completion-threshold"`
WebhookURL string `yaml:"webhook-url"`
MetricExporter string `yaml:"metric-exporter"`
ShardSize int `yaml:"shard-size"`
InputBucketURL string `yaml:"input-bucket-url"`
InputBucketPrefix string `yaml:"input-bucket-prefix"`
AdditionalParams map[string]map[string]string `yaml:"additional-params"`
ProjectID string `yaml:"project-id"`
ResultDataBucketURL string `yaml:"result-data-bucket-url"`
RequestTopicURL string `yaml:"request-topic-url"`
RequestSubscriptionURL string `yaml:"request-subscription-url"`
BigQueryDataset string `yaml:"bigquery-dataset"`
BigQueryTable string `yaml:"bigquery-table"`
CompletionThreshold float32 `yaml:"completion-threshold"`
WebhookURL string `yaml:"webhook-url"`
MetricExporter string `yaml:"metric-exporter"`
MetricStackdriverPrefix string `yaml:"metric-stackdriver-prefix"`
ShardSize int `yaml:"shard-size"`
InputBucketURL string `yaml:"input-bucket-url"`
InputBucketPrefix string `yaml:"input-bucket-prefix"`
AdditionalParams map[string]map[string]string `yaml:"additional-params"`
}

func getParsedConfigFromFile(byteValue []byte) (config, error) {
Expand Down Expand Up @@ -294,6 +296,12 @@ func GetMetricExporter() (string, error) {
return getStringConfigValue(metricExporter, configYAML, "MetricExporter", "metric-exporter")
}

// GetMetricStackdriverPrefix returns the prefix for stackdriver opencensus exporter.
func GetMetricStackdriverPrefix() (string, error) {
return getStringConfigValue(
metricStackdriverPrefix, configYAML, "MetricStackdriverPrefix", "metric-stackdriver-prefix")
}

// GetAPIResultsBucketURL returns the bucket URL for storing cron job results.
func GetAPIResultsBucketURL() (string, error) {
return getScorecardParam("api-results-bucket-url")
Expand Down
1 change: 1 addition & 0 deletions cron/config/config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ completion-threshold: 0.99
shard-size: 10
webhook-url:
metric-exporter: stackdriver
metric-stackdriver-prefix: scorecard-cron
result-data-bucket-url: gs://ossf-scorecard-data2

# TODO temporarily leaving old variables until changes propagate to production
Expand Down
69 changes: 42 additions & 27 deletions cron/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,20 @@ import (
)

const (
testEnvVar string = "TEST_ENV_VAR"
prodProjectID = "openssf"
prodBucket = "gs://ossf-scorecard-data2"
prodTopic = "gcppubsub://projects/openssf/topics/scorecard-batch-requests"
prodSubscription = "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker"
prodBigQueryDataset = "scorecardcron"
prodBigQueryTable = "scorecard-v2"
prodCompletionThreshold = 0.99
prodWebhookURL = ""
prodCIIDataBucket = "gs://ossf-scorecard-cii-data"
prodBlacklistedChecks = "CI-Tests,Contributors"
prodShardSize int = 10
prodMetricExporter string = "stackdriver"
testEnvVar string = "TEST_ENV_VAR"
prodProjectID = "openssf"
prodBucket = "gs://ossf-scorecard-data2"
prodTopic = "gcppubsub://projects/openssf/topics/scorecard-batch-requests"
prodSubscription = "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker"
prodBigQueryDataset = "scorecardcron"
prodBigQueryTable = "scorecard-v2"
prodCompletionThreshold = 0.99
prodWebhookURL = ""
prodCIIDataBucket = "gs://ossf-scorecard-cii-data"
prodBlacklistedChecks = "CI-Tests,Contributors"
prodShardSize int = 10
prodMetricExporter string = "stackdriver"
prodMetricStackdriverPrefix string = "scorecard-cron"
// Raw results.
prodRawBucket = "gs://ossf-scorecard-rawdata"
prodRawBigQueryTable = "scorecard-rawdata"
Expand Down Expand Up @@ -94,22 +95,22 @@ func TestYAMLParsing(t *testing.T) {
name: "validate",
filename: "config.yaml",
expectedConfig: config{
ProjectID: prodProjectID,
ResultDataBucketURL: prodBucket,
RequestTopicURL: prodTopic,
RequestSubscriptionURL: prodSubscription,
BigQueryDataset: prodBigQueryDataset,
BigQueryTable: prodBigQueryTable,
CompletionThreshold: prodCompletionThreshold,
WebhookURL: prodWebhookURL,
ShardSize: prodShardSize,
MetricExporter: prodMetricExporter,
InputBucketURL: prodInputBucketURL,
InputBucketPrefix: prodInputBucketPrefix,
AdditionalParams: prodAdditionalParams,
ProjectID: prodProjectID,
ResultDataBucketURL: prodBucket,
RequestTopicURL: prodTopic,
RequestSubscriptionURL: prodSubscription,
BigQueryDataset: prodBigQueryDataset,
BigQueryTable: prodBigQueryTable,
CompletionThreshold: prodCompletionThreshold,
WebhookURL: prodWebhookURL,
ShardSize: prodShardSize,
MetricExporter: prodMetricExporter,
MetricStackdriverPrefix: prodMetricStackdriverPrefix,
InputBucketURL: prodInputBucketURL,
InputBucketPrefix: prodInputBucketPrefix,
AdditionalParams: prodAdditionalParams,
},
},

{
name: "basic",
filename: "testdata/basic.yaml",
Expand Down Expand Up @@ -393,6 +394,20 @@ func TestGetMetricExporter(t *testing.T) {
})
}

//nolint:paralleltest // Since os.Setenv is used.
func TestGetMetricStackdriverPrefix(t *testing.T) {
t.Run("GetMetricStackdriverPrefix", func(t *testing.T) {
os.Unsetenv(metricStackdriverPrefix)
prefix, err := GetMetricStackdriverPrefix()
if err != nil {
t.Errorf("failed to get production metric stackdriver prefix from config: %v", err)
}
if prefix != prodMetricStackdriverPrefix {
t.Errorf("test failed: expected - %s, got = %s", prodMetricStackdriverPrefix, prefix)
}
})
}

//nolint:paralleltest // Since os.Setenv is used.
func TestGetAPIResultsBucketURL(t *testing.T) {
t.Run("GetBigQueryExportsBucketURL", func(t *testing.T) {
Expand Down
7 changes: 5 additions & 2 deletions cron/internal/monitoring/exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ var errorUndefinedExporter = errors.New("unsupported exporterType")
type exporterType string

const (
stackdriverMetricPrefix = "scorecard-cron"
stackdriverTimeSeriesQuota = 200
stackdriverTimeoutMinutes = 10
stackDriver exporterType = "stackdriver"
Expand Down Expand Up @@ -69,9 +68,13 @@ func newStackDriverExporter() (*stackdriver.Exporter, error) {
if err != nil {
return nil, fmt.Errorf("error getting ProjectID: %w", err)
}
prefix, err := config.GetMetricStackdriverPrefix()
if err != nil {
return nil, fmt.Errorf("error getting stackdriver prefix: %w", err)
}
exporter, err := stackdriver.NewExporter(stackdriver.Options{
ProjectID: projectID,
MetricPrefix: stackdriverMetricPrefix,
MetricPrefix: prefix,
MonitoredResource: gcp.Autodetect(),
Timeout: stackdriverTimeoutMinutes * time.Minute,
// Stackdriver specific quotas based on https://cloud.google.com/monitoring/quotas
Expand Down

0 comments on commit 49e9418

Please sign in to comment.