From 5593c75e958a36e5116cc86b093d10cc310316a5 Mon Sep 17 00:00:00 2001 From: "Pablo E. Colazurdo" Date: Wed, 8 Jun 2022 08:43:40 +0000 Subject: [PATCH 1/6] Working version for Lambda - Prometheus integration - pabcol@ --- .../sample-apps/function/lambda_function.py | 27 +++++++++++++++++++ python/sample-apps/function/requirements.txt | 5 +++- python/sample-apps/run.sh | 0 python/sample-apps/template.yml | 12 +++++++-- 4 files changed, 41 insertions(+), 3 deletions(-) mode change 100644 => 100755 python/sample-apps/run.sh diff --git a/python/sample-apps/function/lambda_function.py b/python/sample-apps/function/lambda_function.py index 82216783fe..c4d37542cc 100644 --- a/python/sample-apps/function/lambda_function.py +++ b/python/sample-apps/function/lambda_function.py @@ -3,8 +3,29 @@ import aiohttp import asyncio import boto3 +import os +import time + +from opentelemetry import _metrics +from opentelemetry.exporter.otlp.proto.grpc._metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry._metrics import ( + get_meter_provider, + set_meter_provider, +) +from opentelemetry.sdk._metrics import MeterProvider +from opentelemetry.sdk._metrics.export import PeriodicExportingMetricReader + +exporter = OTLPMetricExporter(insecure=True) +reader = PeriodicExportingMetricReader(exporter) +provider = MeterProvider(metric_readers=[reader]) +set_meter_provider(provider) +meter = get_meter_provider().get_meter("otel_stack_function", "0.1.2") +print(os.environ) + async def fetch(session, url): async with session.get(url) as response: return await response.text() @@ -21,8 +42,14 @@ def lambda_handler(event, context): loop = asyncio.get_event_loop() loop.run_until_complete(callAioHttp()) + + counter = meter.create_counter(name="first_counter", description="TODO", unit="1",) for bucket in s3.buckets.all(): + counter.add(1, attributes={"hello": bucket.name}) + print("CounterAdd") + print(bucket.name) + time.sleep(300) return {"statusCode": 200, "body": json.dumps(os.environ.get("_X_AMZN_TRACE_ID"))} diff --git a/python/sample-apps/function/requirements.txt b/python/sample-apps/function/requirements.txt index ce2357185a..4be643b8e2 100644 --- a/python/sample-apps/function/requirements.txt +++ b/python/sample-apps/function/requirements.txt @@ -1 +1,4 @@ -aiohttp \ No newline at end of file +aiohttp +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp-proto-grpc \ No newline at end of file diff --git a/python/sample-apps/run.sh b/python/sample-apps/run.sh old mode 100644 new mode 100755 diff --git a/python/sample-apps/template.yml b/python/sample-apps/template.yml index 86d738a7da..2a3fde1a87 100644 --- a/python/sample-apps/template.yml +++ b/python/sample-apps/template.yml @@ -12,19 +12,27 @@ Resources: Type: AWS::Serverless::Function Properties: Handler: lambda_function.lambda_handler - Runtime: python3.9 + Runtime: python3.7 CodeUri: ./function Description: Build OTel Python Lambda Sample from scratch MemorySize: 512 - Timeout: 15 + Timeout: 310 Policies: - AWSLambdaBasicExecutionRole - AWSLambda_ReadOnlyAccess - AWSXrayWriteOnlyAccess - AmazonS3FullAccess + - Statement: + - Sid: AMPRemoteWrite + Effect: Allow + Action: + - aps:RemoteWrite + Resource: '*' Environment: Variables: AWS_LAMBDA_EXEC_WRAPPER: /opt/python/otel-instrument + OTEL_METRICS_EXPORTER: otlp_proto_http + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python Tracing: Active Layers: - !Ref OTelLayer From 25d8121cbd2bcbdb3cb773ce4b41fda0ea7dfabf Mon Sep 17 00:00:00 2001 From: "Pablo E. Colazurdo" Date: Wed, 8 Jun 2022 09:19:32 +0000 Subject: [PATCH 2/6] Working version for Lambda - Prometheus integration - collector changes - pabcol@ --- collector/Makefile | 3 ++- collector/config.yaml | 7 ++++++- collector/go.mod | 3 +++ collector/go.sum | 6 ++++++ collector/lambdacomponents/default.go | 2 ++ collector/lambdacomponents/go.mod | 1 + 6 files changed, 20 insertions(+), 2 deletions(-) diff --git a/collector/Makefile b/collector/Makefile index 66f71921c3..992b7ade75 100644 --- a/collector/Makefile +++ b/collector/Makefile @@ -10,7 +10,8 @@ GOBUILD=GO111MODULE=on CGO_ENABLED=0 installsuffix=cgo go build -trimpath BUILD_INFO_IMPORT_PATH=main LDFLAGS=-ldflags "-s -w -X $(BUILD_INFO_IMPORT_PATH).GitHash=$(GIT_SHA) -X $(BUILD_INFO_IMPORT_PATH).Version=$(VERSION) \ --X github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter.collectorDistribution=opentelemetry-collector-lambda" +-X github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter.collectorDistribution=opentelemetry-collector-lambda \ +-X github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter.collectorDistribution=opentelemetry-collector-lambda" clean: rm -rf build/ diff --git a/collector/config.yaml b/collector/config.yaml index fdec318bb7..ff77629e8b 100644 --- a/collector/config.yaml +++ b/collector/config.yaml @@ -7,6 +7,11 @@ receivers: exporters: logging: loglevel: debug + awsprometheusremotewrite: + endpoint: "https://aps-workspaces.eu-west-1.amazonaws.com/workspaces/ws-f95ea95b-5b07-4072-ba9e-4dcee1538d73/api/v1/remote_write" + aws_auth: + service: "aps" + region: "eu-west-1" service: pipelines: @@ -15,4 +20,4 @@ service: exporters: [logging] metrics: receivers: [otlp] - exporters: [logging] + exporters: [logging, awsprometheusremotewrite] diff --git a/collector/go.mod b/collector/go.mod index b800db94fc..f63f6ff98f 100644 --- a/collector/go.mod +++ b/collector/go.mod @@ -13,6 +13,7 @@ require ( cloud.google.com/go/compute v1.6.0 // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.1 // indirect github.com/antonmedv/expr v1.9.0 // indirect + github.com/aws/aws-sdk-go v1.44.11 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -31,6 +32,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.3 // indirect github.com/knadh/koanf v1.4.1 // indirect @@ -43,6 +45,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mostynb/go-grpc-compression v1.1.16 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter v0.51.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.51.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.51.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.51.0 // indirect diff --git a/collector/go.sum b/collector/go.sum index 040f8bf2e9..3a4d5e07bf 100644 --- a/collector/go.sum +++ b/collector/go.sum @@ -198,6 +198,8 @@ github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm github.com/aws/aws-sdk-go v1.42.31/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.11 h1:eIC59RrNY7xXYmGy/kKkLj4PGB325Jca22lcxZwbpBE= +github.com/aws/aws-sdk-go v1.44.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= @@ -952,7 +954,9 @@ github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= @@ -1186,6 +1190,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter v0.51.0 h1:z/qJ3Lkp/fCjLjXAQLHxQoEepeoEblvEHGQD8Qj9owI= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter v0.51.0/go.mod h1:3KW/KsZb2QPxANoDhpfLIRVmxLWp4ITGVPBRvAgQboM= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.51.0 h1:dYTzPgICn/zrS4mwvO2H7pWtOxc/mWR0hyMva1KeGB8= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.51.0/go.mod h1:8EM88Rm2bwLSWadU8/jqEi/o3LwMcqXMLLwis4MCFMg= github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.51.0 h1:fOaDqCw89M+GDEsCWGWuWZB3qPZek81to/fLshHArNg= diff --git a/collector/lambdacomponents/default.go b/collector/lambdacomponents/default.go index a051e9ffd0..b515610537 100644 --- a/collector/lambdacomponents/default.go +++ b/collector/lambdacomponents/default.go @@ -23,6 +23,7 @@ import ( "go.opentelemetry.io/collector/receiver/otlpreceiver" "go.uber.org/multierr" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" @@ -46,6 +47,7 @@ func Components() (component.Factories, error) { otlpexporter.NewFactory(), otlphttpexporter.NewFactory(), prometheusremotewriteexporter.NewFactory(), + awsprometheusremotewriteexporter.NewFactory(), ) if err != nil { errs = append(errs, err) diff --git a/collector/lambdacomponents/go.mod b/collector/lambdacomponents/go.mod index fc0b00d366..3be22a21bf 100644 --- a/collector/lambdacomponents/go.mod +++ b/collector/lambdacomponents/go.mod @@ -4,6 +4,7 @@ go 1.17 require ( github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.51.0 + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter v0.51.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.51.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.51.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.51.0 From 85b97a73a0dcc2ef0e1c67b3c76f6c7045a87518 Mon Sep 17 00:00:00 2001 From: "Pablo E. Colazurdo" Date: Wed, 8 Jun 2022 09:20:28 +0000 Subject: [PATCH 3/6] Removed sleep and using force_flush mechanism --- python/sample-apps/function/lambda_function.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/sample-apps/function/lambda_function.py b/python/sample-apps/function/lambda_function.py index c4d37542cc..4bdc785a60 100644 --- a/python/sample-apps/function/lambda_function.py +++ b/python/sample-apps/function/lambda_function.py @@ -51,5 +51,8 @@ def lambda_handler(event, context): print(bucket.name) + print("forcing flush") + provider.force_flush() + print("forced flush") time.sleep(300) return {"statusCode": 200, "body": json.dumps(os.environ.get("_X_AMZN_TRACE_ID"))} From 094b2bed1579702265039b837c3e7bbea9b767e3 Mon Sep 17 00:00:00 2001 From: "Pablo E. Colazurdo" Date: Fri, 10 Jun 2022 16:20:27 +0000 Subject: [PATCH 4/6] Added delay after force_flush due to https://github.com/open-telemetry/opentelemetry-specification/issues/1287 --- .../sample-apps/function/lambda_function.py | 24 +++++++++++-------- python/sample-apps/template.yml | 3 +-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/python/sample-apps/function/lambda_function.py b/python/sample-apps/function/lambda_function.py index 4bdc785a60..bed8b2cb88 100644 --- a/python/sample-apps/function/lambda_function.py +++ b/python/sample-apps/function/lambda_function.py @@ -10,6 +10,10 @@ from opentelemetry.exporter.otlp.proto.grpc._metric_exporter import ( OTLPMetricExporter, ) +import opentelemetry.exporter.otlp.proto.grpc.version + +print( "opentelemetry.exporter.otlp.proto.grpc version", opentelemetry.exporter.otlp.proto.grpc.version.__version__) + from opentelemetry._metrics import ( get_meter_provider, set_meter_provider, @@ -22,9 +26,8 @@ provider = MeterProvider(metric_readers=[reader]) set_meter_provider(provider) - meter = get_meter_provider().get_meter("otel_stack_function", "0.1.2") -print(os.environ) + async def fetch(session, url): async with session.get(url) as response: @@ -45,14 +48,15 @@ def lambda_handler(event, context): counter = meter.create_counter(name="first_counter", description="TODO", unit="1",) + i = 0 for bucket in s3.buckets.all(): - counter.add(1, attributes={"hello": bucket.name}) - print("CounterAdd") + i = i + 1 + # print(bucket.name) + + counter.add(i, attributes={"invocation": context.aws_request_id}) - print(bucket.name) - - print("forcing flush") - provider.force_flush() - print("forced flush") - time.sleep(300) + provider.force_flush() # needed to be sure that metrics are sent to OTel before shutting down the worker + time.sleep(1) # This is needed because at the moment there is no API to force the collector to push the message before the Worker is suspended + + return {"statusCode": 200, "body": json.dumps(os.environ.get("_X_AMZN_TRACE_ID"))} diff --git a/python/sample-apps/template.yml b/python/sample-apps/template.yml index 2a3fde1a87..c4bd6f7f1d 100644 --- a/python/sample-apps/template.yml +++ b/python/sample-apps/template.yml @@ -31,8 +31,7 @@ Resources: Environment: Variables: AWS_LAMBDA_EXEC_WRAPPER: /opt/python/otel-instrument - OTEL_METRICS_EXPORTER: otlp_proto_http - PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python # needed because https://github.com/open-telemetry/opentelemetry-python/issues/2717 / when new release of library is released this should be solved and can be removed Tracing: Active Layers: - !Ref OTelLayer From add5813076b17be73bf91bbdbf1134c132c25188 Mon Sep 17 00:00:00 2001 From: "Pablo E. Colazurdo" Date: Wed, 8 Jun 2022 08:43:40 +0000 Subject: [PATCH 5/6] Working version for Lambda - AWS Managed Prometheus integration --- collector/Makefile | 3 +- collector/config.yaml | 7 +++- collector/go.mod | 3 ++ collector/go.sum | 6 +++ collector/lambdacomponents/default.go | 2 + collector/lambdacomponents/go.mod | 1 + .../sample-apps/function/lambda_function.py | 38 ++++++++++++++++++- python/sample-apps/function/requirements.txt | 5 ++- python/sample-apps/run.sh | 0 python/sample-apps/template.yml | 11 +++++- 10 files changed, 69 insertions(+), 7 deletions(-) mode change 100644 => 100755 python/sample-apps/run.sh diff --git a/collector/Makefile b/collector/Makefile index 66f71921c3..992b7ade75 100644 --- a/collector/Makefile +++ b/collector/Makefile @@ -10,7 +10,8 @@ GOBUILD=GO111MODULE=on CGO_ENABLED=0 installsuffix=cgo go build -trimpath BUILD_INFO_IMPORT_PATH=main LDFLAGS=-ldflags "-s -w -X $(BUILD_INFO_IMPORT_PATH).GitHash=$(GIT_SHA) -X $(BUILD_INFO_IMPORT_PATH).Version=$(VERSION) \ --X github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter.collectorDistribution=opentelemetry-collector-lambda" +-X github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter.collectorDistribution=opentelemetry-collector-lambda \ +-X github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter.collectorDistribution=opentelemetry-collector-lambda" clean: rm -rf build/ diff --git a/collector/config.yaml b/collector/config.yaml index fdec318bb7..e8132914a6 100644 --- a/collector/config.yaml +++ b/collector/config.yaml @@ -7,6 +7,11 @@ receivers: exporters: logging: loglevel: debug + awsprometheusremotewrite: + endpoint: "https://aps-workspaces.eu-west-1.amazonaws.com/workspaces/ws-12345678-1234-1234-1234-123456789012/api/v1/remote_write" + aws_auth: + service: "aps" + region: "eu-west-1" service: pipelines: @@ -15,4 +20,4 @@ service: exporters: [logging] metrics: receivers: [otlp] - exporters: [logging] + exporters: [logging, awsprometheusremotewrite] diff --git a/collector/go.mod b/collector/go.mod index b800db94fc..f63f6ff98f 100644 --- a/collector/go.mod +++ b/collector/go.mod @@ -13,6 +13,7 @@ require ( cloud.google.com/go/compute v1.6.0 // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.1 // indirect github.com/antonmedv/expr v1.9.0 // indirect + github.com/aws/aws-sdk-go v1.44.11 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -31,6 +32,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.3 // indirect github.com/knadh/koanf v1.4.1 // indirect @@ -43,6 +45,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mostynb/go-grpc-compression v1.1.16 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter v0.51.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.51.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.51.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.51.0 // indirect diff --git a/collector/go.sum b/collector/go.sum index 040f8bf2e9..3a4d5e07bf 100644 --- a/collector/go.sum +++ b/collector/go.sum @@ -198,6 +198,8 @@ github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm github.com/aws/aws-sdk-go v1.42.31/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.11 h1:eIC59RrNY7xXYmGy/kKkLj4PGB325Jca22lcxZwbpBE= +github.com/aws/aws-sdk-go v1.44.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= @@ -952,7 +954,9 @@ github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= @@ -1186,6 +1190,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter v0.51.0 h1:z/qJ3Lkp/fCjLjXAQLHxQoEepeoEblvEHGQD8Qj9owI= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter v0.51.0/go.mod h1:3KW/KsZb2QPxANoDhpfLIRVmxLWp4ITGVPBRvAgQboM= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.51.0 h1:dYTzPgICn/zrS4mwvO2H7pWtOxc/mWR0hyMva1KeGB8= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.51.0/go.mod h1:8EM88Rm2bwLSWadU8/jqEi/o3LwMcqXMLLwis4MCFMg= github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.51.0 h1:fOaDqCw89M+GDEsCWGWuWZB3qPZek81to/fLshHArNg= diff --git a/collector/lambdacomponents/default.go b/collector/lambdacomponents/default.go index a051e9ffd0..b515610537 100644 --- a/collector/lambdacomponents/default.go +++ b/collector/lambdacomponents/default.go @@ -23,6 +23,7 @@ import ( "go.opentelemetry.io/collector/receiver/otlpreceiver" "go.uber.org/multierr" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" @@ -46,6 +47,7 @@ func Components() (component.Factories, error) { otlpexporter.NewFactory(), otlphttpexporter.NewFactory(), prometheusremotewriteexporter.NewFactory(), + awsprometheusremotewriteexporter.NewFactory(), ) if err != nil { errs = append(errs, err) diff --git a/collector/lambdacomponents/go.mod b/collector/lambdacomponents/go.mod index fc0b00d366..3be22a21bf 100644 --- a/collector/lambdacomponents/go.mod +++ b/collector/lambdacomponents/go.mod @@ -4,6 +4,7 @@ go 1.17 require ( github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.51.0 + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter v0.51.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.51.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.51.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.51.0 diff --git a/python/sample-apps/function/lambda_function.py b/python/sample-apps/function/lambda_function.py index 82216783fe..bed8b2cb88 100644 --- a/python/sample-apps/function/lambda_function.py +++ b/python/sample-apps/function/lambda_function.py @@ -3,6 +3,30 @@ import aiohttp import asyncio import boto3 +import os +import time + +from opentelemetry import _metrics +from opentelemetry.exporter.otlp.proto.grpc._metric_exporter import ( + OTLPMetricExporter, +) +import opentelemetry.exporter.otlp.proto.grpc.version + +print( "opentelemetry.exporter.otlp.proto.grpc version", opentelemetry.exporter.otlp.proto.grpc.version.__version__) + +from opentelemetry._metrics import ( + get_meter_provider, + set_meter_provider, +) +from opentelemetry.sdk._metrics import MeterProvider +from opentelemetry.sdk._metrics.export import PeriodicExportingMetricReader + +exporter = OTLPMetricExporter(insecure=True) +reader = PeriodicExportingMetricReader(exporter) +provider = MeterProvider(metric_readers=[reader]) +set_meter_provider(provider) + +meter = get_meter_provider().get_meter("otel_stack_function", "0.1.2") async def fetch(session, url): @@ -21,8 +45,18 @@ def lambda_handler(event, context): loop = asyncio.get_event_loop() loop.run_until_complete(callAioHttp()) + + counter = meter.create_counter(name="first_counter", description="TODO", unit="1",) + i = 0 for bucket in s3.buckets.all(): - print(bucket.name) - + i = i + 1 + # print(bucket.name) + + counter.add(i, attributes={"invocation": context.aws_request_id}) + + provider.force_flush() # needed to be sure that metrics are sent to OTel before shutting down the worker + time.sleep(1) # This is needed because at the moment there is no API to force the collector to push the message before the Worker is suspended + + return {"statusCode": 200, "body": json.dumps(os.environ.get("_X_AMZN_TRACE_ID"))} diff --git a/python/sample-apps/function/requirements.txt b/python/sample-apps/function/requirements.txt index ce2357185a..4be643b8e2 100644 --- a/python/sample-apps/function/requirements.txt +++ b/python/sample-apps/function/requirements.txt @@ -1 +1,4 @@ -aiohttp \ No newline at end of file +aiohttp +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp-proto-grpc \ No newline at end of file diff --git a/python/sample-apps/run.sh b/python/sample-apps/run.sh old mode 100644 new mode 100755 diff --git a/python/sample-apps/template.yml b/python/sample-apps/template.yml index 86d738a7da..c4bd6f7f1d 100644 --- a/python/sample-apps/template.yml +++ b/python/sample-apps/template.yml @@ -12,19 +12,26 @@ Resources: Type: AWS::Serverless::Function Properties: Handler: lambda_function.lambda_handler - Runtime: python3.9 + Runtime: python3.7 CodeUri: ./function Description: Build OTel Python Lambda Sample from scratch MemorySize: 512 - Timeout: 15 + Timeout: 310 Policies: - AWSLambdaBasicExecutionRole - AWSLambda_ReadOnlyAccess - AWSXrayWriteOnlyAccess - AmazonS3FullAccess + - Statement: + - Sid: AMPRemoteWrite + Effect: Allow + Action: + - aps:RemoteWrite + Resource: '*' Environment: Variables: AWS_LAMBDA_EXEC_WRAPPER: /opt/python/otel-instrument + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python # needed because https://github.com/open-telemetry/opentelemetry-python/issues/2717 / when new release of library is released this should be solved and can be removed Tracing: Active Layers: - !Ref OTelLayer From d76c827ed62e21aa00c6f1ff682ee76a83d471f5 Mon Sep 17 00:00:00 2001 From: "Pablo E. Colazurdo" Date: Sat, 25 Jun 2022 19:25:35 +0100 Subject: [PATCH 6/6] reverted back to original timeout and python version --- python/sample-apps/template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/sample-apps/template.yml b/python/sample-apps/template.yml index c4bd6f7f1d..eb00d98310 100644 --- a/python/sample-apps/template.yml +++ b/python/sample-apps/template.yml @@ -12,11 +12,11 @@ Resources: Type: AWS::Serverless::Function Properties: Handler: lambda_function.lambda_handler - Runtime: python3.7 + Runtime: python3.9 CodeUri: ./function Description: Build OTel Python Lambda Sample from scratch MemorySize: 512 - Timeout: 310 + Timeout: 15 Policies: - AWSLambdaBasicExecutionRole - AWSLambda_ReadOnlyAccess