Skip to content

Commit

Permalink
Merge pull request #597 from mendix/develop
Browse files Browse the repository at this point in the history
Release 2023-01-05
  • Loading branch information
sailhenz authored Jan 5, 2023
2 parents 67c7c63 + 7512fd8 commit 440e6e2
Show file tree
Hide file tree
Showing 9 changed files with 98 additions and 122 deletions.
19 changes: 10 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -720,19 +720,20 @@ The most important metrics ( `before_xid_wraparound` , `connections` , `database

### Dynatrace

[Dynatrace SaaS/Managed](http://www.dynatrace.com/cloud-foundry/) is your full stack monitoring solution - powered by artificial intelligence. Dynatrace SaaS/Managed allows you insights into all application requests from the users click in the browser down to the database statement and code-level.

Dynatrace integration is supported in Mendix version 9.7 and above.
To enable Dynatrace, configure the following environment variables:

| Environment Variable | Description |
| -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `DT_PAAS_TOKEN` | The token for integrating your Dynatrace environment with Cloud Foundry. You can find it in the deploy Dynatrace section within your environment. |
| `DT_SAAS_URL` | Monitoring endpoint url of the Dynatrace service |
| `DT_TENANT` | Your Dynatrace environment ID is the unique identifier of your Dynatrace environment. You can find it in the deploy Dynatrace section within your environment. |
| Environment Variable | Description |
| -------------------- | ----------------------------------------------------------------------------------------------|
| `DT_PAAS_TOKEN` | The token for integrating your Dynatrace environment with your Mendix app |
| `DT_SAAS_URL` | Monitoring endpoint url of the Dynatrace service |

Buildpack attaches these default tags to metrics that are pushed to Dynatrace:

By setting these environment variables automatically the Dynatrace OneAgent will be loaded in the container.
* `app` - Environment Id of the Mendix application
* `instance_index` - Instance index that metrics belong to

OneAgent will be able to measure all J2EE related Metrics from the Application. See OneAgent documention for more details.
Extra tags can be attached via `TAGS` environment variable (example value: `[env:accp]`)

### Logging
The buildpack provides several options to configure logging.
Expand Down
2 changes: 0 additions & 2 deletions buildpack/stage.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from buildpack.telemetry import (
appdynamics,
datadog,
dynatrace,
fluentbit,
splunk,
logs,
Expand Down Expand Up @@ -205,7 +204,6 @@ def cleanup_dependency_cache(cached_dir, dependency_list):
java_version,
)
appdynamics.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
dynatrace.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
splunk.stage()
fluentbit.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
newrelic.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
Expand Down
2 changes: 0 additions & 2 deletions buildpack/start.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from buildpack.telemetry import (
appdynamics,
datadog,
dynatrace,
fluentbit,
splunk,
logs,
Expand Down Expand Up @@ -149,7 +148,6 @@ def _register_signal_handlers():
java.update_config(m2ee, application_name, util.get_vcap_data(), runtime_version)
newrelic.update_config(m2ee, application_name)
appdynamics.update_config(m2ee)
dynatrace.update_config(m2ee, application_name)
splunk.update_config(m2ee)
fluentbit.update_config(m2ee)
mx_java_agent.update_config(m2ee)
Expand Down
117 changes: 16 additions & 101 deletions buildpack/telemetry/dynatrace.py
Original file line number Diff line number Diff line change
@@ -1,110 +1,25 @@
"""
For Dynatrace, metrics are directly ingested through telegraf.
No additional setup is needed.
This module only collects information for telegraf from environment variables.
"""
import logging
import os
import shutil
import json
from urllib.parse import urljoin

from buildpack import util
INGEST_ENDPOINT = "/api/v2/metrics/ingest"

default_env = {
"DT_TENANTTOKEN": None, # optional, default to one found in manifest.json
"DT_APPLICATIONID": None, # optional, default not set
"DT_TENANT": None, # required, environment ID (uuid format)
# "DT_PAAS_TOKEN": None, # required
# "DT_SAAS_URL": None, # required
"DT_LOGSTREAM": "stdout",
"DT_NETWORK_ZONE": None, # optional, not sure what this means :D
"DT_CUSTOM_PROP": None, # optional metadata e.g. Department=Acceptance Stage=Sprint
"DT_TAGS": None, # optional tags e.g. MikesStuff easyTravel=Mike
}


def stage(buildpack_dir, build_path, cache_path):
if is_enabled():
try:
util.resolve_dependency(
"dynatrace.agent",
build_path, # DOT_LOCAL_LOCATION,
buildpack_dir=buildpack_dir,
cache_dir=cache_path, # CACHE_DIR,
unpack=True,
overrides={
"url": os.environ.get("DT_SAAS_URL"),
"environment": os.environ.get("DT_TENANT"),
"token": os.environ.get("DT_PAAS_TOKEN"),
},
ignore_cache=True
)
except Exception as e:
logging.warning(
"Dynatrace agent download and unpack failed", exc_info=True
)
def is_enabled():
return "DT_PAAS_TOKEN" in os.environ.keys() and \
"DT_SAAS_URL" in os.environ.keys()


def update_config(m2ee, app_name):
def get_ingestion_info():
if not is_enabled():
logging.debug(
"Skipping Dynatrace setup, no DT_TENANTTOKEN found in environment"
)
return
logging.info("Enabling Dynatrace")
try:
manifest = get_manifest()
except Exception as e:
logging.warning(
"Failed to parse Dynatrace manifest file", exc_info=True
)
return

# dynamic default
default_env.update({"DT_TENANTTOKEN": manifest.get("tenantToken")})

for key, dv in default_env.items():
value = os.environ.get(key, dv)
if value:
util.upsert_custom_environment_variable(m2ee, key, value)
util.upsert_custom_environment_variable(
m2ee, "DT_CONNECTION_POINT", get_connection_endpoint()
)

agent_path = os.path.join(".local", get_agent_path())
if not os.path.exists(agent_path):
raise Exception(
"Dynatrace Agent not found: {agent_path}".format(agent_path)
)

util.upsert_javaopts(
m2ee,
[
"-agentpath:{path}".format(path=os.path.abspath(agent_path)),
"-Xshare:off",
],
)

return None, None

def get_manifest():
with open(".local/manifest.json", "r") as f:
return json.load(f)


def get_connection_endpoint():
manifest = get_manifest()
endpoints = manifest.get("communicationEndpoints", [])
# prepend the DT_SAAS_URL because the communication endpoints might not be correct
endpoints.insert(
0, "{url}/communication".format(url=os.environ.get("DT_SAAS_URL"))
)
return ";".join(endpoints)


def get_agent_path():
manifest = get_manifest()
technologies = manifest.get("technologies")
java_binaries = technologies.get("java").get("linux-x86-64")
for f in java_binaries:
binary_type = f.get("binarytype")
if binary_type == "loader":
return f.get("path")


def is_enabled():
return "DT_PAAS_TOKEN" in os.environ.keys()
logging.info("Metrics ingestion to Dynatrace is configured")
token = os.getenv("DT_PAAS_TOKEN")
ingest_url = urljoin(os.getenv("DT_SAAS_URL"), INGEST_ENDPOINT)
return token, ingest_url
3 changes: 2 additions & 1 deletion buildpack/telemetry/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from lib.m2ee import munin
from lib.m2ee.version import MXVersion

from . import datadog, appdynamics
from . import datadog, appdynamics, dynatrace

# Runtime configuration for influx registry
# This enables the new stream of metrics coming from micrometer instead
Expand Down Expand Up @@ -200,6 +200,7 @@ def configure_metrics_registry(m2ee):
datadog.is_enabled()
or get_appmetrics_target()
or appdynamics.machine_agent_enabled()
or dynatrace.is_enabled()
):
paidapps_registries.append(STATSD_REGISTRY)

Expand Down
22 changes: 21 additions & 1 deletion buildpack/telemetry/telegraf.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from buildpack.infrastructure import database
from jinja2 import Template

from . import datadog, metrics, mx_java_agent, appdynamics
from . import datadog, metrics, mx_java_agent, appdynamics, dynatrace

NAMESPACE = "telegraf"
DEPENDENCY = "%s.agent" % NAMESPACE
Expand Down Expand Up @@ -91,6 +91,7 @@ def include_db_metrics():
is_appmetrics
or datadog.is_enabled()
or appdynamics.machine_agent_enabled()
or dynatrace.is_enabled()
):
# For customers who have Datadog or AppDynamics or APPMETRICS_TARGET enabled,
# we always include the database metrics. They can opt out
Expand All @@ -109,6 +110,7 @@ def is_enabled(runtime_version):
metrics.get_appmetrics_target() is not None
or datadog.is_enabled()
or appdynamics.machine_agent_enabled()
or dynatrace.is_enabled()
or metrics.micrometer_metrics_enabled(runtime_version)
)

Expand Down Expand Up @@ -182,6 +184,20 @@ def _get_db_config():
return None


def _get_dynatrace_config(app_name):
token, ingest_url = dynatrace.get_ingestion_info()
tags = util.get_tags()
return {
"token": token,
"ingest_url": ingest_url,
"dimensions": {
"app": app_name,
"instance_index": _get_app_index(),
**tags
}
}


def _fix_metrics_registries_config(m2ee):
# Metrics.Registries is a nested JSON entry. As a result, when we read
# from the environment, it is not a list of registries but a string.
Expand Down Expand Up @@ -231,6 +247,8 @@ def update_config(m2ee, app_name):
# app and / or service tag not set
tags["service"] = datadog.get_service_tag()

dynatrace_token, dynatrace_ingest_url = dynatrace.get_ingestion_info()

with open(template_path, "r") as file_:
template = Template(file_.read(), trim_blocks=True, lstrip_blocks=True)
rendered = template.render(
Expand All @@ -252,6 +270,8 @@ def update_config(m2ee, app_name):
# For Telegraf config only AppDynamics Machine Agent makes sense.
appdynamics_enabled=appdynamics.machine_agent_enabled(),
appdynamics_output_script_path=APPDYNAMICS_OUTPUT_SCRIPT_PATH,
dynatrace_enabled=dynatrace.is_enabled(),
dynatrace_config=_get_dynatrace_config(app_name),
)

logging.debug("Writing Telegraf configuration file...")
Expand Down
6 changes: 1 addition & 5 deletions dependencies.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,6 @@ dependencies:
trace-agent:
artifact: datadog/dd-java-agent-{{ version }}.jar
version: 0.101.0
dynatrace:
agent:
artifact: "{{ url }}/e/{{ environment }}/api/v1/deployment/installer/agent/unix/paas/latest?include=java&bitness=64&Api-Token={{ token }}"
managed: false
fluentbit:
artifact: fluentbit/fluent-bit-{{ version }}.tar.gz
version: 1.9.2
Expand Down Expand Up @@ -93,4 +89,4 @@ dependencies:
telegraf:
agent:
artifact: telegraf/telegraf-{{ version }}_linux_amd64.tar.gz
version: 1.16.3
version: 1.24.4
25 changes: 24 additions & 1 deletion etc/telegraf/telegraf.toml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
{% endif %}

{% if db_config %}
{% if not (datadog_api_key or appdynamics_enabled) %}
{% if not (datadog_api_key or appdynamics_enabled or dynatrace_enabled) %}
# PostgreSQL input (standard)
[[inputs.postgresql]]
address = "postgres://{{ db_config['DatabaseUserName'] }}:{{ db_config['DatabasePassword'] }}@{{ db_config['DatabaseHost'] }}/{{ db_config['DatabaseName'] }}"
Expand Down Expand Up @@ -297,6 +297,29 @@
micrometer_metrics = ["true"]
{% endif %}

{% if dynatrace_enabled %}
[[outputs.dynatrace]]
# Your Dynatrace environment URL.
# For Dynatrace SaaS environments the URL scheme is "https://{your-environment-id}.live.dynatrace.com/api/v2/metrics/ingest"
# For Dynatrace Managed environments the URL scheme is "https://{your-domain}/e/{your-environment-id}/api/v2/metrics/ingest"
url = "{{ dynatrace_config['ingest_url'] }}"
#
# Your Dynatrace API token.
# Create an API token within your Dynatrace environment, by navigating to Access Tokens > Generate new token
# The API token needs "Ingest metrics" scope permission.
api_token = "{{ dynatrace_config['token'] }}"
#
# Ignore any micrometer_metrics
[outputs.dynatrace.tagdrop]
micrometer_metrics = ["true"]
#
# Optional dimensions to be added to every metric
[outputs.dynatrace.default_dimensions]
{% for key, value in dynatrace_config['dimensions'].items() %}
{{ key }} = "{{ value }}"
{% endfor %}
{% endif %}

{% if micrometer_metrics %}
####################################################################################
# App metrics via micrometer #
Expand Down
24 changes: 24 additions & 0 deletions tests/unit/test_dynatrace.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import os
from unittest.mock import patch
from urllib.parse import urljoin

from buildpack.telemetry import dynatrace
from unittest import TestCase


class TestDynatrace(TestCase):
def test_is_enabled_false(self):
# Should be False without necessary environment variables
self.assertFalse(dynatrace.is_enabled())

def test_get_ingestion_info(self):
# for get_ingestion_info(), also covers enabled case of is_enabled()
token = "DUMMY_TOKEN"
url = "DUMMY_URL"
expected_ingest_url = urljoin(url, dynatrace.INGEST_ENDPOINT)
env_vars = {"DT_PAAS_TOKEN": token, "DT_SAAS_URL": url}
with patch.dict(os.environ, env_vars):
actual_info = dynatrace.get_ingestion_info()

expected_info = (token, expected_ingest_url)
self.assertEqual(expected_info, actual_info)

0 comments on commit 440e6e2

Please sign in to comment.