Attribute requirement levels (#92)

semantic-conventions/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 Please update the changelog as part of any significant pull request.
 
+## Unreleased
+
+- BREAKING: Introduced attribute requirement levels ([#92](https://github.com/open-telemetry/build-tools/pull/92)):
+  - Schema: Attribute property `required` is removed and replaced by `requirement_level`, supported values are changed to `required` (previously `always`), `conditionally_required` (previously `conditional`), `recommended`, and `optional`.
+  - Templates: `opentelemetry.semconv.model.semantic_attribute.Required` enum is replaced by `RequirementLevel` with supported values listed above, `required_msg` is renamed to `requirement_level_msg`
+
 ## v0.8.0
 
 - Add `name` field for events. It defaults to the `prefix`

semantic-conventions/dev-requirements.txt

@@ -1,4 +1,4 @@
-black==21.8b0
+black==22.3.0
 mypy==0.910
 pytest==6.2.5
 flake8==3.9.2

semantic-conventions/semconv.schema.json

@@ -291,26 +291,45 @@
 			"allOf": [
 				{
 					"properties": {
-						"required": {
-							"description": "specifies if the attribute is mandatory. Can be 'always', or 'conditional'. When omitted, the attribute is not required. When set to 'conditional',the string provided as <condition> MUST specify the conditions under which the attribute is required.",
+						"requirement_level": {
+							"description": "specifies the attribute requirement level. Can be 'required', 'conditionally_required', 'recommended', or 'optional'. When omitted, the attribute is 'recommended'. When set to 'conditionally_required', the string provided as <condition> MUST specify the conditions under which the attribute is required.",
 							"oneOf": [
 								{
-									"type": "string",
-									"enum": [
-										"always"
-									]
+									"const": "required"
 								},
 								{
 									"type": "object",
 									"additionalProperties": false,
 									"required": [
-										"conditional"
+										"conditionally_required"
 									],
 									"properties": {
-										"conditional": {
+										"condition": {
 											"type": "string"
 										}
 									}
+								},
+								{
+									"oneOf": [
+										{
+											"const": "recommended"
+										},
+										{
+											"type": "object",
+											"additionalProperties": false,
+											"required": [
+												"recommended"
+											],
+											"properties": {
+												"recommended": {
+													"type": "string"
+												}
+											}
+										}
+									]
+								},
+								{
+									"const": "optional"
 								}
 							]
 						},

semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py

@@ -28,10 +28,11 @@
 )
 
 
-class Required(Enum):
-    ALWAYS = 1
-    CONDITIONAL = 2
-    NO = 3
+class RequirementLevel(Enum):
+    REQUIRED = 1
+    CONDITIONALLY_REQUIRED = 2
+    RECOMMENDED = 3
+    OPTIONAL = 4
 
 
 class StabilityLevel(Enum):
@@ -59,8 +60,8 @@ class SemanticAttribute:
     tag: str
     stability: StabilityLevel
     deprecated: str
-    required: Required
-    required_msg: str
+    requirement_level: RequirementLevel
+    requirement_level_msg: str
     sampling_relevant: bool
     note: str
     position: List[int]
@@ -98,7 +99,7 @@ def parse(
             "tag",
             "deprecated",
             "stability",
-            "required",
+            "requirement_level",
             "sampling_relevant",
             "note",
         )
@@ -133,32 +134,50 @@ def parse(
                 fqn = ref
 
             required_value_map = {
-                "always": Required.ALWAYS,
-                "conditional": Required.CONDITIONAL,
-                "": Required.NO,
+                "required": RequirementLevel.REQUIRED,
+                "conditionally_required": RequirementLevel.CONDITIONALLY_REQUIRED,
+                "": RequirementLevel.RECOMMENDED,
+                "recommended": RequirementLevel.RECOMMENDED,
+                "optional": RequirementLevel.OPTIONAL,
             }
-            required_msg = ""
-            required_val = attribute.get("required", "")
-            required: Optional[Required]
-            if isinstance(required_val, CommentedMap):
-                required = Required.CONDITIONAL
-                required_msg = required_val.get("conditional", None)
-                if required_msg is None:
-                    position = position_data["required"]
-                    msg = "Missing message for conditional required field!"
+            requirement_level_msg = ""
+            requirement_level_val = attribute.get("requirement_level", "")
+            requirement_level: Optional[RequirementLevel]
+            if isinstance(requirement_level_val, CommentedMap):
+
+                if len(requirement_level_val) != 1:
+                    position = position_data["requirement_level"]
+                    msg = "Multiple requirement_level values are not allowed!"
                     raise ValidationError.from_yaml_pos(position, msg)
+
+                recommended_msg = requirement_level_val.get("recommended", None)
+                condition_msg = requirement_level_val.get(
+                    "conditionally_required", None
+                )
+                if condition_msg is not None:
+                    requirement_level = RequirementLevel.CONDITIONALLY_REQUIRED
+                    requirement_level_msg = condition_msg
+                elif recommended_msg is not None:
+                    requirement_level = RequirementLevel.RECOMMENDED
+                    requirement_level_msg = recommended_msg
             else:
-                required = required_value_map.get(required_val)
-                if required == Required.CONDITIONAL:
-                    position = position_data["required"]
-                    msg = "Missing message for conditional required field!"
-                    raise ValidationError.from_yaml_pos(position, msg)
-            if required is None:
-                position = position_data["required"]
+                requirement_level = required_value_map.get(requirement_level_val)
+
+            if requirement_level is None:
+                position = position_data["requirement_level"]
                 msg = "Value '{}' for required field is not allowed".format(
-                    required_val
+                    requirement_level_val
                 )
                 raise ValidationError.from_yaml_pos(position, msg)
+
+            if (
+                requirement_level == RequirementLevel.CONDITIONALLY_REQUIRED
+                and not requirement_level_msg
+            ):
+                position = position_data["requirement_level"]
+                msg = "Missing message for conditionally required field!"
+                raise ValidationError.from_yaml_pos(position, msg)
+
             tag = attribute.get("tag", "").strip()
             stability, deprecated = SemanticAttribute.parse_stability_deprecated(
                 attribute.get("stability"), attribute.get("deprecated"), position_data
@@ -196,8 +215,8 @@ def parse(
                 tag=tag,
                 deprecated=deprecated,
                 stability=stability,
-                required=required,
-                required_msg=str(required_msg).strip(),
+                requirement_level=requirement_level,
+                requirement_level_msg=str(requirement_level_msg).strip(),
                 sampling_relevant=sampling_relevant,
                 note=parsed_note,
                 position=position,

semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py

@@ -23,7 +23,7 @@
 from opentelemetry.semconv.model.constraints import AnyOf, Include, parse_constraints
 from opentelemetry.semconv.model.exceptions import ValidationError
 from opentelemetry.semconv.model.semantic_attribute import (
-    Required,
+    RequirementLevel,
     SemanticAttribute,
     unique_attributes,
 )
@@ -150,17 +150,21 @@ def all_attributes(self):
 
     def sampling_attributes(self):
         return unique_attributes(
-            [attr for attr in self.attributes if attr.sampling_relevant]
+            attr for attr in self.attributes if attr.sampling_relevant
         )
 
     def required_attributes(self):
         return unique_attributes(
-            [attr for attr in self.attributes if attr.required == Required.ALWAYS]
+            attr
+            for attr in self.attributes
+            if attr.requirement_level == RequirementLevel.REQUIRED
         )
 
     def conditional_attributes(self):
         return unique_attributes(
-            [attr for attr in self.attributes if attr.required == Required.CONDITIONAL]
+            attr
+            for attr in self.attributes
+            if attr.requirement_level == RequirementLevel.CONDITIONALLY_REQUIRED
         )
 
     def any_of(self):

semantic-conventions/src/opentelemetry/semconv/templating/code.py

@@ -20,7 +20,10 @@
 import mistune
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 
-from opentelemetry.semconv.model.semantic_attribute import Required, TextWithLinks
+from opentelemetry.semconv.model.semantic_attribute import (
+    RequirementLevel,
+    TextWithLinks,
+)
 from opentelemetry.semconv.model.semantic_convention import SemanticConventionSet
 from opentelemetry.semconv.model.utils import ID_RE
 
@@ -233,12 +236,12 @@ def render(
                 template = env.get_template(file_name, globals=data)
                 template.globals["now"] = datetime.datetime.utcnow()
                 template.globals["version"] = os.environ.get("ARTIFACT_VERSION", "dev")
-                template.globals["Required"] = Required
+                template.globals["RequirementLevel"] = RequirementLevel
                 template.stream(data).dump(output_name)
         else:
             data = self.get_data_single_file(semconvset, template_path)
             template = env.get_template(file_name, globals=data)
             template.globals["now"] = datetime.datetime.utcnow()
             template.globals["version"] = os.environ.get("ARTIFACT_VERSION", "dev")
-            template.globals["Required"] = Required
+            template.globals["RequirementLevel"] = RequirementLevel
             template.stream(data).dump(output_file)

semantic-conventions/src/opentelemetry/semconv/templating/markdown/__init__.py

@@ -24,7 +24,7 @@
 from opentelemetry.semconv.model.semantic_attribute import (
     EnumAttributeType,
     EnumMember,
-    Required,
+    RequirementLevel,
     SemanticAttribute,
     StabilityLevel,
 )
@@ -69,7 +69,7 @@ class MarkdownRenderer:
     valid_parameters = ["tag", "full", "remove_constraints"]
 
     prelude = "<!-- semconv {} -->\n"
-    table_headers = "| Attribute  | Type | Description  | Examples  | Required |\n|---|---|---|---|---|\n"
+    table_headers = "| Attribute  | Type | Description  | Examples  | Requirement Level |\n|---|---|---|---|---|\n"
 
     def __init__(
         self, md_folder, semconvset: SemanticConventionSet, options=MarkdownOptions()
@@ -150,24 +150,36 @@ def to_markdown_attr(
                 )
             else:
                 examples = "; ".join("`{}`".format(ex) for ex in example_list)
-        if attribute.required == Required.ALWAYS:
-            required = "Yes"
-        elif attribute.required == Required.CONDITIONAL:
-            if len(attribute.required_msg) < self.options.break_count:
-                required = attribute.required_msg
+        if attribute.requirement_level == RequirementLevel.REQUIRED:
+            required = "Required"
+        elif attribute.requirement_level == RequirementLevel.CONDITIONALLY_REQUIRED:
+            if len(attribute.requirement_level_msg) < self.options.break_count:
+                required = "Conditionally Required: " + attribute.requirement_level_msg
             else:
                 # We put the condition in the notes after the table
-                self.render_ctx.add_note(attribute.required_msg)
-                required = "Conditional [{}]".format(len(self.render_ctx.notes))
-        else:
-            # check if they are required by some constraint
+                self.render_ctx.add_note(attribute.requirement_level_msg)
+                required = "Conditionally Required: [{}]".format(
+                    len(self.render_ctx.notes)
+                )
+        elif attribute.requirement_level == RequirementLevel.OPTIONAL:
+            required = "Optional"
+        else:  # attribute.requirement_level == Required.RECOMMENDED
+            # check if there are any notes
             if (
                 not self.render_ctx.is_remove_constraint
                 and self.render_ctx.current_semconv.has_attribute_constraint(attribute)
             ):
                 required = "See below"
             else:
-                required = "No"
+                if not attribute.requirement_level_msg:
+                    required = "Recommended"
+                elif len(attribute.requirement_level_msg) < self.options.break_count:
+                    required = "Recommended: " + attribute.requirement_level_msg
+                else:
+                    # We put the condition in the notes after the table
+                    self.render_ctx.add_note(attribute.requirement_level_msg)
+                    required = "Recommended: [{}]".format(len(self.render_ctx.notes))
+
         output.write(
             "| {} | {} | {} | {} | {} |\n".format(
                 name, attr_type, description, examples, required

semantic-conventions/src/tests/data/markdown/deprecated/expected.md

@@ -2,17 +2,17 @@
 
 <!-- Re-generate TOC with `TODO: ADD cmd` -->
 <!-- semconv http -->
-| Attribute  | Type | Description  | Examples  | Required |
+| Attribute  | Type | Description  | Examples  | Requirement Level |
 |---|---|---|---|---|
-| `http.method` | string | HTTP request method. | `GET`; `POST`; `HEAD` | Yes |
-| `http.url` | string | Full HTTP request URL in the form `scheme://host[:port]/path?query[#fragment]`. Usually the fragment is not transmitted over HTTP, but if it is known, it should be included nevertheless. | `https://www.foo.bar/search?q=OpenTelemetry#SemConv` | No |
-| `http.target` | string | The full request target as passed in a HTTP request line or equivalent. | `/path/12314/?q=ddds#123` | No |
-| `http.host` | string | The value of the [HTTP host header](https://tools.ietf.org/html/rfc7230#section-5.4). When the header is empty or not present, this attribute should be the same. | `www.example.org` | No |
-| `http.scheme` | string | The URI scheme identifying the used protocol. | `http`; `https` | No |
-| `http.status_code` | int | [HTTP response status code](https://tools.ietf.org/html/rfc7231#section-6). | `200` | If and only if one was received/sent |
-| `http.status_text` | string | **Deprecated: Use attribute `status_description` instead.**<br>[HTTP reason phrase](https://tools.ietf.org/html/rfc7230#section-3.1.2). | `OK` | No |
-| `http.flavor` | string | **Deprecated. Use attribute `flavor_new` instead.**<br>Kind of HTTP protocol used [1] | `1.0` | No |
-| `http.user_agent` | string | Value of the [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) header sent by the client. | `CERN-LineMode/2.15 libwww/2.17b3` | No |
+| `http.method` | string | HTTP request method. | `GET`; `POST`; `HEAD` | Required |
+| `http.url` | string | Full HTTP request URL in the form `scheme://host[:port]/path?query[#fragment]`. Usually the fragment is not transmitted over HTTP, but if it is known, it should be included nevertheless. | `https://www.foo.bar/search?q=OpenTelemetry#SemConv` | Recommended |
+| `http.target` | string | The full request target as passed in a HTTP request line or equivalent. | `/path/12314/?q=ddds#123` | Recommended |
+| `http.host` | string | The value of the [HTTP host header](https://tools.ietf.org/html/rfc7230#section-5.4). When the header is empty or not present, this attribute should be the same. | `www.example.org` | Recommended |
+| `http.scheme` | string | The URI scheme identifying the used protocol. | `http`; `https` | Recommended |
+| `http.status_code` | int | [HTTP response status code](https://tools.ietf.org/html/rfc7231#section-6). | `200` | Conditionally Required: if and only if one was received/sent |
+| `http.status_text` | string | **Deprecated: Use attribute `status_description` instead.**<br>[HTTP reason phrase](https://tools.ietf.org/html/rfc7230#section-3.1.2). | `OK` | Recommended |
+| `http.flavor` | string | **Deprecated. Use attribute `flavor_new` instead.**<br>Kind of HTTP protocol used [1] | `1.0` | Recommended |
+| `http.user_agent` | string | Value of the [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) header sent by the client. | `CERN-LineMode/2.15 libwww/2.17b3` | Recommended |
 
 **[1]:** If `net.transport` is not specified, it can be assumed to be `IP.TCP` except if `http.flavor` is `QUIC`, in which case `IP.UDP` is assumed.
 

semantic-conventions/src/tests/data/markdown/deprecated/http.yaml

@@ -9,7 +9,7 @@ groups:
     attributes:
       - id: method
         type: string
-        required: always
+        requirement_level: required
         sampling_relevant: false
         brief: 'HTTP request method.'
         examples: ["GET", "POST", "HEAD"]
@@ -35,8 +35,8 @@ groups:
         examples: ["http", "https"]
       - id: status_code
         type: int
-        required:
-          conditional: "If and only if one was received/sent"
+        requirement_level:
+          conditionally_required: "if and only if one was received/sent"
         brief: '[HTTP response status code](https://tools.ietf.org/html/rfc7231#section-6).'
         examples: [200]
       - id: status_text
@@ -96,8 +96,8 @@ groups:
     attributes:
       - id: server_name
         type: string
-        required:
-          conditional: >
+        requirement_level:
+          conditionally_required: >
             This should be obtained via configuration.
         sampling_relevant: false
         brief: >

semantic-conventions/src/tests/data/markdown/deprecated/input.md

@@ -5,15 +5,15 @@
 
 | Attribute name | Notes and examples                                           | Required? |
 | :------------- | :----------------------------------------------------------- | --------- |
-| `http.method` | HTTP request method. E.g. `"GET"`. | Yes |
+| `http.method` | HTTP request method. E.g. `"GET"`. | Required |
 | `http.url` | Full HTTP request URL in the form `scheme://host[:port]/path?query[#fragment]`. Usually the fragment is not transmitted over HTTP, but if it is known, it should be included nevertheless. | Defined later. |
 | `http.target` | The full request target as passed in a [HTTP request line][] or equivalent, e.g. `"/path/12314/?q=ddds#123"`. | Defined later. |
 | `http.host` | The value of the [HTTP host header][]. When the header is empty or not present, this attribute should be the same. | Defined later. |
 | `http.scheme` | The URI scheme identifying the used protocol: `"http"` or `"https"` | Defined later. |
-| `http.status_code` | [HTTP response status code][]. E.g. `200` (integer) | If and only if one was received/sent. |
-| `http.status_text` | [HTTP reason phrase][]. E.g. `"OK"` | No |
+| `http.status_code` | [HTTP response status code][]. E.g. `200` (integer) | Conditionally Required: if and only if one was received/sent. |
+| `http.status_text` | [HTTP reason phrase][]. E.g. `"OK"` | Recommended |
 | `http.flavor` | Kind of HTTP protocol used: `"1.0"`, `"1.1"`, `"2"`, `"SPDY"` or `"QUIC"`. |  No |
-| `http.user_agent` | Value of the HTTP [User-Agent][] header sent by the client. | No |
+| `http.user_agent` | Value of the HTTP [User-Agent][] header sent by the client. | Recommended |
 
 <!-- endsemconv -->