Update semantic-conventions

.htmltest.yml

@@ -33,6 +33,10 @@ IgnoreURLs: # list of regexs of paths or URLs to be ignored
   # Ignore links to GH repo content for now.
   - ^https?://github\.com/.*?/.*?/(blob|tree)/
 
+  # Too many redirects as the server tries to figure out the country and language,
+  # e.g.: https://www.microsoft.com/en-ca/sql-server.
+  - ^https://www.microsoft.com/sql-server$
+
   # TODO: drop after fix to https://github.com/rust-lang/crates.io/issues/788
   - ^https://crates\.io/crates
   # TODO: drop after fix to https://github.com/micrometer-metrics/micrometer-docs/issues/239

hugo.yaml

@@ -187,14 +187,8 @@ module:
       target: content/docs/specs/otlp/_index.md
     - source: tmp/otlp/docs/img
       target: content/docs/specs/otlp/img
-    - source: tmp/semconv/specification/logs
-      target: content/docs/specs/semconv/logs
-    - source: tmp/semconv/specification/metrics
-      target: content/docs/specs/semconv/metrics
-    - source: tmp/semconv/specification/resource
-      target: content/docs/specs/semconv/resource
-    - source: tmp/semconv/specification/trace
-      target: content/docs/specs/semconv/trace
+    - source: tmp/semconv/docs
+      target: content/docs/specs/semconv
     - source: tmp/community/mission-vision-values.md
       target: content/community/mission.md
     - source: tmp/community/roadmap.md

scripts/content-modules/adjust-pages.pl

@@ -14,6 +14,7 @@
 my $otelSpecRepoUrl = 'https://github.com/open-telemetry/opentelemetry-specification';
 my $otlpSpecRepoUrl = 'https://github.com/open-telemetry/opentelemetry-proto';
 my $opAmpSpecRepoUrl = 'https://github.com/open-telemetry/opamp-spec';
+my $semconvSpecRepoUrl = 'https://github.com/open-telemetry/semantic-conventions';
 my $semConvRef = "$otelSpecRepoUrl/blob/main/semantic_conventions/README.md";
 my $specBasePath = '/docs/specs';
 my $path_base_for_github_subdir = "content/en$specBasePath";
@@ -23,7 +24,7 @@
 );
 my $otelSpecVers = $versions{'spec:'};
 my $otlpSpecVers = $versions{'otlp:'};
-my $unused;
+
 # TODO: remove once OpAMP spec has been updated
 my $opampFrontMatter = << "EOS";
 title: Open Agent Management Protocol
@@ -36,6 +37,47 @@
   to: specification.md
 EOS
 
+# TODO: remove once Semconv spec has been updated
+my $semconvFrontMatter = << "EOS";
+linkTitle: Semantic Conventions
+no_list: true
+cascade:
+  body_class: otel-docs-spec
+  github_repo: &repo $semconvSpecRepoUrl
+  github_subdir: docs
+  path_base_for_github_subdir: content/en/docs/specs/semconv/
+  github_project_repo: *repo
+  draft: true
+EOS
+
+# Adjust semconv title capitalization
+sub toTitleCase($) {
+    my $str = shift;
+    my @specialCaseWords = qw(
+        CloudEvents
+        CouchDB
+        DynamoDB
+        FaaS
+        GraphQL
+        gRPC
+        HBase
+        MongoDB
+        OpenTelemetry
+        RabbitMQ
+        RocketMQ
+    );
+    my %specialCases = map { lc($_) => $_ } @specialCaseWords;
+    while ($str =~ /(\b[A-Z]+\b)/g) {
+        $specialCases{lc $1} = $1;
+    }
+    $str =~ s/(\w+)/\u\L$1/g;
+    while (my ($key, $value) = each %specialCases) {
+        $str =~ s/\b\u\L$key\b/$value/g;
+    }
+    $str =~ s/\b(A|And|As|For|In|On)\b/\L$1/g;
+    return $str;
+}
+
 sub printTitleAndFrontMatter() {
   print "---\n";
   if ($title eq 'OpenTelemetry Specification') {
@@ -49,10 +91,52 @@ ()
     $frontMatterFromFile .= "weight: 20\n" if $frontMatterFromFile !~ /^\s*weight/;
   } elsif ($title eq 'OpAMP: Open Agent Management Protocol') {
     $frontMatterFromFile = $opampFrontMatter unless $frontMatterFromFile;
+  } elsif ($title eq 'OpenTelemetry Semantic Conventions') {
+    $frontMatterFromFile = $semconvFrontMatter unless $frontMatterFromFile;
+  } elsif ($ARGV =~ /tmp\/semconv\/docs/) {
+    $title = toTitleCase($title);
+    $linkTitle = 'Database' if $title =~ /Database Calls and Systems$/i;
+    if ($linkTitle =~ /^Database (.*)$/i) {
+      $linkTitle = "$1";
+    } elsif ($linkTitle =~ /^FaaS (.*)$/i) {
+      $linkTitle = "$1";
+    } elsif ($linkTitle =~ /^HTTP (.*)$/i) {
+      $linkTitle = "$1";
+    } elsif ($linkTitle =~ /^Microsoft (.*)$/i) {
+      $linkTitle = "$1";
+    } elsif ($linkTitle =~ /^RPC (.*)$/i) {
+      $linkTitle = "$1";
+    } elsif ($linkTitle =~ /^(Exceptions|Feature Flags) .. (.*)$/i) {
+      $linkTitle = "$2";
+    }
+    if ($linkTitle =~ /^(.*) Attributes$/i && $title ne 'General Attributes') {
+      $linkTitle = "$1";
+    }
+    $linkTitle = 'Attributes' if $title eq 'General Attributes';
+    $linkTitle = 'Events' if $linkTitle eq 'Event';
+    $linkTitle = 'Logs' if $title =~ /Logs Attributes$/;
+    $linkTitle = 'Connect' if $title =~ /Connect RPC$/;
+    $linkTitle = 'SQL' if $title =~ /SQL Databases$/;
+    $title = 'Semantic Conventions for Function-as-a-Service' if $title eq 'Semantic Conventions for FaaS';
+    $linkTitle = 'Tracing Compatibility' if $linkTitle eq 'Tracing Compatibility Components';
+    if ($title =~ /Semantic Convention\b/) {
+      $title =~ s/Semantic Convention\b/$&s/g;
+    }
   }
   my $titleMaybeQuoted = ($title =~ ':') ? "\"$title\"" : $title;
   print "title: $titleMaybeQuoted\n" if $frontMatterFromFile !~ /title: /;
-  ($unused, $linkTitle) = $title =~ /^OpenTelemetry (Protocol )?(.*)/;
+  printf STDOUT ">1 $title -> $linkTitle\n" if $title =~ /Function/;
+  if ($title =~ /^OpenTelemetry (Protocol )?(.*)/) {
+    $linkTitle = $2;
+  } elsif ($title =~ /^(.*?) Semantic Conventions?$/i && !$linkTitle) {
+    $linkTitle = $1;
+  } elsif ($title =~ /^Semantic Conventions? for (.*)$/i && !$linkTitle) {
+    $linkTitle = $1;
+  }
+  if ($linkTitle =~ /^Function.as.a.Service$/i) {
+    $linkTitle = 'FaaS';
+  }
+  printf STDOUT ">2 $title -> $linkTitle\n" if $title =~ /Function/;
   # TODO: add to front matter of OTel spec file and drop next line:
   $linkTitle = 'Design Goals' if $title eq 'Design Goals for OpenTelemetry Wire Protocol';
   print "linkTitle: $linkTitle\n" if $linkTitle and $frontMatterFromFile !~ /linkTitle: /;
@@ -61,6 +145,13 @@ ()
     print "path_base_for_github_subdir:\n";
     print "  from: $path_base_for_github_subdir/otel/$1_index.md\n";
     print "  to: $1README.md\n";
+  } elsif ($ARGV =~ /tmp\/semconv\/docs\/(.*?)_index.md$/) {
+    print "path_base_for_github_subdir:\n";
+    print "  from: $path_base_for_github_subdir/semconv/$1_index.md\n";
+    print "  to: $1README.md\n";
+    if ($linkTitle eq 'General') {
+      print "weight: -1\n";
+    }
   }
   print "---\n";
 }
@@ -84,6 +175,7 @@ ()
   }
   if(! $title) {
     ($title) = /^#\s+(.*)/;
+    $linkTitle = '';
     printTitleAndFrontMatter() if $title;
     next;
   }
@@ -97,6 +189,16 @@ ()
     next;
   }
 
+  ## Semconv
+
+  if ($ARGV =~ /\/semconv/) {
+    s|(\]\()/docs/|$1$specBasePath/semconv/|g;
+    s|(\]:\s*)/docs/|$1$specBasePath/semconv/|;
+
+    # TODO: drop once semconv pages are fixed:
+    s|(/resource/faas\.md)#function-as-a-service|$1|;
+  }
+
   # SPECIFICATION custom processing
 
   s|\(https://github.com/open-telemetry/opentelemetry-specification\)|($specBasePath/otel/)|;

scripts/content-modules/cp-pages.sh

@@ -68,8 +68,8 @@ echo "COMMUNITY pages: copied and processed"
 
 ## Semantic Conventions
 
-SRC=content-modules/semantic-conventions/specification
-DEST=$DEST_BASE/semconv/specification
+SRC=content-modules/semantic-conventions/docs
+DEST=$DEST_BASE/semconv/docs
 
 rm -Rf $DEST
 mkdir -p $DEST

scripts/content-modules/normalize-titles.pl

@@ -0,0 +1,180 @@
+#!/usr/bin/perl -w -i
+#
+# DRAFT script used to normalize semconv doc-page tiles and add Hugo front matter
+#
+
+$^W = 1;
+
+use strict;
+use warnings;
+use diagnostics;
+
+my $file = '';
+my $frontMatterFromFile = '';
+my $title = '';
+my $linkTitle = '';
+my $gD = 0;
+my $otelSpecRepoUrl = 'https://github.com/open-telemetry/opentelemetry-specification';
+my $otlpSpecRepoUrl = 'https://github.com/open-telemetry/opentelemetry-proto';
+my $opAmpSpecRepoUrl = 'https://github.com/open-telemetry/opamp-spec';
+my $semconvSpecRepoUrl = 'https://github.com/open-telemetry/semantic-conventions';
+my $semConvRef = "$otelSpecRepoUrl/blob/main/semantic_conventions/README.md";
+my $specBasePath = '/docs/specs';
+my $path_base_for_github_subdir = "content/en$specBasePath";
+my %versions = qw(
+  spec: 1.22.0
+  otlp: 1.0.0
+);
+my $otelSpecVers = $versions{'spec:'};
+my $otlpSpecVers = $versions{'otlp:'};
+
+# TODO: remove once OpAMP spec has been updated
+my $opampFrontMatter = << "EOS";
+title: Open Agent Management Protocol
+linkTitle: OpAMP
+body_class: otel-docs-spec
+github_repo: &repo $opAmpSpecRepoUrl
+github_project_repo: *repo
+path_base_for_github_subdir:
+  from: content/en/docs/specs/opamp/index.md
+  to: specification.md
+EOS
+
+# TODO: remove once Semconv spec has been updated
+my $semconvFrontMatter = << "EOS";
+linkTitle: Semantic Conventions
+# no_list: true
+cascade:
+  body_class: otel-docs-spec
+  github_repo: &repo $semconvSpecRepoUrl
+  github_subdir: docs
+  path_base_for_github_subdir: content/en/docs/specs/semconv/
+  github_project_repo: *repo
+EOS
+
+# Adjust semconv title capitalization
+sub toTitleCase($) {
+    my $str = shift;
+    my @specialCaseWords = qw(
+        CloudEvents
+        CouchDB
+        DynamoDB
+        FaaS
+        GraphQL
+        gRPC
+        HBase
+        MongoDB
+        OpenTelemetry
+        RabbitMQ
+        RocketMQ
+    );
+    my %specialCases = map { lc($_) => $_ } @specialCaseWords;
+    while ($str =~ /(\b[A-Z]+\b)/g) {
+        $specialCases{lc $1} = $1;
+    }
+    $str =~ s/(\w+)/\u\L$1/g;
+    while (my ($key, $value) = each %specialCases) {
+        $str =~ s/\b\u\L$key\b/$value/g;
+    }
+    $str =~ s/\b(A|And|As|For|In|On)\b/\L$1/g;
+    return $str;
+}
+
+sub printTitleAndFrontMatter() {
+  my $frontMatter = '';
+  my $originalTitle = $title;
+  if ($frontMatterFromFile) {
+    # printf STDOUT "> $file has front matter:\n$frontMatterFromFile\n"; # if $gD;
+    $frontMatterFromFile = '' unless $ARGV =~ /\/system\/[^R]/;
+    # printf STDOUT "> $file\n" if $ARGV =~ /\/system\b/;
+  }
+  if ($title eq 'OpenTelemetry Semantic Conventions') {
+    $frontMatterFromFile = $semconvFrontMatter unless $frontMatterFromFile;
+  } elsif ($ARGV =~ /json-rpc/) {
+    $title = 'Semantic Conventions for JSON-RPC';
+  }
+  $title = toTitleCase($title);
+  my $titleMaybeQuoted = ($title =~ ':') ? "\"$title\"" : $title;
+  # $frontMatter .= "title: $titleMaybeQuoted\n" if $frontMatterFromFile !~ /title: /;
+  if ($title =~ /^OpenTelemetry (Protocol )?(.*)/) {
+    $linkTitle = $2;
+  } elsif ($title =~ /^(.*?) Semantic Conventions?$/i) {
+    $linkTitle = toTitleCase($1);
+  } elsif ($title =~ /^Semantic Conventions? for (.*)$/i) {
+    $linkTitle = toTitleCase($1);
+  }
+  if ($linkTitle =~ /^Function.as.a.Service$/i) {
+    $linkTitle = 'FaaS';
+  }
+  $linkTitle = 'Database' if $title =~ /Database Calls and Systems$/i;
+  if ($linkTitle =~ /^Database (.*)$/i) {
+    $linkTitle = "$1";
+  } elsif ($linkTitle =~ /^FaaS (.*)$/i) {
+    $linkTitle = "$1";
+  } elsif ($linkTitle =~ /^HTTP (.*)$/i) {
+    $linkTitle = "$1";
+  } elsif ($linkTitle =~ /^Microsoft (.*)$/i) {
+    $linkTitle = "$1";
+  } elsif ($linkTitle =~ /^RPC (.*)$/i) {
+    $linkTitle = "$1";
+  } elsif ($linkTitle =~ /^(Exceptions|Feature Flags) .. (.*)$/i) {
+    $linkTitle = "$2";
+  }
+  if ($linkTitle =~ /^(.*) Attributes$/i && $title ne 'General Attributes') {
+    $linkTitle = "$1";
+  }
+  $linkTitle = 'Attributes' if $title eq 'General Attributes';
+  $linkTitle = 'Events' if $linkTitle eq 'Event';
+  $linkTitle = 'Logs' if $title =~ /Logs Attributes$/;
+  $linkTitle = 'Connect' if $title =~ /Connect RPC$/;
+  $linkTitle = 'SQL' if $title =~ /SQL Databases$/;
+  $title = 'Semantic Conventions for Function-as-a-Service' if $title eq 'Semantic Conventions for FaaS';
+  $linkTitle = 'Tracing Compatibility' if $linkTitle eq 'Tracing Compatibility Components';
+  if ($title =~ /Semantic Convention\b/) {
+    $title =~ s/Semantic Convention\b/$&s/g;
+    printf STDOUT "> $title -> $linkTitle\n";
+  }
+
+  $frontMatter .= "linkTitle: $linkTitle\n" if $linkTitle and $frontMatterFromFile !~ /linkTitle: /;
+  $frontMatter .= $frontMatterFromFile if $frontMatterFromFile;
+  if ($ARGV =~ /docs\/(.*?)README.md$/) {
+    $frontMatter .= "path_base_for_github_subdir:\n";
+    $frontMatter .= "  from: $path_base_for_github_subdir/semconv/$1_index.md\n";
+    $frontMatter .= "  to: $1README.md\n";
+  }
+  $frontMatter .= "weight: -1\n" if $title eq 'General Semantic Conventions';
+  if ($frontMatter) {
+    $frontMatter = "<!--- Hugo front matter used to generate the website version of this page:\n" . $frontMatter;
+    $frontMatter .= "--->\n";
+    print "$frontMatter\n";
+  }
+  print "# $title\n"
+}
+
+# main
+
+while(<>) {
+  # printf STDOUT "$ARGV Got: $_" if $gD;
+
+  if ($file ne $ARGV) {
+    $file = $ARGV;
+    # printf STDOUT "> $file\n"; # if $gD;
+    $frontMatterFromFile = '';
+    $title = '';
+    if (/^<!---? Hugo/) {
+        while(<>) {
+          last if /^-?-->/;
+          $frontMatterFromFile .= $_;
+        }
+        next;
+    }
+  }
+  if(! $title) {
+    ($title) = /^#\s+(.*)/;
+    $linkTitle = '';
+    printTitleAndFrontMatter() if $title;
+    next;
+  }
+
+  print;
+}