feat: Separate footer parsing, information extraction and link genera…

…tion. (#51)

extracting information from footers separately to link generation allows
all extracted information to be made available in post_process body
templates, and removes the need to hack the "link.text" into
post_process bodies to get access to the triggering issue_ref.

closes #47

changelog_gen/config.py

@@ -27,6 +27,8 @@
 
 FOOTER_PARSERS = [
     r"(Refs)(: )(#?[\w-]+)",
+    # TODO(edgy): Parse github behind a github config
+    # https://github.com/NRWLDev/changelog-gen/issues/50
     r"(closes)( )(#[\w-]+)",
     r"(fixes)( )(#[\w-]+)",
     r"(Authors)(: )(.*)",
@@ -37,7 +39,7 @@
 class PostProcessConfig:
     """Post Processor configuration options."""
 
-    link_parser: dict[str, str] | None = None
+    link_generator: dict[str, str] | None = None
     verb: str = "POST"
     # The body to send as a post-processing command,
     # can have the entries: ::issue_ref::, ::version::
@@ -90,7 +92,9 @@ class Config:
     date_format: str | None = None
     version_string: str = "v{new_version}"
     footer_parsers: list[str] = dataclasses.field(default_factory=lambda: FOOTER_PARSERS[::])
-    link_parsers: list[dict[str, str]] = dataclasses.field(default_factory=list)
+    # footers can be a list to simplify configuration for footers sharing related information
+    extractors: list[dict[str, str | list[str]]] = dataclasses.field(default_factory=list)
+    link_generators: list[dict[str, str]] = dataclasses.field(default_factory=list)
     change_template: str | None = None
 
     # Hooks

changelog_gen/extractor.py

@@ -3,6 +3,7 @@
 import dataclasses
 import re
 import typing as t
+from collections import defaultdict
 
 from changelog_gen.util import timer
 
@@ -35,6 +36,7 @@ class Change:  # noqa: D101
     scope: str = ""
     breaking: bool = False
     footers: list[Footer] = dataclasses.field(default_factory=list)
+    extractions: dict[str, list[str]] = dataclasses.field(default_factory=dict)
     links: list[Link] = dataclasses.field(default_factory=list)
     rendered: str = ""  # This is populated by the writer at run time
 
@@ -123,6 +125,22 @@ def extract(self: t.Self) -> list[Change]:  # noqa: C901, PLR0912, PLR0915
                             self.context.info("  '%s' footer extracted '%s%s%s'", parser, m[1], m[2], m[3])
                             footers[m[1].lower()] = Footer(m[1], m[2], m[3])
 
+                extractions = defaultdict(list)
+
+                for extractor in self.context.config.extractors:
+                    footer_keys = extractor["footer"]
+                    if not isinstance(footer_keys, list):
+                        footer_keys = [footer_keys]
+
+                    for fkey in footer_keys:
+                        footer = footers.get(fkey.lower())
+                        if footer is None:
+                            continue
+
+                        for m in re.finditer(extractor["pattern"], footer.value):
+                            for k, v in m.groupdict().items():
+                                extractions[k].append(v)
+
                 header = self.type_headers.get(commit_type, commit_type)
                 change = Change(
                     header=header,
@@ -133,25 +151,22 @@ def extract(self: t.Self) -> list[Change]:  # noqa: C901, PLR0912, PLR0915
                     commit_hash=commit_hash,
                     commit_type=commit_type,
                     footers=list(footers.values()),
+                    extractions=extractions,
                 )
 
                 links = []
 
-                for parser in self.context.config.link_parsers:
-                    if parser["target"] == "__change__":
-                        text_template = parser.get("text", "{0}")
-                        link_template = parser["link"]
-                        links.append(Link(text_template.format(change), link_template.format(change)))
+                for generator in self.context.config.link_generators:
+                    if generator["source"] == "__change__":
+                        values = [change]
                     else:
-                        footer = footers.get(parser["target"].lower())
-                        if footer is None:
+                        values = extractions.get(generator["source"].lower())
+                        if not values:
                             continue
-                        text_template = parser.get("text", "{0}")
-                        link_template = parser["link"]
-                        matches = re.findall(parser["pattern"], footer.value)
-                        links.extend([
-                            Link(text_template.format(match), link_template.format(match)) for match in matches
-                        ])
+
+                    text_template = generator.get("text", "{0}")
+                    link_template = generator["link"]
+                    links.extend([Link(text_template.format(value), link_template.format(value)) for value in values])
 
                 change.links = links
                 changes.append(change)

changelog_gen/post_processor.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import os
-import re
 import typing as t
 from http import HTTPStatus
 
@@ -60,13 +59,6 @@ def make_client(context: Context, cfg: PostProcessConfig) -> httpx.Client:
     )
 
 
-def _get_footer(change: Change, footer: str) -> str | None:
-    for footer_ in change.footers:
-        if footer_.footer.lower() == footer.lower():
-            return footer_
-    return None
-
-
 @timer
 def per_issue_post_process(
     context: Context,
@@ -77,8 +69,8 @@ def per_issue_post_process(
     dry_run: bool = False,
 ) -> None:
     """Run post process for all provided issue references."""
-    link_parser, body_template = cfg.link_parser, cfg.body_template
-    if link_parser is None:
+    link_generator, body_template = cfg.link_generator, cfg.body_template
+    if link_generator is None:
         return
     context.warning("Post processing:")
 
@@ -87,33 +79,33 @@ def per_issue_post_process(
     for change in changes:
         link = None
 
-        footer = _get_footer(change, link_parser["target"])
-        if footer is None:
+        source = change.extractions.get(link_generator["source"].lower())
+        if not source:
             continue
 
-        text_template = link_parser.get("text", "{0}")
-        link_template = link_parser["link"]
-        matches = re.findall(link_parser["pattern"], footer.value)
-        link = Link(text_template.format(matches[0]), link_template.format(matches[0]))
+        text_template = link_generator.get("text", "{0}")
+        link_template = link_generator["link"]
+        for value in source:
+            link = Link(text_template.format(value), link_template.format(value))
 
-        rtemplate = Environment(loader=BaseLoader()).from_string(body_template)  # noqa: S701
+            rtemplate = Environment(loader=BaseLoader()).from_string(body_template)  # noqa: S701
 
-        body = rtemplate.render(link=link, change=change, version=version_tag)
-        context.indent()
-        if dry_run:
-            context.warning("Would request: %s %s %s", cfg.verb, link.link, body)
-        else:
-            context.info("Request: %s %s", cfg.verb, link.link)
-            r = client.request(
-                method=cfg.verb,
-                url=link.link,
-                content=body,
-            )
+            body = rtemplate.render(source=value, change=change, version=version_tag, **change.extractions)
             context.indent()
-            try:
-                context.info("Response: %s", HTTPStatus(r.status_code).name)
-                r.raise_for_status()
-            except httpx.HTTPError as e:
-                context.error("Post process request failed.")
-                context.warning("%s", e.response.text)
+            if dry_run:
+                context.warning("Would request: %s %s %s", cfg.verb, link.link, body)
+            else:
+                context.info("Request: %s %s", cfg.verb, link.link)
+                r = client.request(
+                    method=cfg.verb,
+                    url=link.link,
+                    content=body,
+                )
+                context.indent()
+                try:
+                    context.info("Response: %s", HTTPStatus(r.status_code).name)
+                    r.raise_for_status()
+                except httpx.HTTPError as e:
+                    context.error("Post process request failed.")
+                    context.warning("%s", e.response.text)
     context.reset()

docs/commits.md

@@ -36,9 +36,11 @@ Optional footers that are parsed by `changelog-gen` are:
 
 * `BREAKING CHANGE:[ details]`
 * `Refs: [#]<issue_ref>`
-* `PR: [#]<pull_ref>`
 * `Authors: (<author>, ...)`
 
+Parsing additional/custom footers is supported with
+[footer_parsers](https://nrwldev.github.io/changelog-gen/configuration/#footer_parsers).
+
 ### Github support
 
 Github makes use of `closes #<issue_ref>` to close an issue when merged, this

docs/configuration.md

@@ -122,33 +122,57 @@ footer_parsers = [
 ]
 ```
 
-### `link_parsers`
+### `extractors`
   _**[optional]**_<br />
   **default**: None
 
-  Define parsers to extract information from footers (or changelog entry) and
-  generate a link. Define a target footer, provide a regex pattern to extract
+  Define parsers to extract information from footers and store them on the
+  change object. Extractors should used named groups in regex expressions, this
+  groups are the key to retrieve the information later in link generation or
+  post processing. Extractors find all matches in a footer, so will be a
+  list of all matched values. `Refs: 1, 2` will be parsed as
+  `{"issue_ref": ["1", "2"]}` for example.
+
+  The footer configuration can be a single footer, or a list of related footers.
+
+  Example:
+
+```toml
+[[tool.changelog_gen.extractors]]
+footer = "Refs"
+pattern = '(?P<issue_ref>\d+)'
+
+[[tool.changelog_gen.extractors]]
+footer = ["closes", "fixes"]
+pattern = '#(?P<issue_ref>\d+)'
+```
+
+### `link_generators`
+  _**[optional]**_<br />
+  **default**: None
+
+  Make use of extracted information to generate links to different content.
+  Define an extraction source, provide a regex pattern to extract
   information from the footer, and define the link format, optionally define
   the link text format. Link text will default to the extracted information.
 
-  A special target `__change__` is provided to generate links using information
+  A special source `__change__` is provided to generate links using information
   directly from the change object (namely commit hashes).
 
-  Where a pattern is matched multiple times, a link for each match will be
-  created. This allows adding links to multiple authors from the Author for for
-  example.
+  Where an extraction contains multiple values, a link for each match will be
+  created. This allows adding links to multiple authors from the Author footer
+  for example.
 
   Example:
 
 ```toml
-[[tool.changelog_gen.link_parsers]]
-target = "Refs"
-pattern = "#(\\d+)$"
-link = "https =//github.com/NRWLDev/changelog-gen/issues/{0}"
-
-[[tool.changelog_gen.link_parsers]]
-target = "__change__"
-link = "https =//github.com/NRWLDev/changelog-gen/commit/{0.commit_hash}"
+[[tool.changelog_gen.link_generators]]
+source = "issue_ref"
+link = "https://github.com/NRWLDev/changelog-gen/issues/{0}"
+
+[[tool.changelog_gen.link_generators]]
+source = "__change__"
+link = "https://github.com/NRWLDev/changelog-gen/commit/{0.commit_hash}"
 text = "{0.short_hash}"
 ```
 
@@ -404,7 +428,7 @@ pre_l = ["dev", "rc"]
 
   See example on below Jira configuration information.
 
-#### `post_process.url`
+#### `post_process.link_generator`
   _**[required]**_<br />
   **default**: None<br />
   The url to contact.
@@ -415,11 +439,14 @@ pre_l = ["dev", "rc"]
   **default**: POST<br />
   HTTP method to use.
 
-#### `post_process.body`
+#### `post_process.body_template`
   _**[optional]**_<br />
-  **default**: `{"body": "Released on ::version::"}`<br />
+  **default**: `{"body": "Released on {{ version }}"}`<br />
   The text to send to the API.
-  Can have the placeholders `::issue_ref::` and `::version::`.
+  Can have the placeholders
+  * `source` (usually the issue ref from the extracted information)
+  * `version` the version being released
+  * Any extracted key from defined extractors that had a match.
 
 #### `post_process.headers`
   _**[optional]**_<br />
@@ -444,9 +471,10 @@ pre_l = ["dev", "rc"]
 
 ```toml
 [tool.changelog_gen.post_process]
-url = "https://your-domain.atlassian.net/rest/api/2/issue/ISSUE-::issue_ref::/comment"
+link_generator.source = "issue_ref"
+link_generator.link = "https://your-domain.atlassian.net/rest/api/2/issue/ISSUE-{0}/comment"
 verb = "POST"
-body = '{"body": "Released on ::version::"}'
+body = '{"body": "Released on {{ version }}"}'
 auth_env = "JIRA_AUTH"
 headers."content-type" = "application/json"
 ```

docs/post_process.md

@@ -7,16 +7,16 @@ released as a part of.
 
 The configured [post
 process](https://nrwldev.github.io/changelog-gen/configuration/#post_process)
-url should contain an `::issue_ref::` place holder, when processing each commit
-issue, the url will be dynamically updated before
-being called.
-
-By default the url will be called using a `POST` request, but the http verb can
-be changed depending on the service being called, and its requirements. The
-request
-[body](https://nrwldev.github.io/changelog-gen/configuration/#post_processbody) can
-also be configured with a `::version::` placeholder to add a comment to an
-existing issue.
+link generator can refer to any extracted information, if an extractor matches
+multiple times, the post process will be called for each match for that change.
+
+By default the generated link will be called using a `POST` request, but the
+http verb can be changed depending on the service being called, and its
+requirements. The request
+[body](https://nrwldev.github.io/changelog-gen/configuration/#post_processbody_template)
+can also be configured with a jinja template, provided at render time will be
+the `version` string, the `source` from the extracted match, and all extracted
+key values.
 
 Optional
 [headers](https://nrwldev.github.io/changelog-gen/configuration/#post_processheaders)

pyproject.toml

@@ -71,26 +71,18 @@ allowed_branches = [
 ]
 date_format = "- %Y-%m-%d"
 
-[[tool.changelog_gen.link_parsers]]
-target = "closes"
-pattern = '#(\d+)$'
-link = "https =//github.com/NRWLDev/changelog-gen/issues/{0}"
-
-[[tool.changelog_gen.link_parsers]]
-target = "fixes"
-pattern = '#(\d+)$'
-link = "https =//github.com/NRWLDev/changelog-gen/issues/{0}"
-
-# Fall back in case a PR does not close an issue
-[[tool.changelog_gen.link_parsers]]
-target = "Refs"
-pattern = '#(\d+)$'
-link = "https =//github.com/NRWLDev/changelog-gen/issues/{0}"
-
-[[tool.changelog_gen.link_parsers]]
-target = "__change__"
+[[tool.changelog_gen.extractors]]
+footer = ["closes", "fixes", "Refs"]
+pattern = '#(?P<issue_ref>\d+)'
+
+[[tool.changelog_gen.link_generators]]
+source = "issue_ref"
+link = "https://github.com/NRWLDev/changelog-gen/issues/{0}"
+
+[[tool.changelog_gen.link_generators]]
+source = "__change__"
 text = "{0.short_hash}"
-link = "https =//github.com/NRWLDev/changelog-gen/commit/{0.commit_hash}"
+link = "https://github.com/NRWLDev/changelog-gen/commit/{0.commit_hash}"
 
 [[tool.changelog_gen.files]]
 filename = "README.md"