Allow to configure config extensions for repos

cmd/server/docs/docs.go

@@ -5049,6 +5049,9 @@ const docTemplate = `{
                 "clone_url_ssh": {
                     "type": "string"
                 },
+                "config_extension_endpoint": {
+                    "type": "string"
+                },
                 "config_file": {
                     "type": "string"
                 },
@@ -5095,6 +5098,9 @@ const docTemplate = `{
                 "scm": {
                     "$ref": "#/definitions/SCMKind"
                 },
+                "secret_extension_endpoint": {
+                    "type": "string"
+                },
                 "timeout": {
                     "type": "integer"
                 },
@@ -5121,6 +5127,9 @@ const docTemplate = `{
                         "$ref": "#/definitions/WebhookEvent"
                     }
                 },
+                "config_extension_endpoint": {
+                    "type": "string"
+                },
                 "config_file": {
                     "type": "string"
                 },
@@ -5130,6 +5139,9 @@ const docTemplate = `{
                 "netrc_only_trusted": {
                     "type": "boolean"
                 },
+                "secret_extension_endpoint": {
+                    "type": "string"
+                },
                 "timeout": {
                     "type": "integer"
                 },

cmd/server/flags.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/urfave/cli/v3"
 
+	"go.woodpecker-ci.org/woodpecker/v2/server/services/utils/hostmatcher"
 	"go.woodpecker-ci.org/woodpecker/v2/shared/constant"
 	"go.woodpecker-ci.org/woodpecker/v2/shared/logger"
 )
@@ -203,6 +204,12 @@ var flags = append([]cli.Flag{
 		Name:    "config-service-endpoint",
 		Usage:   "url used for calling configuration service endpoint",
 	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("WOODPECKER_ALLOWED_EXTENSIONS_HOSTS"),
+		Name:    "allowed-extensions-hosts",
+		Usage:   "Hosts that are allowed to be used by extensions",
+		Value:   hostmatcher.MatchBuiltinExternal,
+	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("WOODPECKER_DATABASE_DRIVER"),
 		Name:    "driver",

docs/docs/20-usage/72-extensions/40-configuration-extension.md

@@ -0,0 +1,164 @@
+# Configuration extension
+
+The configuration extension can be used to modify or generate Woodpeckers pipeline configurations. You can configure a HTTP
+endpoint in the repository settings in the extensions tab.
+
+Using such an extension can be useful if you want to:
+
+<!-- cSpell:words templating,Starlark,Jsonnet -->
+- Preprocess the original configuration file with something like go templating
+- Convert custom attributes to Woodpecker attributes
+- Add defaults to the configuration like default steps
+- Convert configuration files from a totally different format like Gitlab CI config, Starlark, Jsonnet, ...
+- Centralize configuration for multiple repositories in one place
+
+## Security
+
+:::warning
+As Woodpecker will pass private information like tokens and will execute the returned configuration, it is extremely important to secure the external extension. Therefore Woodpecker signs every request. Read more about it in the [security section](./10-extensions.md#security).
+:::
+
+## Global configuration
+
+In addition to the ability to configure the extension per repository, you can also configure a global endpoint in the Woodpecker server configuration. This can be useful if you want to use the extension for all repositories. Be careful if
+you share your Woodpecker server with others as they will also use your configuration extension.
+
+The global configuration will be called before the repository specific configuration extension if both are configured.
+
+```ini title="Server"
+WOODPECKER_CONFIG_SERVICE_ENDPOINT=https://example.com/ciconfig
+```
+
+## How it works
+
+When a pipeline is triggered Woodpecker will fetch the pipeline configuration from the repository, then make a HTTP POST request to the configured extension with a JSON payload containing some data like the repository, pipeline information and the current config files retrieved from the repository. The extension can then send back modified or even new pipeline configurations following Woodpeckers official yaml format that should be used.
+
+:::tip
+The netrc data is pretty powerful as it contains credentials to access the repository. You can use this to clone the repository or even use the forge (Github or Gitlab, ...) api to get more information about the repository.
+:::
+
+### Request
+
+The extension receives an HTTP POST request with the following JSON payload:
+
+```ts
+class Request {
+  repo: Repo;
+  pipeline: Pipeline;
+  netrc: Netrc;
+  configuration: {
+    name: string; // filename of the configuration file
+    data: string; // content of the configuration file
+  }[];
+}
+```
+
+Checkout the following models for more information:
+
+- [repo model](https://github.com/woodpecker-ci/woodpecker/blob/main/server/model/repo.go)
+- [pipeline model](https://github.com/woodpecker-ci/woodpecker/blob/main/server/model/pipeline.go)
+- [netrc model](https://github.com/woodpecker-ci/woodpecker/blob/main/server/model/netrc.go)
+
+:::tip
+The `netrc` data is pretty powerful as it contains credentials to access the repository. You can use this to clone the repository or even use the forge (Github or Gitlab, ...) api to get more information about the repository.
+:::
+
+Example request:
+
+```json
+{
+  "repo": {
+    "id": 100,
+    "uid": "",
+    "user_id": 0,
+    "namespace": "",
+    "name": "woodpecker-test-pipeline",
+    "slug": "",
+    "scm": "git",
+    "git_http_url": "",
+    "git_ssh_url": "",
+    "link": "",
+    "default_branch": "",
+    "private": true,
+    "visibility": "private",
+    "active": true,
+    "config": "",
+    "trusted": false,
+    "protected": false,
+    "ignore_forks": false,
+    "ignore_pulls": false,
+    "cancel_pulls": false,
+    "timeout": 60,
+    "counter": 0,
+    "synced": 0,
+    "created": 0,
+    "updated": 0,
+    "version": 0
+  },
+  "pipeline": {
+    "author": "myUser",
+    "author_avatar": "https://myforge.com/avatars/d6b3f7787a685fcdf2a44e2c685c7e03",
+    "author_email": "my@email.com",
+    "branch": "main",
+    "changed_files": ["some-filename.txt"],
+    "commit": "2fff90f8d288a4640e90f05049fe30e61a14fd50",
+    "created_at": 0,
+    "deploy_to": "",
+    "enqueued_at": 0,
+    "error": "",
+    "event": "push",
+    "finished_at": 0,
+    "id": 0,
+    "link_url": "https://myforge.com/myUser/woodpecker-testpipe/commit/2fff90f8d288a4640e90f05049fe30e61a14fd50",
+    "message": "test old config\n",
+    "number": 0,
+    "parent": 0,
+    "ref": "refs/heads/main",
+    "refspec": "",
+    "clone_url": "",
+    "reviewed_at": 0,
+    "reviewed_by": "",
+    "sender": "myUser",
+    "signed": false,
+    "started_at": 0,
+    "status": "",
+    "timestamp": 1645962783,
+    "title": "",
+    "updated_at": 0,
+    "verified": false
+  },
+  "configs": [
+    {
+      "name": ".woodpecker.yaml",
+      "data": "steps:\n  - name: backend\n    image: alpine\n    commands:\n      - echo \"Hello there from Repo (.woodpecker.yaml)\"\n"
+    }
+  ]
+}
+```
+
+### Response
+
+The extension should respond with a JSON payload containing the new configuration files in Woodpeckers official yaml format.
+If the extension wants to keep the existing configuration files, it can respond with **HTTP 204**.
+
+```ts
+class Response {
+  configs: {
+    name: string; // filename of the configuration file
+    data: string; // content of the configuration file
+  }[];
+}
+```
+
+Example response:
+
+```json
+{
+  "configs": [
+    {
+      "name": "central-override",
+      "data": "steps:\n  - name: backend\n    image: alpine\n    commands:\n      - echo \"Hello there from ConfigAPI\"\n"
+    }
+  ]
+}
+```

docs/docs/20-usage/72-extensions/72-extensions.md

@@ -0,0 +1,34 @@
+# Extensions
+
+Woodpecker allows you to replace internal logic with external extensions by using pre-defined http endpoints.
+
+There are currently two types of extensions available:
+
+- [Configuration extension](./40-configuration-extension.md) to modify or generate Woodpeckers pipeline configurations.
+
+## Security
+
+:::warning
+You need to trust the extensions as they are receiving private information like secrets and tokens and might return harmful
+data like malicious pipeline configurations that could be executed.
+:::
+
+To prevent your extensions from such attacks, Woodpecker is signing all http-requests using [http signatures](https://tools.ietf.org/html/draft-cavage-http-signatures). Woodpecker therefore uses a public-private ed25519 key pair.
+To verify the requests your extension has to verify the signature of all request using the public key with some library like [httpsign](https://github.com/yaronf/httpsign).
+You can get the public Woodpecker key by opening `http://my-woodpecker.tld/api/signature/public-key` or by visiting the Woodpecker UI, going to you repo settings and opening the extensions page.
+
+## Example extensions
+
+A simplistic service providing endpoints for a config and secrets extension can be found here: [https://github.com/woodpecker-ci/example-extensions](https://github.com/woodpecker-ci/example-extensions)
+
+## Configuration
+
+To prevent extensions from calling local services by default only external hosts / ip-addresses are allowed. You can change this behavior by setting the `WOODPECKER_ALLOWED_EXTENSIONS_HOSTS` environment variable. You can use a comma separated list of:
+
+- Built-in networks:
+  - `loopback`: 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included.
+  - `private`: RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet.
+  - `external`: A valid non-private unicast IP, you can access all hosts on public internet.
+  - `*`: All hosts are allowed.
+- CIDR list: `1.2.3.0/8` for IPv4 and `2001:db8::/32` for IPv6
+- (Wildcard) hosts: `example.com`, `*.example.com`, `192.168.100.*`

docs/docs/20-usage/72-extensions/_category_.yml

@@ -0,0 +1,7 @@
+label: 'Extensions'
+# position: 3
+collapsible: true
+collapsed: true
+link:
+  type: 'doc'
+  id: 'extensions'

docs/docs/30-administration/10-server-config.md

@@ -482,6 +482,12 @@ Supported variables:
 
 Specify a configuration service endpoint, see [Configuration Extension](./40-advanced/100-external-configuration-api.md)
 
+### `WOODPECKER_ALLOWED_EXTENSIONS_HOSTS`
+
+> Default: `external`
+
+Comma-separated list of allowed hosts for extensions. Possible values are `loopback`, `private`, `external`, `*` or CIDR list.
+
 ### `WOODPECKER_FORGE_TIMEOUT`
 
 > Default: 3s

server/api/repo.go

@@ -263,6 +263,9 @@ func PatchRepo(c *gin.Context) {
 			return
 		}
 	}
+	if in.ConfigExtensionEndpoint != nil {
+		repo.ConfigExtensionEndpoint = *in.ConfigExtensionEndpoint
+	}
 
 	err := _store.UpdateRepo(repo)
 	if err != nil {

server/model/pagination.go

@@ -14,6 +14,11 @@
 
 package model
 
+import (
+	"fmt"
+	"strings"
+)
+
 type ListOptions struct {
 	All     bool
 	Page    int
@@ -32,3 +37,21 @@ func ApplyPagination[T any](d *ListOptions, slice []T) []T {
 	}
 	return slice[d.PerPage*(d.Page-1) : d.PerPage*(d.Page)]
 }
+
+func (d *ListOptions) Encode() string {
+	var query []string
+
+	if d.Page != 0 {
+		query = append(query, fmt.Sprintf("page=%d", d.Page))
+	}
+
+	if d.PerPage != 0 {
+		query = append(query, fmt.Sprintf("per_page=%d", d.PerPage))
+	}
+
+	if d.All {
+		query = append(query, "all=true")
+	}
+
+	return strings.Join(query, "&")
+}

server/model/repo.go

@@ -50,6 +50,7 @@ type Repo struct {
 	Hash                         string         `json:"-"                               xorm:"varchar(500) 'hash'"`
 	Perm                         *Perm          `json:"-"                               xorm:"-"`
 	CancelPreviousPipelineEvents []WebhookEvent `json:"cancel_previous_pipeline_events" xorm:"json 'cancel_previous_pipeline_events'"`
+	ConfigExtensionEndpoint      string         `json:"config_extension_endpoint"       xorm:"varchar(500) 'config_extension_endpoint'"`
 	NetrcOnlyTrusted             bool           `json:"netrc_only_trusted"              xorm:"NOT NULL DEFAULT true 'netrc_only_trusted'"`
 } //	@name Repo
 
@@ -116,6 +117,7 @@ type RepoPatch struct {
 	AllowPull                    *bool           `json:"allow_pr,omitempty"`
 	AllowDeploy                  *bool           `json:"allow_deploy,omitempty"`
 	CancelPreviousPipelineEvents *[]WebhookEvent `json:"cancel_previous_pipeline_events"`
+	ConfigExtensionEndpoint      *string         `json:"config_extension_endpoint,omitempty"`
 	NetrcOnlyTrusted             *bool           `json:"netrc_only_trusted"`
 } //	@name RepoPatch
 

server/services/config/combined_test.go

@@ -35,6 +35,7 @@ import (
 	forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types"
 	"go.woodpecker-ci.org/woodpecker/v2/server/model"
 	"go.woodpecker-ci.org/woodpecker/v2/server/services/config"
+	"go.woodpecker-ci.org/woodpecker/v2/server/services/utils"
 )
 
 func TestFetchFromConfigService(t *testing.T) {
@@ -186,7 +187,13 @@ func TestFetchFromConfigService(t *testing.T) {
 
 	ts := httptest.NewServer(http.HandlerFunc(fixtureHandler))
 	defer ts.Close()
-	httpFetcher := config.NewHTTP(ts.URL+"/", privEd25519Key)
+
+	client, err := utils.NewHTTPClient(privEd25519Key, "loopback")
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	httpFetcher := config.NewHTTP(ts.URL+"/", client)
 
 	for _, tt := range testTable {
 		t.Run(tt.name, func(t *testing.T) {