Ft header rewrite

src/dashboard/apigateway/apigateway/apps/plugin/constants.py

@@ -32,6 +32,7 @@ class PluginTypeEnum(StructuredEnum):
 class PluginTypeCodeEnum(StructuredEnum):
     BK_RATE_LIMIT = EnumField("bk-rate-limit", label=_("频率控制"))
     BK_CORS = EnumField("bk-cors", label="CORS")
+    BK_HEADER_REWRITE = EnumField("bk-header-rewrite", label=_("Header 转换"))
 
 
 class PluginBindingScopeEnum(StructuredEnum):

src/dashboard/apigateway/apigateway/apps/plugin/management/commands/migrate_header_rewrite.py

@@ -0,0 +1,77 @@
+#
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+#
+import json
+import logging
+
+from django.core.management.base import BaseCommand
+from django.core.paginator import Paginator
+
+from apigateway.apigateway.apps.plugin.constants import PluginBindingScopeEnum
+from apigateway.apigateway.common.plugin.header_rewrite import HeaderRewriteConvertor
+from apigateway.apigateway.core.constants import ContextScopeTypeEnum, ContextTypeEnum
+from apigateway.core.models import Context, Release, Stage
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """将stage/resource的proxy请求头配置迁移成bk-header-rewrite插件配置"""
+
+    def handle(self, *args, **options):
+        # 遍历stage, 迁移proxy请求头
+        qs = Stage.objects.prefetch_related("api").all()
+
+        logger.info("start migrate header rewrite plugin config, all stage count %s", qs.count())
+
+        paginator = Paginator(qs, 100)
+        for i in paginator.page_range:
+
+            logger.info("migrate stage count %s", (i + 1) * 100)
+
+            for stage in paginator.page(i):
+                context = Context.objects.filter(
+                    scope_type=ContextScopeTypeEnum.STAGE.value,
+                    scope_id=stage.id,
+                    type=ContextTypeEnum.STAGE_PROXY_HTTP.value,
+                ).first()
+
+                # 1. 迁移stage的proxy请求头
+                stage_transform_headers = context.config.get("transform_headers", {}) if context else {}
+                stage_config = HeaderRewriteConvertor.transform_headers_to_plugin_config(stage_transform_headers)
+                HeaderRewriteConvertor.alter_plugin(
+                    stage.api, PluginBindingScopeEnum.STAGE.value, stage.id, stage_config
+                )
+
+                # 2. 迁移resource的proxy请求头
+                release = Release.objects.filter(stage=stage).prefetch_related("resource_version").first()
+                if not release:
+                    continue
+
+                for resource in release.resource_version.data:
+                    resource_proxy = json.loads(resource["proxy"]["config"])
+                    resource_transform_headers = resource_proxy.get("transform_headers", {})
+                    resource_config = HeaderRewriteConvertor.transform_headers_to_plugin_config(
+                        resource_transform_headers
+                    )
+
+                    plugin_config = HeaderRewriteConvertor.merge_plugin_config(stage_config, resource_config)
+                    HeaderRewriteConvertor.alter_plugin(
+                        stage.api, PluginBindingScopeEnum.RESOURCE.value, resource["id"], plugin_config
+                    )
+
+        logger.info("finish migrate header rewrite plugin config")

src/dashboard/apigateway/apigateway/apps/plugin/models.py

@@ -22,7 +22,6 @@
 from django.db.models.signals import post_delete, pre_save
 from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
-from jsonschema import validate
 from tencent_apigateway_common.i18n.field import I18nProperty
 
 from apigateway.apps.plugin.constants import PluginBindingScopeEnum, PluginStyleEnum, PluginTypeEnum
@@ -154,20 +153,8 @@ def config(self) -> Dict[str, Any]:
 
     @config.setter
     def config(self, yaml_: str):
-        loaded_config = yaml_loads(yaml_)
-        self._validate_config(loaded_config)
         self.yaml = yaml_
 
-    def _validate_config(self, config: Dict[str, Any]):
-        if not isinstance(config, dict):
-            raise ValueError("config must be a dict")
-
-        schema = self.type and self.type.schema
-        if not schema:
-            return
-
-        validate(config, schema=schema.schema)
-
     def __str__(self) -> str:
         return f"<PluginConfig {self.name}({self.pk})>"
 

src/dashboard/apigateway/apigateway/apps/plugin/plugin/checker.py

@@ -87,9 +87,20 @@ def _check_duplicate_items(self, data: List[str], key: str):
             raise ValueError(_("{} 存在重复的元素：{}。").format(key, ", ".join(duplicate_items)))
 
 
+class HeaderRewriteChecker(BaseChecker):
+    def check(self, yaml_: str):
+        loaded_data = yaml_loads(yaml_)
+
+        set_keys = [item["key"] for item in loaded_data["set"]]
+        duplicate_keys = [key for key, count in Counter(set_keys).items() if count >= 2]
+        if duplicate_keys:
+            raise ValueError(_("set 存在重复的元素：{}。").format(", ".join(duplicate_keys)))
+
+
 class PluginConfigYamlChecker:
     type_code_to_checker: ClassVar[Dict[str, BaseChecker]] = {
         PluginTypeCodeEnum.BK_CORS.value: BkCorsChecker(),
+        PluginTypeCodeEnum.BK_HEADER_REWRITE.value: HeaderRewriteChecker(),
     }
 
     def __init__(self, type_code: str):

src/dashboard/apigateway/apigateway/apps/plugin/plugin/serializers.py

@@ -21,6 +21,7 @@
 from django.utils.translation import gettext as _
 from django.utils.translation import gettext_lazy
 from jsonschema import ValidationError as SchemaValidationError
+from jsonschema import validate
 from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
 from rest_framework.settings import api_settings
@@ -31,6 +32,7 @@
 from apigateway.apps.plugin.plugin.checker import PluginConfigYamlChecker
 from apigateway.apps.plugin.plugin.convertor import PluginConfigYamlConvertor
 from apigateway.common.fields import CurrentGatewayDefault
+from apigateway.controller.crds.release_data.plugin import PluginConvertorFactory
 
 
 class PluginConfigSLZ(serializers.ModelSerializer):
@@ -94,6 +96,12 @@ def _update_plugin(self, plugin: PluginConfig, validated_data: Dict[str, Any]):
 
         try:
             plugin.config = validated_data["yaml"]
+            # 转换数据, 校验apisix schema
+            schema = plugin.type and plugin.type.schema
+            if schema:
+                convertor = PluginConvertorFactory.get_convertor(plugin.type.code)
+                _data = convertor.convert(plugin)
+                validate(_data, schema=schema.schema)
         except SchemaValidationError as err:
             raise ValidationError(
                 {api_settings.NON_FIELD_ERRORS_KEY: f"{err.message}, path {list(err.absolute_path)}"}

src/dashboard/apigateway/apigateway/apps/stage/serializers.py

@@ -199,6 +199,9 @@ def create(self, validated_data):
             validated_data.get("rate_limit") or settings.DEFAULT_STAGE_RATE_LIMIT_CONFIG,
         )
 
+        # 创建header插件配置并绑定到stage
+        StageHandler().save_header_rewrite_plugin(instance, validated_data["proxy_http"].get("transform_headers", {}))
+
         # 3. record audit log
         StageHandler().add_create_audit_log(validated_data["api"], instance, validated_data.get("created_by", ""))
 
@@ -220,6 +223,9 @@ def update(self, instance, validated_data):
             validated_data.get("rate_limit"),
         )
 
+        # 更新header插件配置并绑定到stage
+        StageHandler().save_header_rewrite_plugin(instance, validated_data["proxy_http"].get("transform_headers", {}))
+
         # 3. send signal
         reversion_update_signal.send(sender=Stage, instance_id=instance.id, action="update")
 

src/dashboard/apigateway/apigateway/biz/stage.py

@@ -26,7 +26,9 @@
 from apigateway.apps.access_strategy.models import AccessStrategyBinding
 from apigateway.apps.audit.constants import OpObjectTypeEnum, OpStatusEnum, OpTypeEnum
 from apigateway.apps.audit.utils import record_audit_log
+from apigateway.apps.plugin.constants import PluginBindingScopeEnum
 from apigateway.common.contexts import StageProxyHTTPContext, StageRateLimitContext
+from apigateway.common.plugin.header_rewrite import HeaderRewriteConvertor
 from apigateway.core.constants import DEFAULT_STAGE_NAME, ContextScopeTypeEnum, StageStatusEnum
 from apigateway.core.models import Context, MicroGateway, Release, ReleaseHistory, Stage
 from apigateway.utils.time import now_datetime
@@ -157,3 +159,10 @@ def add_update_audit_log(gateway, stage, username: str):
             op_object=stage.name,
             comment=_("更新环境"),
         )
+
+    def save_header_rewrite_plugin(self, stage, transform_headers: dict):
+        # 生成header rewrite插件配置
+        plugin_config = HeaderRewriteConvertor.transform_headers_to_plugin_config(transform_headers)
+
+        # 创建header rewrite插件配置
+        HeaderRewriteConvertor.alter_plugin(stage.api, PluginBindingScopeEnum.STAGE.value, stage.id, plugin_config)

src/dashboard/apigateway/apigateway/common/plugin/__init__.py

@@ -0,0 +1,17 @@
+#
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+#

src/dashboard/apigateway/apigateway/common/plugin/header_rewrite.py

@@ -0,0 +1,108 @@
+#
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+#
+from typing import Optional
+
+from apigateway.apps.plugin.constants import PluginTypeCodeEnum
+from apigateway.apps.plugin.models import PluginBinding, PluginConfig, PluginType
+from apigateway.core.models import Gateway
+from apigateway.utils.yaml import yaml_dumps
+
+
+class HeaderRewriteConvertor:
+    @staticmethod
+    def transform_headers_to_plugin_config(transform_headers: dict) -> Optional[dict]:
+        # both set and delete empty
+        if not transform_headers or (
+            not transform_headers.get("set", None) and not transform_headers.get("delete", None)
+        ):
+            return None
+
+        return {
+            "set": [{"key": key, "value": value} for key, value in (transform_headers.get("set") or {}).items()],
+            "remove": [{"key": key} for key in transform_headers.get("delete") or []],
+        }
+
+    @staticmethod
+    def merge_plugin_config(stage_config: Optional[dict], resource_config: Optional[dict]) -> Optional[dict]:
+        if not stage_config and not resource_config:
+            return None
+
+        if not stage_config and resource_config:
+            return resource_config
+
+        if stage_config and not resource_config:
+            return stage_config
+
+        remove_keys = {item["key"] for item in resource_config["remove"]} | {  # type: ignore
+            item["key"] for item in stage_config["remove"]  # type: ignore
+        }
+        set_headers = {item["key"]: item["value"] for item in stage_config["set"]}  # type: ignore
+        set_headers.update({item["key"]: item["value"] for item in resource_config["set"]})  # type: ignore
+
+        return {
+            "set": [{"key": key, "value": value} for key, value in set_headers.items()],
+            "remove": [{"key": key} for key in remove_keys],
+        }
+
+    @staticmethod
+    def alter_plugin(gateway: Gateway, scope_type: str, scope_id: int, plugin_config: Optional[dict]):
+        # 1. 判断resource是否已经绑定header rewrite插件
+        binding = (
+            PluginBinding.objects.filter(
+                scope_type=scope_type,
+                scope_id=scope_id,
+                config__type__code=PluginTypeCodeEnum.BK_HEADER_REWRITE.value,
+            )
+            .prefetch_related("config")
+            .first()
+        )
+
+        # 2. 创建/更新/删除对应的插件与插件配置
+        # 插件配置为空, 清理数据
+        if binding and not plugin_config:
+            config = binding.config
+            # NOTE: 用bulk_delete避免触发信号
+            PluginBinding.objects.bulk_delete([binding])
+            PluginConfig.objects.bulk_delete([config])
+            return
+
+        # 如果没有绑定, 新建插件配置, 并绑定到stage
+        if not binding and plugin_config:
+            config = PluginConfig(
+                api=gateway,
+                name=f"{scope_type} [{scope_id}] header rewrite",
+                type=PluginType.objects.get(code=PluginTypeCodeEnum.BK_HEADER_REWRITE.value),
+                yaml=yaml_dumps(plugin_config),
+            )
+            config.save()
+            binding = PluginBinding(
+                api=gateway,
+                scope_type=scope_type,
+                scope_id=scope_id,
+                config=config,
+            )
+            # NOTE: 用bulk_create避免触发信号
+            PluginBinding.objects.bulk_create([binding])
+            return
+
+        # 如果已经绑定, 更新插件配置
+        if binding and plugin_config:
+            config = binding.config
+            config.yaml = yaml_dumps(plugin_config)
+            # NOTE: 用bulk_update避免触发信号
+            PluginConfig.objects.bulk_update([config], ["yaml"])