diff --git a/apps/backend/agent/artifact_builder/base.py b/apps/backend/agent/artifact_builder/base.py index 8bcb8e8c72..219236ab87 100644 --- a/apps/backend/agent/artifact_builder/base.py +++ b/apps/backend/agent/artifact_builder/base.py @@ -517,7 +517,7 @@ def update_or_create_support_files(self, package_infos: typing.List[typing.Dict] agent_name=self.NAME, ) - def update_or_create_package_records(self, v): + def update_or_create_package_records(self, package_infos): """ 创建或更新安装包记录，待 Agent 包管理完善 :param package_infos: diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index 4905c79726..5aecf495ff 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -361,6 +361,11 @@ def get_optional_items(cls) -> List[str]: IAM_ACTION_CHOICES = tuple_choices(IAM_ACTION_TUPLE) IamActionType = choices_to_namedtuple(IAM_ACTION_CHOICES) +GSE_PACKAGE_ENABLE_ALIAS_MAP = { + True: _("启用"), + False: _("禁用"), +} + class SubscriptionType: POLICY = "policy" @@ -1136,3 +1141,6 @@ class CommonExecutionSolutionStepType(EnhanceEnum): @classmethod def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.DEPENDENCIES: _("依赖文件"), cls.COMMANDS: _("命令")} + + +BUILT_IN_TAG_NAMES: List[str] = ["稳定版本", "最新版本", "测试版本"] diff --git a/apps/node_man/exceptions.py b/apps/node_man/exceptions.py index a1c7cb2a42..3fe1dc6e8d 100644 --- a/apps/node_man/exceptions.py +++ b/apps/node_man/exceptions.py @@ -220,3 +220,13 @@ class YunTiPolicyConfigNotExistsError(NodeManBaseException): MESSAGE = _("云梯策略配置不存在") MESSAGE_TPL = _("云梯策略配置不存在") ERROR_CODE = 43 + + +class FileDoesNotExistError(NodeManBaseException): + MESSAGE = _("文件不存在") + ERROR_CODE = 44 + + +class PluginParseError(NodeManBaseException): + MESSAGE = _("插件解析错误") + ERROR_CODE = 45 diff --git a/apps/node_man/handlers/meta.py b/apps/node_man/handlers/meta.py index 42fe4875f5..e335fc7fa1 100644 --- a/apps/node_man/handlers/meta.py +++ b/apps/node_man/handlers/meta.py @@ -20,6 +20,9 @@ from apps.node_man.handlers.cloud import CloudHandler from apps.node_man.handlers.cmdb import CmdbHandler from apps.node_man.handlers.install_channel import InstallChannelHandler +from apps.node_man.models import GsePackages +from apps.node_man.permissions.package_manage import PackageManagePermission +from apps.node_man.serializers.package_manage import FilterConditionPackageSerializer from apps.utils import APIModel @@ -471,28 +474,64 @@ def fetch_os_type_children(os_types: Tuple = constants.OsType): @staticmethod def fetch_agent_pkg_manager_children(): - mock_version = [ - {"name": "2.1.8", "id": "2.1.8"}, - {"name": "2.1.7", "id": "2.1.7"}, - ] - mock_tags = [ - {"name": "稳定版本", "id": "stable"}, - {"name": "最新版本", "id": "latest"}, - ] - mock_creator = [ - {"name": "user1", "id": "user1"}, - {"name": "user2", "id": "user2"}, - ] - mock_is_ready = [ - {"name": "启用", "id": True}, - {"name": "停用", "id": False}, - ] + if not PackageManagePermission().has_permission(None, None): + return {"message": "该用户不是管理员"} + + version_set, tag_name_description_map, creator_set, is_ready_set = set(), dict(), set(), set() + gse_packages = FilterConditionPackageSerializer(GsePackages.objects.all(), many=True).data + for gse_package in gse_packages: + version_set.add(gse_package.get("version")) + for parent_tag in gse_package.get("tags"): + for child_tag in parent_tag.get("children"): + tag_name_description_map[child_tag.get("name")] = child_tag.get("description") + creator_set.add(gse_package.get("created_by")) + is_ready_set.add(gse_package.get("is_ready")) return [ - {"name": _("版本号"), "id": "version", "children": mock_version}, - {"name": _("标签信息"), "id": "tags", "children": mock_tags}, - {"name": _("上传用户"), "id": "creator", "children": mock_creator}, - {"name": _("状态"), "id": "is_ready", "children": mock_is_ready}, + { + "name": _("版本号"), + "id": "version", + "children": [ + { + "id": version, + "name": version, + } + for version in version_set + ], + }, + { + "name": _("标签信息"), + "id": "tags", + "children": [ + { + "id": tag_name, + "name": tag_description, + } + for tag_name, tag_description in tag_name_description_map.items() + ], + }, + { + "name": _("上传用户"), + "id": "creator", + "children": [ + { + "id": creator_name, + "name": creator_name, + } + for creator_name in creator_set + ], + }, + { + "name": _("状态"), + "id": "is_ready", + "children": [ + { + "id": is_ready, + "name": constants.GSE_PACKAGE_ENABLE_ALIAS_MAP.get(is_ready, is_ready), + } + for is_ready in is_ready_set + ], + }, ] def filter_condition(self, category): diff --git a/apps/node_man/migrations/0078_gsepackagedesc_gsepackages.py b/apps/node_man/migrations/0078_gsepackagedesc_gsepackages.py new file mode 100644 index 0000000000..d42babdd59 --- /dev/null +++ b/apps/node_man/migrations/0078_gsepackagedesc_gsepackages.py @@ -0,0 +1,57 @@ +# Generated by Django 3.2.4 on 2023-11-10 09:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('node_man', '0077_auto_20231029_1336'), + ] + + operations = [ + migrations.CreateModel( + name='GsePackageDesc', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')), + ('created_by', models.CharField(default='', max_length=32, verbose_name='创建者')), + ('updated_time', models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间')), + ('updated_by', models.CharField(blank=True, default='', max_length=32, verbose_name='修改者')), + ('project', models.CharField(db_index=True, max_length=32, unique=True, verbose_name='工程名')), + ('description', models.TextField(verbose_name='安装包描述')), + ('description_en', models.TextField(blank=True, null=True, verbose_name='英文插件描述')), + ('category', models.CharField(choices=[('official', 'official'), ('external', 'external'), ('scripts', 'scripts')], max_length=32, verbose_name='所属范围')), + ], + options={ + 'verbose_name': 'Gse包描述（GsePackageDesc）', + 'verbose_name_plural': 'Gse包描述（GsePackageDesc）', + }, + ), + migrations.CreateModel( + name='GsePackages', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')), + ('created_by', models.CharField(default='', max_length=32, verbose_name='创建者')), + ('updated_time', models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间')), + ('updated_by', models.CharField(blank=True, default='', max_length=32, verbose_name='修改者')), + ('pkg_name', models.CharField(max_length=128, verbose_name='压缩包名')), + ('version', models.CharField(max_length=128, verbose_name='版本号')), + ('project', models.CharField(db_index=True, max_length=32, verbose_name='工程名')), + ('pkg_size', models.IntegerField(verbose_name='包大小')), + ('pkg_path', models.CharField(max_length=128, verbose_name='包路径')), + ('md5', models.CharField(max_length=32, verbose_name='md5值')), + ('location', models.CharField(max_length=512, verbose_name='安装包链接')), + ('os', models.CharField(choices=[('windows', 'windows'), ('linux', 'linux'), ('aix', 'aix'), ('solaris', 'solaris')], db_index=True, default='linux', max_length=32, verbose_name='系统类型')), + ('cpu_arch', models.CharField(choices=[('x86', 'x86'), ('x86_64', 'x86_64'), ('powerpc', 'powerpc'), ('aarch64', 'aarch64'), ('sparc', 'sparc')], db_index=True, default='x86_64', max_length=32, verbose_name='CPU类型')), + ('is_ready', models.BooleanField(default=True, verbose_name='插件是否可用')), + ('version_log', models.TextField(blank=True, null=True, verbose_name='版本日志')), + ('version_log_en', models.TextField(blank=True, null=True, verbose_name='英文版本日志')), + ], + options={ + 'verbose_name': 'Gse包（GsePackages）', + 'verbose_name_plural': 'Gse包（GsePackages）', + }, + ), + ] diff --git a/apps/node_man/serializers/package_manage.py b/apps/node_man/serializers/package_manage.py index 177f6c6445..a00933cb19 100644 --- a/apps/node_man/serializers/package_manage.py +++ b/apps/node_man/serializers/package_manage.py @@ -11,14 +11,21 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers +from apps.core.tag.models import Tag from apps.exceptions import ValidationError -from apps.node_man.constants import GsePackageCode +from apps.node_man.constants import BUILT_IN_TAG_NAMES, GsePackageCode +from apps.node_man.models import GsePackageDesc class TagsSerializer(serializers.Serializer): - id = serializers.CharField() name = serializers.CharField() - children = serializers.ListField() + description = serializers.CharField() + + +class ParentTagSerializer(serializers.Serializer): + name = serializers.CharField() + description = serializers.CharField() + children = TagsSerializer(many=True) class ConditionsSerializer(serializers.Serializer): @@ -26,18 +33,70 @@ class ConditionsSerializer(serializers.Serializer): values = serializers.ListField() -class PackageSerializer(serializers.Serializer): +class BasePackageSerializer(serializers.Serializer): + def get_tags(self, obj): + agent_project_ids = GsePackageDesc.objects.filter(project=obj.project).values_list("id", flat=True) + tags = Tag.objects.filter(target_id__in=agent_project_ids, target_version=obj.version).values_list( + "name", "description" + ) + + mock_built_in_tags, mock_custom_tags = self.split_builtin_tags_and_custom_tags(tags) + mock_data = [ + { + "name": "builtin", + "description": "内置标签", + "children": mock_built_in_tags, + }, + {"name": "custom", "description": "自定义标签", "children": mock_custom_tags}, + ] + self.filter_no_children_parent_tag(mock_data) + return ParentTagSerializer(mock_data, many=True).data + + @classmethod + def split_builtin_tags_and_custom_tags(cls, tags): + """将标签拆分为内置的和自定义的""" + built_in_tags, custom_tags = [], [] + for name, description in tags: + if name in BUILT_IN_TAG_NAMES: + built_in_tags.append({"name": name, "description": description}) + else: + custom_tags.append({"name": name, "description": description}) + + return built_in_tags, custom_tags + + @classmethod + def filter_no_children_parent_tag(cls, parent_tags): + for i in range(len(parent_tags) - 1, -1, -1): + if not parent_tags[i].get("children"): + parent_tags.pop(i) + + +class PackageSerializer(BasePackageSerializer): id = serializers.IntegerField() pkg_name = serializers.CharField() version = serializers.CharField() os = serializers.CharField() cpu_arch = serializers.CharField() - tags = TagsSerializer(many=True) - creator = serializers.CharField() - pkg_ctime = serializers.DateTimeField() + # tags = TagsSerializer(many=True) + tags = serializers.SerializerMethodField() + created_by = serializers.CharField() + created_time = serializers.DateTimeField() + is_ready = serializers.BooleanField() + + +class FilterConditionPackageSerializer(BasePackageSerializer): + version = serializers.CharField() + tags = serializers.SerializerMethodField() + created_by = serializers.CharField() is_ready = serializers.BooleanField() +class QuickFilterConditionPackageSerializer(BasePackageSerializer): + version = serializers.CharField() + os = serializers.CharField() + cpu_arch = serializers.CharField() + + class PackageDescSerializer(serializers.Serializer): id = serializers.IntegerField() version = serializers.CharField() @@ -59,6 +118,13 @@ class PackageDescResponseSerialiaer(serializers.Serializer): class OperateSerializer(serializers.Serializer): is_ready = serializers.BooleanField() + def update(self, instance, validated_data): + for attr, value in validated_data.items(): + setattr(instance, attr, value) + + instance.save() + return instance + class QuickSearchSerializer(serializers.Serializer): project = serializers.ChoiceField(choices=GsePackageCode.list_choices()) @@ -90,13 +156,20 @@ class ParseSerializer(serializers.Serializer): class ParseResponseSerializer(serializers.Serializer): class ParsePackageSerializer(serializers.Serializer): - module = serializers.ChoiceField(choices=["agent", "proxy"]) - pkg_name = serializers.CharField() - pkg_abs_path = serializers.CharField() - version = serializers.CharField() + project = serializers.ChoiceField(choices=["agent", "proxy"], required=False) + pkg_name = serializers.CharField(required=False) + pkg_abs_path = serializers.CharField(source="pkg_absolute_path") + version = serializers.CharField(required=False) os = serializers.CharField() cpu_arch = serializers.CharField() - config_templates = serializers.ListField() + config_templates = serializers.ListField(default=[]) + + def to_representation(self, instance): + data = super().to_representation(instance) + data["project"] = self.context.get("project", "") + data["pkg_name"] = self.context.get("pkg_name", "") + data["version"] = self.context.get("version", "") + return data description = serializers.CharField() packages = ParsePackageSerializer(many=True) diff --git a/apps/node_man/tests/test_views/test_package_manage_views.py b/apps/node_man/tests/test_views/test_package_manage_views.py new file mode 100644 index 0000000000..31eb65f445 --- /dev/null +++ b/apps/node_man/tests/test_views/test_package_manage_views.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from unittest.mock import patch + +from apps.backend.tests.subscription.agent_adapter.test_adapter import ( + Proxy2StepAdapterTestCase, +) +from apps.core.tag.models import Tag +from apps.node_man.constants import GSE_PACKAGE_ENABLE_ALIAS_MAP +from apps.node_man.handlers.meta import MetaHandler +from apps.node_man.models import GsePackageDesc, GsePackages +from apps.node_man.tests.utils import ( + create_gse_package, + update_or_create_package_records, +) + + +class PackageManageViewsTestCase(Proxy2StepAdapterTestCase): + @patch( + "apps.backend.agent.artifact_builder.base.BaseArtifactBuilder.update_or_create_package_records", + update_or_create_package_records, + ) + def setUp(self): + super().setUp() + with self.ARTIFACT_BUILDER_CLASS(initial_artifact_path=self.ARCHIVE_PATH, tags=["tag1", "tag2"]) as builder: + builder.make() + + gse_package = GsePackages.objects.first() + gse_package.created_at = "admin" + gse_package.save() + + @classmethod + def clear_agent_data(cls): + GsePackages.objects.all().delete() + GsePackageDesc.objects.all().delete() + Tag.objects.all().delete() + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_list(self, *args, **kwargs): + # 和之前的builder.make加起来100 + create_gse_package(99, start_id=1000, project="gse_proxy") + + result = self.client.get(path="/api/agent/package/", data={"project": "gse_proxy"}) + self.assertEqual(result["result"], True) + self.assertEqual(result["data"]["total"], 100) + self.assertEqual(len(result["data"]["list"]), 100) + + result = self.client.get(path="/api/agent/package/", data={"page": 1, "pagesize": 2, "project": "gse_proxy"}) + self.assertEqual(result["result"], True) + self.assertEqual(result["data"]["total"], 100) + self.assertEqual(len(result["data"]["list"]), 2) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_list_with_filter_condition(self, *args, **kwargs): + # 不筛选 + result = self.client.get(path="/api/agent/package/", data={"project": "gse_proxy"}) + self.assertEqual(result["data"]["total"], 1) + self.assertEqual(len(result["data"]["list"]), 1) + + gse_package = GsePackages.objects.first() + + # 筛选tags + result = self.client.get(path="/api/agent/package/", data={"tags": "tag1", "project": "gse_proxy"}) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertIn("tag1", self.collect_all_tag_names(result["data"]["list"][0]["tags"])) + result = self.client.get(path="/api/agent/package/", data={"tags": "tag2", "project": "gse_proxy"}) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertIn("tag2", self.collect_all_tag_names(result["data"]["list"][0]["tags"])) + result = self.client.get(path="/api/agent/package/", data={"tags": "tag3", "project": "gse_proxy"}) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选os_cpu_arch + os, cpu_arch = "linux", "x86_64" + result = self.client.get( + path="/api/agent/package/", + data={"os_cpu_arch": f"{gse_package.os}_{gse_package.cpu_arch}", "project": "gse_proxy"}, + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["os"], os) + self.assertEqual(result["data"]["list"][0]["cpu_arch"], cpu_arch) + result = self.client.get( + path="/api/agent/package/", data={"os_cpu_arch": "windows_x86_64", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选created_by + result = self.client.get( + path="/api/agent/package/", data={"created_by": gse_package.created_by, "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["created_by"], gse_package.created_by) + result = self.client.get(path="/api/agent/package/", data={"created_by": "system", "project": "gse_proxy"}) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选is_ready + result = self.client.get( + path="/api/agent/package/", data={"is_ready": str(gse_package.is_ready), "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["is_ready"], True) + result = self.client.get(path="/api/agent/package/", data={"is_ready": "false", "project": "gse_proxy"}) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选version + result = self.client.get( + path="/api/agent/package/", data={"version": gse_package.version, "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["version"], "1.0.1") + result = self.client.get(path="/api/agent/package/", data={"version": "1.0.2", "project": "gse_proxy"}) + self.assertEqual(len(result["data"]["list"]), 0) + + @classmethod + def collect_all_tag_names(cls, tags): + """ + tags: [ + { + "id": "builtin", + "name": "内置标签", + "children": [ + {"id": "stable", "name": "稳定版本", "children": []}, + {"id": "latest", "name": "最新版本", "children": []}, + ], + }, + { + "id": "custom", + "name": "自定义标签", + "children": [ + {"id": "custom", "name": "自定义版本", "children": []} + ] + }, + ] + """ + tag_name_set = set() + for parent_tag in tags: + for children_tag in parent_tag["children"]: + tag_name_set.add(children_tag["name"]) + + return tag_name_set + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_update(self, *args, **kwargs): + first_gse_package = GsePackages.objects.first() + self.assertEqual(first_gse_package.is_ready, True) + self.client.put( + path=f"/api/agent/package/{first_gse_package.id}/", data={"is_ready": False, "project": "gse_proxy"} + ) + self.assertEqual(GsePackages.objects.first().is_ready, False) + + # 测试更新不存在的id是否服务器异常 + self.client.put(path="/api/agent/package/10000/", data={"is_ready": False}) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_destroy(self, *args, **kwargs): + gse_packages = GsePackages.objects.all() + self.assertEqual(len(gse_packages), 1) + self.client.delete(path=f"/api/agent/package/{gse_packages.first().id}/", data={"project": "gse_proxy"}) + self.assertEqual(len(GsePackages.objects.all()), 0) + + # 测试删除存在的id是否服务器异常 + self.client.delete(path="/api/agent/package/10000/") + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_quick_search_condition(self, *args, **kwargs): + result = self.client.get(path="/api/agent/package/quick_search_condition/", data={"project": "gse_proxy"}) + for condition in result["data"]: + if condition["id"] == "os_cpu_arch": + self.assertCountEqual( + condition["children"], + [ + {"id": "ALL", "name": "ALL", "count": 1}, + {"id": "linux_x86_64", "name": "linux_x86_64", "count": 1}, + ], + ) + elif condition["id"] == "version": + self.assertCountEqual( + condition["children"], + [ + {"id": "ALL", "name": "ALL", "count": 1}, + {"id": "1.0.1", "name": "1.0.1", "count": 1}, + ], + ) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_filter_condition_with_agent_pkg_manage(self, *args, **kwargs): + result = MetaHandler().filter_condition("agent_pkg_manage") + self.assertEqual(len(GsePackages.objects.all()), 1) + gse_package = GsePackages.objects.first() + is_ready = gse_package.is_ready + for condition in result: + if condition["id"] == "version": + self.assertCountEqual( + condition["children"], + [ + {"id": gse_package.version, "name": gse_package.version}, + ], + ) + elif condition["id"] == "tags": + self.assertCountEqual( + condition["children"], + [ + {"id": "tag1", "name": ""}, + {"id": "tag2", "name": ""}, + ], + ) + elif condition["id"] == "creator": + self.assertCountEqual( + condition["children"], + [ + {"id": gse_package.created_by, "name": gse_package.created_by}, + ], + ) + elif condition["id"] == "is_ready": + self.assertCountEqual( + condition["children"], + [ + {"id": is_ready, "name": GSE_PACKAGE_ENABLE_ALIAS_MAP.get(is_ready, is_ready)}, + ], + ) diff --git a/apps/node_man/tests/utils.py b/apps/node_man/tests/utils.py index bab618de80..935ea98a66 100644 --- a/apps/node_man/tests/utils.py +++ b/apps/node_man/tests/utils.py @@ -16,6 +16,7 @@ from django.utils import timezone +from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP from apps.exceptions import ApiResultError, ComponentCallError from apps.mock_data import common_unit from apps.node_man import constants, tools @@ -27,6 +28,8 @@ from apps.node_man.models import ( AccessPoint, Cloud, + GsePackageDesc, + GsePackages, Host, IdentityData, InstallChannel, @@ -677,8 +680,9 @@ def create_ap(number): def create_job(number, id=None, end_time=None, bk_biz_scope=None, task_id_list=None, created_by=None): job_types = list(chain(*list(constants.JOB_TYPE_MAP.values()))) job_types = [ - job_type for job_type in job_types if - tools.JobTools.unzip_job_type(job_type)["op_type"] in constants.OP_TYPE_TUPLE + job_type + for job_type in job_types + if tools.JobTools.unzip_job_type(job_type)["op_type"] in constants.OP_TYPE_TUPLE ] if bk_biz_scope == {} or bk_biz_scope: @@ -1233,3 +1237,64 @@ def ret_to_validate_data(data): login_ip_info = HostHandler.get_host_infos_gby_ip_key(login_ips, constants.CmdbIpVersion.V4.value) return biz_info, data, cloud_info, ap_id_name, inner_ip_info, outer_ip_info, login_ip_info, bk_biz_scope + + +def create_gse_package( + number, + start_id=1, + version=None, + project=None, + pkg_size=None, + is_ready=None, + pkg_path=None, + location=None, + created_by=None, + os=None, + cpu_arch=None, +): + gse_packages = [] + for i in range(start_id, number + start_id): + gse_package = GsePackages( + id=i, + pkg_name=f"pkg_name{i+1}", + version=version or random.choice(["version1", "version2", "version3"]), + project=project or random.choice(["agent", "proxy"]), + pkg_size=pkg_size or random.randint(1, 2000), + pkg_path=pkg_path or "/tmp/", + md5="", + location=location or "", + os=os or random.choice(constants.OS_TUPLE), + cpu_arch=cpu_arch or random.choice(constants.CPU_TUPLE), + is_ready=is_ready or random.choice(list(constants.GSE_PACKAGE_ENABLE_ALIAS_MAP.keys())), + created_by=created_by or "admin", + updated_by=created_by or "admin", + version_log="", + version_log_en="", + ) + gse_packages.append(gse_package) + gse_packages = GsePackages.objects.bulk_create(gse_packages) + return [gse_package.id for gse_package in gse_packages] + + +def update_or_create_package_records(self, package_infos): + """ + 创建或更新安装包记录，待 Agent 包管理完善 + :param package_infos: + :return: + """ + for package_info in package_infos: + GsePackages.objects.update_or_create( + pkg_name=package_info["package_upload_info"]["pkg_name"], + version=package_info["artifact_meta_info"]["version"], + project=self.NAME, + pkg_size=package_info["package_upload_info"]["pkg_size"], + pkg_path=package_info["package_upload_info"]["pkg_path"], + md5=package_info["package_upload_info"]["md5"], + os=package_info["package_dir_info"]["os"], + cpu_arch=package_info["package_dir_info"]["cpu_arch"], + ) + + GsePackageDesc.objects.update_or_create( + id=AGENT_NAME_TARGET_ID_MAP[self.NAME], + project=self.NAME, + ) diff --git a/apps/node_man/utils/filters.py b/apps/node_man/utils/filters.py index 722985b0a4..2c90d3441f 100644 --- a/apps/node_man/utils/filters.py +++ b/apps/node_man/utils/filters.py @@ -8,22 +8,66 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - import django_filters from django_filters.rest_framework import FilterSet from apps.node_man.models import GsePackages +from apps.node_man.serializers import package_manage as pkg_manage class GsePackageFilter(FilterSet): - tags = django_filters.CharFilter(name="tags", method="filter_tags") - os_cpu_arch = django_filters.CharFilter(name="os_cpu_arch", method="filter_os_cpu_arch") + tags = django_filters.CharFilter(field_name="tags", method="filter_tags") + os_cpu_arch = django_filters.CharFilter(field_name="os_cpu_arch", method="filter_os_cpu_arch") + project = django_filters.CharFilter(field_name="project", method="filter_project") + created_by = django_filters.CharFilter(field_name="created_by", method="filter_created_by") + is_ready = django_filters.CharFilter(field_name="is_ready", method="filter_is_ready") + version = django_filters.CharFilter(field_name="version", method="filter_version") def filter_tags(self, queryset, name, value): - pass + tag_names = self.str_to_list(value) + package_ids = [ + gse_package.get("id") + for tag_name in tag_names + for gse_package in pkg_manage.PackageSerializer(queryset, many=True).data + if tag_name + in set( + children_tag.get("name") + for parent_tag in gse_package.get("tags") + for children_tag in parent_tag["children"] + ) + ] + + return queryset.filter(id__in=package_ids) def filter_os_cpu_arch(self, queryset, name, value): - pass + os_cpu_arch_list = self.str_to_list(value) + for os_cpu_arch in os_cpu_arch_list: + os_cpu_arch = os_cpu_arch.split("_", 1) + if len(os_cpu_arch) != 2: + continue + + os, cpu_arch = os_cpu_arch[0], os_cpu_arch[1] + queryset = queryset.filter(os=os, cpu_arch=cpu_arch) + return queryset + + def filter_project(self, queryset, name, value): + return queryset.filter(project__in=self.str_to_list(value)) + + def filter_created_by(self, queryset, name, value): + return queryset.filter(created_by__in=self.str_to_list(value)) + + def filter_is_ready(self, queryset, name, value): + return queryset.filter(is_ready__in=self.str_to_list(value, excepted_type="bool")) + + def filter_version(self, queryset, name, value): + return queryset.filter(version__in=self.str_to_list(value)) + + @classmethod + def str_to_list(cls, s, excepted_type="str"): + li = [tag_name.replace(" ", "").strip() for tag_name in s.split(",")] + if excepted_type == "bool": + li = [True if v.lower() == "true" else False for v in li] + return li class Meta: model = GsePackages diff --git a/apps/node_man/views/package_manage.py b/apps/node_man/views/package_manage.py index b4793495e7..34fef4cc70 100644 --- a/apps/node_man/views/package_manage.py +++ b/apps/node_man/views/package_manage.py @@ -8,6 +8,14 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import json +import logging +import tarfile +import typing +from collections import defaultdict + +from django.http import JsonResponse +from django.utils.translation import ugettext as _ from django_filters.rest_framework import DjangoFilterBackend from drf_yasg import openapi from rest_framework import filters @@ -15,8 +23,11 @@ from rest_framework.response import Response from rest_framework.status import HTTP_200_OK +from apps.backend.agent.artifact_builder import agent, base, proxy +from apps.core.files.storage import get_storage from apps.generic import ApiMixinModelViewSet as ModelViewSet -from apps.node_man import models +from apps.node_man import exceptions, models +from apps.node_man.handlers.plugin_v2 import PluginV2Handler from apps.node_man.permissions import package_manage as pkg_permission from apps.node_man.serializers import package_manage as pkg_manage from apps.node_man.utils.filters import GsePackageFilter @@ -24,6 +35,7 @@ PACKAGE_MANAGE_VIEW_TAGS = ["PKG_Manager"] PACKAGE_DES_VIEW_TAGS = ["PKG_Desc"] +logger = logging.getLogger("app") class PackageManageViewSet(ModelViewSet): @@ -35,41 +47,29 @@ class PackageManageViewSet(ModelViewSet): filter_backends = (DjangoFilterBackend, filters.SearchFilter, filters.OrderingFilter) filter_class = GsePackageFilter + def dispatch(self, request, *args, **kwargs): + project = "gse_agent" + if request.FILES: + pass + elif "project" in request.GET: + project = request.GET["project"] + elif request.body and "project" in json.loads(request.body): + project = json.loads(request.body)["project"] + self.queryset = self.queryset.filter(project=project) + return super().dispatch(request, *args, **kwargs) + @swagger_auto_schema( responses={200: pkg_manage.ListResponseSerializer}, operation_summary="安装包列表", tags=PACKAGE_MANAGE_VIEW_TAGS, ) def list(self, request, *args, **kwargs): - mock_data = { - "total": 2, - "list": [ - { - "id": 1, - "pkg_name": "pkg_name", - "version": "1.1.1", - "os": "Linux", - "cpu_arch": "x86_64", - "tags": [{"id": "stable", "name": "稳定版本"}], - "creator": "string", - "pkg_ctime": "2019-08-24 14:15:22", - "is_ready": True, - }, - { - "id": 2, - "pkg_name": "pkg_name", - "version": "1.1.2", - "os": "Linux", - "os_cpu_arch": "x86_64", - "tags": [{"id": "stable", "name": "稳定版本"}], - "creator": "string", - "pkg_ctime": "2019-08-24 14:15:22", - "is_ready": True, - }, - ], + queryset = self.filter_queryset(self.queryset) + res = { + "total": queryset.count(), + "list": self.paginate_queryset(queryset) if "pagesize" in request.query_params else queryset, } - return Response(mock_data) - # return super().list(request, *args, **kwargs) + return Response(pkg_manage.ListResponseSerializer(res).data) @swagger_auto_schema( operation_summary="操作类动作：启用/停用", @@ -78,27 +78,21 @@ def list(self, request, *args, **kwargs): tags=PACKAGE_MANAGE_VIEW_TAGS, ) def update(self, request, validated_data, *args, **kwargs): - mock_data = { - "id": 1, - "pkg_name": "pkg_name", - "version": "1.1.1", - "os": "Linux", - "cpu_arch": "x86_64", - "tags": [{"id": "stable", "name": "稳定版本"}], - "creator": "string", - "pkg_ctime": "2019-08-24 14:15:22", - "is_ready": True, - } + instance = self.get_object() + serializer = pkg_manage.OperateSerializer(instance, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + self.perform_update(serializer) - return Response(mock_data) + updated_instance = self.get_object() + + return Response(pkg_manage.PackageSerializer(updated_instance).data) @swagger_auto_schema( operation_summary="删除安装包", tags=PACKAGE_MANAGE_VIEW_TAGS, ) def destroy(self, request, *args, **kwargs): - - return Response() + return super(PackageManageViewSet, self).destroy(request, *args, **kwargs) @swagger_auto_schema( operation_summary="获取快速筛选信息", @@ -111,24 +105,52 @@ def destroy(self, request, *args, **kwargs): ) @action(detail=False, methods=["GET"]) def quick_search_condition(self, request, *args, **kwargs): - mock_version = [ - {"name": "2.1.8", "id": "2.1.8", "count": 10}, - {"name": "2.1.7", "id": "2.1.7", "count": 10}, - {"name": "ALL", "id": "all", "count": 20}, - ] + gse_packages = pkg_manage.QuickFilterConditionPackageSerializer(self.queryset, many=True).data + version_set, os_cpu_arch_set = set(), set() + version_count_map, os_cpu_arch_count_map = defaultdict(int), defaultdict(int) + if gse_packages: + for gse_package in gse_packages: + version, os, cpu_arch = gse_package.get("version"), gse_package.get("os"), gse_package.get("cpu_arch") + os_cpu_arch = f"{os}_{cpu_arch}" - mock_os_cpu_arch = [ - {"name": "Linux_x86_64", "id": "linux_x86_64", "count": 10}, - {"name": "Linux_x86", "id": "linux_x86", "count": 10}, - {"name": "ALL", "id": "all", "count": 20}, - ] + version_set.add(version) + os_cpu_arch_set.add(os_cpu_arch) - mock_data = [ - {"name": "操作系统/架构", "id": "os_cpu_arch", "children": mock_os_cpu_arch}, - {"name": "版本号", "id": "version", "children": mock_version}, - ] + version_count_map[version] += 1 + version_count_map["ALL"] += 1 + os_cpu_arch_count_map[os_cpu_arch] += 1 + os_cpu_arch_count_map["ALL"] += 1 + version_set.add("ALL") + os_cpu_arch_set.add("ALL") - return Response(mock_data) + return Response( + [ + { + "name": "操作系统/架构", + "id": "os_cpu_arch", + "children": [ + { + "id": os_cpu_arch, + "name": os_cpu_arch, + "count": os_cpu_arch_count_map[os_cpu_arch], + } + for os_cpu_arch in os_cpu_arch_set + ], + }, + { + "name": "版本号", + "id": "version", + "children": [ + { + "id": version, + "name": version, + "count": version_count_map[version], + } + for version in version_set + ], + }, + ] + ) @swagger_auto_schema( operation_summary="Agent包上传", @@ -137,13 +159,14 @@ def quick_search_condition(self, request, *args, **kwargs): ) @action(detail=False, methods=["POST"], serializer_class=pkg_manage.UploadSerializer) def upload(self, request): - # data = self.validated_data - mock_data = { - "id": 1, - "name": "gse_agent.tgz", - "pkg_size": 100, - } - return Response(mock_data) + ser = self.serializer_class(data=request.data) + ser.is_valid(raise_exception=True) + data = ser.validated_data + result = PluginV2Handler.upload(package_file=data["package_file"], module=data["module"]) + if "result" in result: + return JsonResponse(result) + else: + return Response(result) @swagger_auto_schema( operation_summary="解析Agent包", @@ -152,28 +175,40 @@ def upload(self, request): ) @action(detail=False, methods=["POST"], serializer_class=pkg_manage.ParseSerializer) def parse(self, request): - mock_data = { - "description": "test", - "packages": [ - { - "pkg_abs_path": "xxx/xxxxx", - "pkg_name": "gseagent_2.1.7_linux_x86_64.tgz", - "module": "agent", - "version": "2.1.7", - "config_templates": [], - "os": "x86_64", - }, - { - "pkg_abs_path": "xxx/xxxxx", - "pkg_name": "gseagent_2.1.7_linux_x86.tgz", - "module": "agent", - "version": "2.1.7", - "config_templates": [], - "os": "x86", - }, - ], + upload_package_obj = ( + models.UploadPackage.objects.filter(file_name=request.data["file_name"]).order_by("-upload_time").first() + ) + if upload_package_obj is None: + raise exceptions.FileDoesNotExistError(_("找不到请求发布的文件，请确认后重试")) + + file_path = upload_package_obj.file_path + + storage = get_storage() + if not storage.exists(name=file_path): + raise exceptions.PluginParseError(_(f"插件不存在: file_path -> {file_path}")) + + with storage.open(name=file_path, mode="rb") as tf_from_storage: + with tarfile.open(fileobj=tf_from_storage) as tf: + if "gse/server" in tf.getnames(): + project = "gse_proxy" + artifact_builder_class: typing.Type[base.BaseArtifactBuilder] = proxy.ProxyArtifactBuilder + else: + project = "gse_agent" + artifact_builder_class: typing.Type[base.BaseArtifactBuilder] = agent.AgentArtifactBuilder + + with artifact_builder_class(initial_artifact_path=upload_package_obj.file_path) as builder: + extract_dir, package_dir_infos = builder.list_package_dir_infos() + artifact_meta_info: typing.Dict[str, typing.Any] = builder.get_artifact_meta_info(extract_dir) + + res = {"description": artifact_meta_info.get("changelog"), "packages": package_dir_infos} + + context = { + "project": project, + "pkg_name": f"{artifact_meta_info['name']}-{artifact_meta_info['version']}.tgz", + "version": artifact_meta_info["version"], } - return Response(mock_data) + + return Response(pkg_manage.ParseResponseSerializer(res, context=context).data) @swagger_auto_schema( operation_summary="创建Agent包注册任务", @@ -209,16 +244,34 @@ def query_register_task(self, request, validated_data): @action(detail=False, methods=["GET"]) def tags(self, request): # 由tag handler 实现 + # mock_data = [ + # { + # "id": "builtin", + # "name": "内置标签", + # "children": [ + # {"id": "stable", "name": "稳定版本", "children": []}, + # {"id": "latest", "name": "最新版本", "children": []}, + # ], + # }, + # { + # "id": "custom", + # "name": "自定义标签", + # "children": [ + # {"id": "custom", "name": "自定义版本", "children": []} + # ] + # }, + # ] mock_data = [ { "id": "builtin", "name": "内置标签", - "children": [ - {"id": "stable", "name": "稳定版本", "children": []}, - {"id": "latest", "name": "最新版本", "children": []}, - ], + "children": [], + }, + { + "id": "custom", + "name": "自定义标签", + "children": [], }, - {"id": "custom", "name": "自定义标签", "children": [{"id": "custom", "name": "自定义版本", "children": []}]}, ] return Response(mock_data)