From c8888a526873c11eeaf2b61f49534f078fe8d323 Mon Sep 17 00:00:00 2001 From: LoRexxar Date: Mon, 27 Dec 2021 14:33:18 +0800 Subject: [PATCH 1/7] fix bug for load project list bug --- web/dashboard/controller/project.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/web/dashboard/controller/project.py b/web/dashboard/controller/project.py index 6cbf9281..45657d98 100644 --- a/web/dashboard/controller/project.py +++ b/web/dashboard/controller/project.py @@ -39,7 +39,17 @@ def get_context_data(self, **kwargs): rows = Project.objects.all().order_by('-id') project_count = Project.objects.all().count() - context['projects'] = rows + if 'p' in self.request.GET: + page = int(self.request.GET['p']) + else: + page = 1 + + # check page + if page*50 > project_count: + page = 1 + + context['projects'] = rows[(page-1)*50: page*50] + context['page'] = page for project in context['projects']: @@ -61,17 +71,7 @@ def get_context_data(self, **kwargs): context['projects'] = sorted(context['projects'], key=lambda x:x.last_scan_time)[::-1] - if 'p' in self.request.GET: - page = int(self.request.GET['p']) - else: - page = 1 - - # check page - if page*50 > project_count: - page = 1 - - context['projects'] = context['projects'][(page-1)*50: page*50] - context['page'] = page + # context['projects'] = context['projects'][(page-1)*50: page*50] max_page = project_count // 50 if project_count % 50 == 0 else (project_count // 50)+1 max_page = max_page+1 if max_page == 1 else max_page From 5447071b19f9679f8c1d8b53889f89c5ba125356 Mon Sep 17 00:00:00 2001 From: LoRexxar Date: Mon, 27 Dec 2021 17:34:06 +0800 Subject: [PATCH 2/7] update project search --- .../dashboard/projects/projects_list.html | 117 ++++++++++++------ web/dashboard/controller/project.py | 11 +- web/index/models.py | 27 ++++ 3 files changed, 112 insertions(+), 43 deletions(-) diff --git a/templates/dashboard/projects/projects_list.html b/templates/dashboard/projects/projects_list.html index 3344614c..4c808fae 100644 --- a/templates/dashboard/projects/projects_list.html +++ b/templates/dashboard/projects/projects_list.html @@ -3,50 +3,85 @@ {% block body %}
-
-
-
-

Projects List

-
- -
- - - - - - - - - - {% for project in projects %} - - - - - - - +
-
- {% endfor %} -
IDOriginLast Scan TimeTask CountVul CountVendor Count
{{ project.id }} {{ project.project_name }}{{ project.project_origin }}{{ project.last_scan_time }}{{ project.tasks_count }}{{ project.results_count }}{{ project.vendors_count }}
-
- +
-
-
    -
  • «
    • - {% for i in page_range %} -
  • {{ i }}
    • - {% endfor %} -
  • »
    • -
-
-
- -
+ +
+
+

Search Project

+
+
+ + +
+ + +
+
+
+ +
+ ps: use * to indicate wildcard, just like *test*. + +
+
+ +
+
+ +
+
+
+ +
+ +
+
+
+

Projects List

+
+ +
+ + + + + + + + + + {% for project in projects %} + + + + + + + + + + {% endfor %} +
IDOriginLast Scan TimeTask CountVul CountVendor Count
{{ project.id }} {{ project.project_name }}{{ project.project_origin }}{{ project.last_scan_time }}{{ project.tasks_count }}{{ project.results_count }}{{ project.vendors_count }}
+
+ + +
+
    +
  • «
    • + {% for i in page_range %} +
  • {{ i }}
    • + {% endfor %} +
  • »
    • +
+
+
+ +
+ {% endblock %} {% block script %} diff --git a/web/dashboard/controller/project.py b/web/dashboard/controller/project.py index 45657d98..df6c2100 100644 --- a/web/dashboard/controller/project.py +++ b/web/dashboard/controller/project.py @@ -26,7 +26,7 @@ from utils.utils import del_sensitive_for_config from web.index.models import ScanTask, ScanResultTask, Rules, Tampers, NewEvilFunc, Project, ProjectVendors, VendorVulns -from web.index.models import get_and_check_scanresult, get_and_check_evil_func +from web.index.models import search_project_by_name class ProjectListView(TemplateView): @@ -36,9 +36,15 @@ class ProjectListView(TemplateView): def get_context_data(self, **kwargs): context = super(ProjectListView, self).get_context_data(**kwargs) - rows = Project.objects.all().order_by('-id') + # 搜索项目 + search_project_name = "" + if "project_name" in self.request.GET: + search_project_name = self.request.GET['project_name'] + + rows = search_project_by_name(search_project_name) project_count = Project.objects.all().count() + # 分页 if 'p' in self.request.GET: page = int(self.request.GET['p']) else: @@ -78,6 +84,7 @@ def get_context_data(self, **kwargs): context['max_page'] = max_page context['page_range'] = range(int(max_page))[1:] + context['search_project_name'] = search_project_name return context diff --git a/web/index/models.py b/web/index/models.py index dc32acc5..f9859c0f 100644 --- a/web/index/models.py +++ b/web/index/models.py @@ -47,6 +47,33 @@ class Project(models.Model): project_hash = models.CharField(max_length=32) +def search_project_by_name(project_name): + """ + 支持*语法的查询 + :param project_name: + :return: + """ + if not project_name: + ps = Project.objects.all().order_by('-id') + return ps + + if project_name.startswith('*'): + if project_name.endswith('*'): + ps = Project.objects.filter(project_name__icontains=project_name.strip('*')).order_by('-id') + + else: + ps = Project.objects.filter(project_name__iendswith=project_name.strip('*')).order_by('-id') + + else: + if project_name.endswith('*'): + ps = Project.objects.filter(project_name__istartswith=project_name.strip('*')).order_by('-id') + + else: + ps = Project.objects.filter(project_name__iexact=project_name.strip('*')).order_by('-id') + + return ps + + class ProjectVendors(models.Model): project_id = models.IntegerField() name = models.CharField(max_length=200) From 74a367b51f235910125008c0422b27a433171891 Mon Sep 17 00:00:00 2001 From: LoRexxar Date: Tue, 28 Dec 2021 15:57:47 +0800 Subject: [PATCH 3/7] fix sca bug for stuck --- core/cli.py | 3 +++ web/index/models.py | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/cli.py b/core/cli.py index 793fff02..ce676649 100644 --- a/core/cli.py +++ b/core/cli.py @@ -143,6 +143,9 @@ def display_result(scan_id, is_ask=False): logger.info("[Chain] {}, {}, {}:{}".format(rf.node_type, rf.node_content, rf.node_path, rf.node_lineno)) try: + if author == 'SCA': + continue + if not show_context(rf.node_path, rf.node_lineno): logger_console.info(rf.node_source) except: diff --git a/web/index/models.py b/web/index/models.py index f9859c0f..783d3aad 100644 --- a/web/index/models.py +++ b/web/index/models.py @@ -90,14 +90,13 @@ def save(self, *args, **kwargs): def update_and_new_project_vendor(project_id, name, version, language, ext=None): hash = md5("{},{},{}".format(project_id, name, language)) - vendor = ProjectVendors.objects.filter(hash=hash, project_id=project_id).first() + vendor = ProjectVendors.objects.filter(hash=hash, project_id=project_id, ext=ext).first() if vendor: - if vendor.version != version or vendor.ext != ext: + if vendor.version != version: logger.debug("[Vendors] Component {} update to version {}".format(name, version)) vendor.version = version - vendor.ext = ext try: vendor.save() except IntegrityError: From 201544548db741f012832b440fc4217414bc4d8e Mon Sep 17 00:00:00 2001 From: LoRexxar Date: Wed, 29 Dec 2021 18:36:09 +0800 Subject: [PATCH 4/7] update sca pom.xml search --- core/__init__.py | 4 +- core/__version__.py | 5 +- core/vendors.py | 48 +++++++++++++++---- .../dashboard/projects/project_detail.html | 2 + .../migrations/0009_projectvendors_source.py | 18 +++++++ web/index/models.py | 33 +++++++++---- 6 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 web/index/migrations/0009_projectvendors_source.py diff --git a/core/__init__.py b/core/__init__.py index df0bc3ff..6f4340cd 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -32,7 +32,7 @@ from .__version__ import __title__, __introduction__, __url__, __version__ from .__version__ import __author__, __author_email__, __license__ -from .__version__ import __copyright__, __epilog__, __scan_epilog__ +from .__version__ import __copyright__, __epilog__, __scan_epilog__, __database_epilog__ from core.rule import RuleCheck, TamperCheck from core.console import KunlunInterpreter @@ -65,7 +65,7 @@ def main(): parser_group_init.add_argument('migrationname', default='migrationname', nargs='?', help='Check migration name') # load config into database - parser_group_core = subparsers.add_parser('config', help='config for rule&tamper', description=__introduction__.format(detail='config for rule&tamper'), formatter_class=argparse.RawDescriptionHelpFormatter, usage=argparse.SUPPRESS, add_help=True) + parser_group_core = subparsers.add_parser('config', help='config for rule&tamper', description=__introduction__.format(detail='config for rule&tamper'), epilog=__database_epilog__, formatter_class=argparse.RawDescriptionHelpFormatter, usage=argparse.SUPPRESS, add_help=True) parser_group_core.add_argument('load', choices=['load', 'recover', 'loadtamper', 'retamper'], default=False, help='operate for rule&tamper') parser_group_scan = subparsers.add_parser('scan', help='scan target path', description=__introduction__.format(detail='scan target path'), epilog=__scan_epilog__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=True) diff --git a/core/__version__.py b/core/__version__.py index 28efe30c..a8af3c69 100644 --- a/core/__version__.py +++ b/core/__version__.py @@ -45,4 +45,7 @@ python {m} scan -t {td} --lan php -b vendor --debug python {m} scan -t {td} --lan php -tp roundcube -d -uc -""".format(m='kunlun.py', td='tests/vulnerabilities') \ No newline at end of file +""".format(m='kunlun.py', td='tests/vulnerabilities') +__database_epilog__ = """Usage: + python {m} init initialize +""".format(m='kunlun.py') diff --git a/core/vendors.py b/core/vendors.py index 413dc302..af73f886 100644 --- a/core/vendors.py +++ b/core/vendors.py @@ -279,15 +279,17 @@ def check_vendor(self): vendor_version = vendor[-1].strip() if len(vendor) < 2: vendor_version = None + ext = "php" update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, - language=language, ext=savefilepath) + language=language, source=savefilepath, ext=ext) get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language) elif filename == 'composer.json': vendors = json.loads(filecontent) vendors_list = [] + ext = "php" if not len(vendors): continue @@ -300,7 +302,7 @@ def check_vendor(self): vendor_version = vendors_list[vendor].strip() update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, - language=language, ext=savefilepath) + language=language, source=savefilepath, ext=ext) get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language) @@ -327,15 +329,17 @@ def check_vendor(self): vendor_name = vendor[0].strip() vendor_version = vendor[-1].strip() + ext = go_version update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, - language=language, ext=savefilepath) + language=language, source=savefilepath, ext=ext) get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language) elif filename == 'pom.xml': reg = r'xmlns="([\w\.\\/:]+)"' pom_ns = None + ext = "" if re.search(reg, filecontent, re.I): @@ -345,13 +349,26 @@ def check_vendor(self): for match in matchs: pom_ns = match.group(1) + tree = self.parse_xml(filepath) + root = tree.getroot() + + # 匹配default + if pom_ns: + default_xpath_reg = ".//{%s}parent" % pom_ns + else: + default_xpath_reg = ".//parent" + + parents = root.findall(default_xpath_reg) + default_version = "lastest" + for parent in parents: + default_version = parent.getchildren()[2].text + + # 匹配dependency if pom_ns: xpath_reg = ".//{%s}dependency" % pom_ns else: xpath_reg = ".//dependency" - tree = self.parse_xml(filepath) - root = tree.getroot() childs = root.findall(xpath_reg) for child in childs: group_id = child.getchildren()[0].text @@ -359,7 +376,7 @@ def check_vendor(self): if len(child.getchildren()) > 2: version = child.getchildren()[2].text else: - version = 'latest' + version = default_version var_reg = "\${([\w\.\_-]+)}" if re.search(var_reg, version, re.I): @@ -368,6 +385,12 @@ def check_vendor(self): for match in matchs: varname = match.group(1) + + # 处理内置变量 + if varname == "project.version": + version = default_version + continue + if pom_ns: var_xpath_reg = ".//{%s}%s" % (pom_ns, varname) else: @@ -377,13 +400,20 @@ def check_vendor(self): for child in varchilds: version = child.text + ext = varname + + # 如果没有匹配到，那么需要去数据库查询 + if not varchilds: + pv = ProjectVendors.objects.filter(project_id=self.project_id, ext=varname).first() + if pv: + version = pv.version vendor_name = "{}:{}".format(group_id, artifact_id) vendor_version = version - ext = "maven" + # ext = "maven" update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, - language=language, ext=savefilepath) + language=language, source=savefilepath, ext=ext) get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language, ext) @@ -417,7 +447,7 @@ def check_vendor(self): if vendor_name and vendor_version: update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, - language=language, ext=savefilepath) + language=language, source=savefilepath, ext=ext) get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language, ext) continue diff --git a/templates/dashboard/projects/project_detail.html b/templates/dashboard/projects/project_detail.html index 0404e879..9ffddf71 100644 --- a/templates/dashboard/projects/project_detail.html +++ b/templates/dashboard/projects/project_detail.html @@ -163,6 +163,7 @@

dependencies Vendors

Version Language Remark + Ext {% for project_vendor in project_vendors %} @@ -170,6 +171,7 @@

dependencies Vendors

{{ project_vendor.name }} {{ project_vendor.version }} {{ project_vendor.language }} + {{ project_vendor.source }} {{ project_vendor.ext }} diff --git a/web/index/migrations/0009_projectvendors_source.py b/web/index/migrations/0009_projectvendors_source.py new file mode 100644 index 00000000..91ce5283 --- /dev/null +++ b/web/index/migrations/0009_projectvendors_source.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.7 on 2021-12-28 10:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('index', '0008_auto_20210816_1131'), + ] + + operations = [ + migrations.AddField( + model_name='projectvendors', + name='source', + field=models.CharField(default=None, max_length=100, null=True), + ), + ] diff --git a/web/index/models.py b/web/index/models.py index 783d3aad..4ae3089a 100644 --- a/web/index/models.py +++ b/web/index/models.py @@ -79,6 +79,7 @@ class ProjectVendors(models.Model): name = models.CharField(max_length=200) version = models.CharField(max_length=50, null=True) language = models.CharField(max_length=20) + source = models.CharField(max_length=100, null=True, default=None) ext = models.CharField(max_length=100, null=True, default=None) hash = models.CharField(max_length=32) @@ -88,23 +89,39 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) -def update_and_new_project_vendor(project_id, name, version, language, ext=None): - hash = md5("{},{},{}".format(project_id, name, language)) - vendor = ProjectVendors.objects.filter(hash=hash, project_id=project_id, ext=ext).first() +def update_and_new_project_vendor(project_id, name, version, language, source=None, ext=None): + hash = md5("{},{},{},{}".format(project_id, name, language, source)) + vendor = ProjectVendors.objects.filter(project_id=project_id, name=name, language=language).first() if vendor: - if vendor.version != version: - logger.debug("[Vendors] Component {} update to version {}".format(name, version)) - + # 兼容性处理，如果source未指定，先更新source进去 + if not vendor.source: + vendor.source = source + vendor.ext = ext vendor.version = version + vendor.hash = hash + try: vendor.save() except IntegrityError: logger.warn("[Model Save] vendor model not changed") else: - v = ProjectVendors(project_id=project_id, name=name, version=version, language=language, ext=ext) - v.save() + vendor = ProjectVendors.objects.filter(project_id=project_id, hash=hash).first() + + if vendor: + if vendor.version != version: + logger.debug("[Vendors] Component {} update to version {}".format(name, version)) + + vendor.version = version + try: + vendor.save() + except IntegrityError: + logger.warn("[Model Save] vendor model not changed") + + else: + v = ProjectVendors(project_id=project_id, name=name, version=version, language=language, ext=ext) + v.save() return True From 1ad0ed75ee55197b84a9d55e9ccc6f1af1c5e6bc Mon Sep 17 00:00:00 2001 From: LoRexxar Date: Thu, 30 Dec 2021 16:13:28 +0800 Subject: [PATCH 5/7] new vendor list --- core/vendors.py | 22 +++++++++++++++++++++- web/index/models.py | 23 ++++++++++++----------- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/core/vendors.py b/core/vendors.py index af73f886..856e69c0 100644 --- a/core/vendors.py +++ b/core/vendors.py @@ -193,6 +193,9 @@ def __init__(self, task_id, project_id, target, files): self.ext_list = [] self.exist_file_list = [] + # java temp vendor list + self.java_temp_vendor_list = {} + for lan in VENDOR_FILE_DICT: self.vendor_file_list.extend(VENDOR_FILE_DICT[lan]) @@ -363,6 +366,19 @@ def check_vendor(self): for parent in parents: default_version = parent.getchildren()[2].text + # 匹配通用配置 + if pom_ns: + java_base_xpath_reg = ".//{%s}properties" % pom_ns + else: + java_base_xpath_reg = ".//properties" + + base_tags = root.findall(java_base_xpath_reg) + + if base_tags: + btags = base_tags[0].getchildren() + for btag in btags: + self.java_temp_vendor_list[btag.tag.replace("{%s}" % pom_ns, "")] = btag.text + # 匹配dependency if pom_ns: xpath_reg = ".//{%s}dependency" % pom_ns @@ -373,7 +389,7 @@ def check_vendor(self): for child in childs: group_id = child.getchildren()[0].text artifact_id = child.getchildren()[1].text - if len(child.getchildren()) > 2: + if len(child.getchildren()) > 2 and "version" in child.getchildren()[2].tag: version = child.getchildren()[2].text else: version = default_version @@ -391,6 +407,10 @@ def check_vendor(self): version = default_version continue + if varname in self.java_temp_vendor_list: + version = self.java_temp_vendor_list[varname] + continue + if pom_ns: var_xpath_reg = ".//{%s}%s" % (pom_ns, varname) else: diff --git a/web/index/models.py b/web/index/models.py index 4ae3089a..e3f94db2 100644 --- a/web/index/models.py +++ b/web/index/models.py @@ -96,6 +96,7 @@ def update_and_new_project_vendor(project_id, name, version, language, source=No if vendor: # 兼容性处理，如果source未指定，先更新source进去 if not vendor.source: + vendor.version = version vendor.source = source vendor.ext = ext vendor.version = version @@ -109,19 +110,19 @@ def update_and_new_project_vendor(project_id, name, version, language, source=No else: vendor = ProjectVendors.objects.filter(project_id=project_id, hash=hash).first() - if vendor: - if vendor.version != version: - logger.debug("[Vendors] Component {} update to version {}".format(name, version)) + if vendor: + if vendor.version != version: + logger.debug("[Vendors] Component {} update to version {}".format(name, version)) - vendor.version = version - try: - vendor.save() - except IntegrityError: - logger.warn("[Model Save] vendor model not changed") + vendor.version = version + try: + vendor.save() + except IntegrityError: + logger.warn("[Model Save] vendor model not changed") - else: - v = ProjectVendors(project_id=project_id, name=name, version=version, language=language, ext=ext) - v.save() + else: + v = ProjectVendors(project_id=project_id, name=name, version=version, language=language, ext=ext) + v.save() return True From d645c9c26e5c7b85358eaff8b221b75b56fae52e Mon Sep 17 00:00:00 2001 From: LoRexxar Date: Thu, 30 Dec 2021 16:25:44 +0800 Subject: [PATCH 6/7] update vendor display --- templates/dashboard/vendors/vendor_detail.html | 2 ++ templates/dashboard/vendors/vendors_list.html | 2 ++ 2 files changed, 4 insertions(+) diff --git a/templates/dashboard/vendors/vendor_detail.html b/templates/dashboard/vendors/vendor_detail.html index f43cf56d..93bed6a4 100644 --- a/templates/dashboard/vendors/vendor_detail.html +++ b/templates/dashboard/vendors/vendor_detail.html @@ -18,6 +18,7 @@

Search Result for {{ vendor_name }}

Name Version Language + Source Ext {% for vendor in vendors %} @@ -28,6 +29,7 @@

Search Result for {{ vendor_name }}

{{ vendor.name }} {{ vendor.version }} {{ vendor.language }} + {{ vendor.source }} {{ vendor.ext }} {% endfor %} diff --git a/templates/dashboard/vendors/vendors_list.html b/templates/dashboard/vendors/vendors_list.html index 0f2d3801..09c00e7b 100644 --- a/templates/dashboard/vendors/vendors_list.html +++ b/templates/dashboard/vendors/vendors_list.html @@ -52,6 +52,7 @@

Vendors List

Name Version Language + Source Ext {% for vendor in vendors %} @@ -62,6 +63,7 @@

Vendors List

{{ vendor.name }} {{ vendor.version }} {{ vendor.language }} + {{ vendor.source }} {{ vendor.ext }} {% endfor %} From 3cdde8e0b5d42d04a0a9f1285260865f924ba220 Mon Sep 17 00:00:00 2001 From: LoRexxar Date: Thu, 30 Dec 2021 16:53:49 +0800 Subject: [PATCH 7/7] update 2.6.4.1 --- core/__version__.py | 3 ++- docs/changelog.md | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/__version__.py b/core/__version__.py index a8af3c69..20a5ec11 100644 --- a/core/__version__.py +++ b/core/__version__.py @@ -7,7 +7,7 @@ __issue_page__ = 'https://github.com/LoRexxar/Kunlun-M/issues/new' __python_version__ = sys.version.split()[0] __platform__ = platform.platform() -__version__ = '2.6.4' +__version__ = '2.6.4.1' __author__ = 'LoRexxar' __author_email__ = 'LoRexxar@gmail.com' __license__ = 'MIT License' @@ -48,4 +48,5 @@ """.format(m='kunlun.py', td='tests/vulnerabilities') __database_epilog__ = """Usage: python {m} init initialize + python {m} init checksql index 0009_projectvendors_source """.format(m='kunlun.py') diff --git a/docs/changelog.md b/docs/changelog.md index 6591d9fd..08fd9a06 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -287,4 +287,11 @@ - 修复了墨非api的部分使用问题 - 删除了tasklog中无意义的数据显示，优化使用体验 - 在组件数据中加入数据来源路径便于检查 - - 修复了部分bug#197 #199 #200 \ No newline at end of file + - 修复了部分bug#197 #199 #200 +- 2021-12-30 + - KunLun-M 2.6.4.1 + - 修复了一部分组件数据的解析bug，maven的数据更精确了 + - 为组件数据添加了source字段，标准了组件的来源位置 + - 更新了相应的前端显示 + - 为项目页面做了数据优化，现在不那么烧资源了，并添加了项目搜索功能 + \ No newline at end of file