diff --git a/.travis.yml b/.travis.yml index 1f561654..70a8136a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "3.5" - "3.6" - "3.7" - "nightly" diff --git a/README.md b/README.md index b8714fad..34a3196c 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -**写在最前，Cobra-W仍然处于测试开发阶段，未发布正式版本，谨慎应用...** +**写在最前，Cobra-W就像手中的一把剑，这把剑好不好用是Cobra-W的事，如何使用是你的事，希望能有更多的人参与到Cobra-W的变化中来...** -**请使用python3运行该工具，已停止维护python2.7环境** +**请使用python3.6+运行该工具，已停止维护python2.7环境** # Cobra-W [![GitHub (pre-)release](https://img.shields.io/github/release/LoRexxar/Cobra-W/all.svg)](https://github.com/LoRexxar/Cobra-W/releases) [![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/wufeifei/cobra/blob/master/LICENSE) [![Build Status](https://travis-ci.org/LoRexxar/Cobra-W.svg?branch=master)](https://travis-ci.org/LoRexxar/Cobra-W) -[![Coverage Status](https://coveralls.io/repos/github/LoRexxar/Cobra-W/badge.svg?branch=master)](https://coveralls.io/github/LoRexxar/Cobra-W?branch=master) +![](https://img.shields.io/badge/language-python3.7-orange.svg) ## Introduction Cobra是一款**源代码安全审计**工具，支持检测多种开发语言源代码中的**大部分显著**的安全问题和漏洞。 diff --git a/cobra/__version__.py b/cobra/__version__.py index abf4c611..9621888b 100644 --- a/cobra/__version__.py +++ b/cobra/__version__.py @@ -7,7 +7,7 @@ __issue_page__ = 'https://github.com/LoRexxar/Cobra-W/issues/new' __python_version__ = sys.version.split()[0] __platform__ = platform.platform() -__version__ = '1.8.2' +__version__ = '1.9.0' __author__ = 'LoRexxar' __author_email__ = 'LoRexxar@gmail.com' __license__ = 'MIT License' diff --git a/cobra/cli.py b/cobra/cli.py index ec789bc8..4a3a0ca7 100644 --- a/cobra/cli.py +++ b/cobra/cli.py @@ -96,7 +96,7 @@ def start(target, formatter, output, special_rules, a_sid=None, language=None, s # Pretreatment ast object ast_object.init_pre(target_directory, files) - ast_object.pre_ast(main_language) + ast_object.pre_ast_all(main_language) # scan scan(target_directory=target_directory, a_sid=a_sid, s_sid=s_sid, special_rules=pa.special_rules, diff --git a/cobra/engine.py b/cobra/engine.py index a5cdc314..a06c3e79 100644 --- a/cobra/engine.py +++ b/cobra/engine.py @@ -14,7 +14,9 @@ import json import os import re +import asyncio import traceback +import functools import portalocker from phply import phpast as php @@ -167,11 +169,20 @@ def store(result): else: logger.debug('[SCAN] [STORE] Not found vulnerabilities on this rule!') + async def start_scan(target_directory, rule, files, language, secret_name): + + result = scan_single(target_directory, rule, files, language, secret_name) + store(result) + if len(rules) == 0: logger.critical('no rules!') return False logger.info('[PUSH] {rc} Rules'.format(rc=len(rules))) push_rules = [] + scan_list = [] + + # async start task + loop = asyncio.get_event_loop() for idx, single_rule in enumerate(sorted(rules.keys())): @@ -189,8 +200,12 @@ def store(result): vulnerability=rule.vulnerability, language=rule.language )) - result = scan_single(target_directory, rule, files, language, secret_name) - store(result) + # result = scan_single(target_directory, rule, files, language, secret_name) + scan_list.append(start_scan(target_directory, rule, files, language, secret_name)) + # store(result) + + loop.run_until_complete(asyncio.gather(*scan_list)) + loop.close() # print data = [] diff --git a/cobra/pretreatment.py b/cobra/pretreatment.py index 26cbb107..dc89bf69 100644 --- a/cobra/pretreatment.py +++ b/cobra/pretreatment.py @@ -22,6 +22,8 @@ import codecs import traceback import zipfile +import queue +import asyncio could_ast_pase_lans = ["php", "chromeext", "javascript"] @@ -76,12 +78,13 @@ class Pretreatment: def __init__(self): self.file_list = [] + self.target_queue = queue.Queue() self.target_directory = "" self.pre_result = {} self.define_dict = {} - self.pre_ast() + self.pre_ast_all() def init_pre(self, target_directory, files): self.file_list = files @@ -99,7 +102,7 @@ def get_path(self, filepath): else: return os.path.join(self.target_directory, filepath) - def pre_ast(self, lan=None): + def pre_ast_all(self, lan=None): if lan is not None: # 检查是否在可ast pasre列表中 @@ -109,6 +112,17 @@ def pre_ast(self, lan=None): return True for fileext in self.file_list: + self.target_queue.put(fileext) + + loop = asyncio.get_event_loop() + scan_list = (self.pre_ast() for i in range(20)) + loop.run_until_complete(asyncio.gather(*scan_list)) + + async def pre_ast(self): + + while not self.target_queue.empty(): + + fileext = self.target_queue.get() if fileext[0] in ext_dict['php']: # 下面是对于php文件的处理逻辑 @@ -145,7 +159,8 @@ def pre_ast(self, lan=None): for node in all_nodes: if isinstance(node, php.FunctionCall) and node.name == "define": define_params = node.params - logger.debug("[AST][Pretreatment] new define {}={}".format(define_params[0].node, define_params[1].node)) + logger.debug( + "[AST][Pretreatment] new define {}={}".format(define_params[0].node, define_params[1].node)) self.define_dict[define_params[0].node] = define_params[1].node @@ -168,7 +183,7 @@ def pre_ast(self, lan=None): # target可能是单个文件，这里需要专门处理 if not self.target_directory.endswith("/") and not self.target_directory.endswith("\\"): - relative_path = os.path.join(re.split(r'[\\|/]', self.target_directory)[-1]+"_files") + relative_path = os.path.join(re.split(r'[\\|/]', self.target_directory)[-1] + "_files") else: relative_path = target_files_path.split(self.target_directory)[-1] @@ -212,7 +227,6 @@ def pre_ast(self, lan=None): new_filepath = filepath + ".pretty" if not os.path.isfile(new_filepath): - fi2 = codecs.open(new_filepath, "w", encoding='utf-8', errors='ignore') code_content = jsbeautifier.beautify(code_content) fi2.write(code_content) @@ -235,6 +249,9 @@ def pre_ast(self, lan=None): except: logger.warning('[AST] something error, {}'.format(traceback.format_exc())) + return True + + def get_nodes(self, filepath, vul_lineno=None, lan=None): filepath = os.path.normpath(filepath) diff --git a/docs/changelog.md b/docs/changelog.md index 0c6ff042..d62b9f4e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -142,4 +142,9 @@ - 2019-10-23 - Cobra-W 1.8.2 - 修复了针对chrome 插件扫描时无法扫描js的bug - - 添加了js美化代码模块以便于更好的扫描以及人工确认 \ No newline at end of file + - 添加了js美化代码模块以便于更好的扫描以及人工确认 +- 2019-11-04 + - Cobra-W 1.9.0 + - 将Python版本限制升级到3.6并添加多处异步优化 + - 现在工具整体扫描效率优化了3倍左右 + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e5f8e0fb..015e4b3e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ portalocker==1.1.0 prettytable==0.7.2 rarfile==2.7 requests>=2.20.0 -Werkzeug==0.11.9 +Werkzeug==0.16.0 esprima==4.0.1 jsbeautifier==1.10.2 \ No newline at end of file diff --git a/tests/test_parser.py b/tests/test_parser.py index e700facf..14dfccce 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -19,7 +19,7 @@ files = [('.php', {'list': ["v_parser.php", "v.php"]})] ast_object.init_pre(project_directory + '/tests/vulnerabilities/', files) -ast_object.pre_ast() +ast_object.pre_ast_all() target_projects = project_directory + '/tests/vulnerabilities/v_parser.php'