From 1bf5747810f8b69dd96096d801d7801267b5fbf1 Mon Sep 17 00:00:00 2001 From: Victor Gabriel Leandro Alves Date: Tue, 24 Nov 2015 16:31:07 -0200 Subject: [PATCH 1/2] Pull Request: Python 3.4.3 port. --- metatrader/backtest.py | 35 +++++------ metatrader/mt4.py | 39 +++++++----- metatrader/report.py | 140 +++++++++++++++++++++-------------------- 3 files changed, 113 insertions(+), 101 deletions(-) diff --git a/metatrader/backtest.py b/metatrader/backtest.py index 1a631f6..d12675a 100644 --- a/metatrader/backtest.py +++ b/metatrader/backtest.py @@ -4,11 +4,13 @@ @author: samuraitaiga ''' -import logging +from __future__ import absolute_import, division import os -from mt4 import get_mt4 -from mt4 import DEFAULT_MT4_NAME -from __builtin__ import str +from metatrader.mt4 import get_mt4 +from metatrader.mt4 import DEFAULT_MT4_NAME +from metatrader.report import BacktestReport +from metatrader.report import OptimizationReport + class BackTest(object): """ @@ -18,7 +20,7 @@ class BackTest(object): symbol(string): currency symbol. e.g.: USDJPY from_date(datetime.datetime): backtest from date to_date(datetime.datetime): backtest to date - model(int): backtest model + model(int): backtest model 0: Every tick 1: Control points 2: Open prices only @@ -27,11 +29,11 @@ class BackTest(object): replace_report(bool): replace report flag. replace report is enabled if True """ + def __init__(self, ea_name, param, symbol, period, from_date, to_date, model=0, spread=5, replace_repot=True): self.ea_name = ea_name self.param = param self.symbol = symbol - self.period = period self.from_date = from_date self.to_date = to_date self.model = model @@ -51,8 +53,8 @@ def _create_conf(self, alias=DEFAULT_MT4_NAME): Notes: create config file(.conf) which is used parameter of terminal.exe in %APPDATA%\\MetaQuotes\\Terminal\\\\tester - - file contents goes to + + file contents goes to TestExpert=SampleEA TestExpertParameters=SampleEA.set TestSymbol=USDJPY @@ -80,7 +82,6 @@ def _create_conf(self, alias=DEFAULT_MT4_NAME): fp.write('TestExpertParameters=%s.set\n' % self.ea_name) fp.write('TestSymbol=%s\n' % self.symbol) fp.write('TestModel=%s\n' % self.model) - fp.write('TestPeriod=%s\n' % self.period) fp.write('TestSpread=%s\n' % self.spread) fp.write('TestOptimization=%s\n' % str(self.optimization).lower()) fp.write('TestDateEnable=true\n') @@ -110,9 +111,9 @@ def _create_param(self, alias=DEFAULT_MT4_NAME): fp.write('%s,F=1\n' % k) fp.write('%s,1=%s\n' % (k, value)) interval = values.pop('interval') - fp.write('%s,2=%s\n' % (k,interval)) + fp.write('%s,2=%s\n' % (k, interval)) maximum = values.pop('max') - fp.write('%s,3=%s\n' % (k,maximum)) + fp.write('%s,3=%s\n' % (k, maximum)) else: # if this value won't be optimized, write unused dummy data for same format. fp.write('%s,F=0\n' % k) @@ -130,7 +131,6 @@ def _create_param(self, alias=DEFAULT_MT4_NAME): fp.write('%s,2=0\n' % k) fp.write('%s,3=0\n' % k) - def _get_conf_abs_path(self, alias=DEFAULT_MT4_NAME): mt4 = get_mt4(alias=alias) conf_file = os.path.join(mt4.appdata_path, 'tester', '%s.conf' % self.ea_name) @@ -141,31 +141,28 @@ def run(self, alias=DEFAULT_MT4_NAME): Notes: run backtest """ - from report import BacktestReport self.optimization = False self._prepare(alias=alias) bt_conf = self._get_conf_abs_path(alias=alias) - + mt4 = get_mt4(alias=alias) mt4.run(self.ea_name, conf=bt_conf) - + ret = BacktestReport(self) return ret def optimize(self, alias=DEFAULT_MT4_NAME): """ """ - from report import OptimizationReport - self.optimization = True self._prepare(alias=alias) bt_conf = self._get_conf_abs_path(alias=alias) - + mt4 = get_mt4(alias=alias) mt4.run(self.ea_name, conf=bt_conf) - + ret = OptimizationReport(self) return ret diff --git a/metatrader/mt4.py b/metatrader/mt4.py index 9200851..3cec388 100644 --- a/metatrader/mt4.py +++ b/metatrader/mt4.py @@ -4,20 +4,28 @@ """ import os import logging +import subprocess + +try: + import winreg +except: + import _winreg as winreg +import codecs _mt4s = {} DEFAULT_MT4_NAME = 'default' -# mt4 program file path is written in origin.txt +# mt4 program file path is written in origin.txt ORIGIN_TXT = 'origin.txt' MT4_EXE = 'terminal.exe' + class MT4(object): """ Notes: meta trader4 class which can lunch metatrader4. this class will only lunch metatrader4, - because metatrader4 can lunch either normal mode or backtest mode. + because metatrader4 can lunch either normal mode or backtest mode. """ prog_path = None appdata_path = None @@ -45,11 +53,10 @@ def run(self, ea_name, conf=None): Notes: run terminal.exe. Args: - conf(string): abs path of conf file. - details see mt4 help doc Client Terminal/Tools/Configuration at Startup + conf(string): abs path of conf file. + details see mt4 help doc Client Terminal/Tools/Configuration at Startup """ - import subprocess - + if conf: prog = '"%s"' % os.path.join(self.prog_path, MT4_EXE) conf = '"%s"' % conf @@ -91,6 +98,7 @@ def has_mt4_subdirs(appdata_path): return ret + def is_uac_enabled(): """ Note: @@ -98,17 +106,18 @@ def is_uac_enabled(): Returns: True if uac is enabled, False if uac is disabled. """ - import _winreg - - reg_key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System', 0, _winreg.KEY_READ) - value, regtype = _winreg.QueryValueEx(reg_key, 'EnableLUA') - + + reg_key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System', 0, + winreg.KEY_READ) + value, regtype = winreg.QueryValueEx(reg_key, 'EnableLUA') + if value == 1: - #reg value 1 means UAC is enabled + # reg value 1 means UAC is enabled return True else: return False + def get_appdata_path(program_file_dir): """ Returns: @@ -128,7 +137,6 @@ def get_appdata_path(program_file_dir): if ORIGIN_TXT in files: origin_file = os.path.join(root, ORIGIN_TXT) - import codecs with codecs.open(origin_file, 'r', 'utf-16') as fp: line = fp.read() if line == program_file_dir: @@ -145,18 +153,19 @@ def get_appdata_path(program_file_dir): return app_dir + def initizalize(ntpath, alias=DEFAULT_MT4_NAME): """ Notes: initialize mt4 Args: ntpath(string): mt4 install folder path. - e.g.: C:\\Program Files (x86)\\MetaTrader 4 - Alpari Japan + e.g.: C:\\Program Files (x86)\\MetaTrader 4 - Alpari Japan alias(string): mt4 object alias name. default value is DEFAULT_MT4_NAME """ global _mt4s if alias not in _mt4s: - #store mt4 objecct with alias name + # store mt4 objecct with alias name _mt4s[alias] = MT4(ntpath, ) else: logging.info('%s is already initialized' % alias) diff --git a/metatrader/report.py b/metatrader/report.py index 1d0d386..e5bb4e1 100644 --- a/metatrader/report.py +++ b/metatrader/report.py @@ -4,13 +4,17 @@ @author: Taiga ''' -from mt4 import get_mt4 -from mt4 import DEFAULT_MT4_NAME +from metatrader.mt4 import get_mt4 +from metatrader.mt4 import DEFAULT_MT4_NAME import logging +import re +from metatrader.exception import InvalidReportFormat + def has_divtag_with_style(tag): return tag.name == 'div' and tag.has_attr('style') + class BaseReport(object): """ Notes: @@ -21,12 +25,13 @@ class BaseReport(object): symbol(string): currency symbol. e.g.: USDJPY from_date(datetime.datetime): backtest from date to_date(datetime.datetime): backtest to date - model(int): backtest model + model(int): backtest model 0: Every tick 1: Control points 2: Open prices only spread(int): spread """ + def __init__(self, backtest): self.ea_name = backtest.ea_name self.param = backtest.param @@ -36,6 +41,7 @@ def __init__(self, backtest): self.model = backtest.model self.spread = backtest.spread + class BacktestReport(BaseReport): """ Note: @@ -88,106 +94,106 @@ def __init__(self, backtest, alias=DEFAULT_MT4_NAME): report_file = get_report_abs_path(backtest.ea_name, alias=alias) with open(report_file, 'r') as fp: raw_html = fp.read() - + b_soup = BeautifulSoup(raw_html) summary_in_table = b_soup.find_all('table')[0] tds = summary_in_table.find_all('td') - + for index, td in enumerate(tds): if td.text == 'Initial deposit': - self.initial_deposit = float(tds[index+1].text) + self.initial_deposit = float(tds[index + 1].text) if td.text == 'Modelling quality': - modeling_quality_percentage_str = re.sub('%', '', tds[index+1].text) + modeling_quality_percentage_str = re.sub('%', '', tds[index + 1].text) self.modeling_quality_percentage = float(modeling_quality_percentage_str) elif td.text == 'Total net profit': - self.profit = float(tds[index+1].text) + self.profit = float(tds[index + 1].text) elif td.text == 'Gross profit': - self.gross_profit = float(tds[index+1].text) + self.gross_profit = float(tds[index + 1].text) elif td.text == 'Gross loss': - self.gross_loss = float(tds[index+1].text) + self.gross_loss = float(tds[index + 1].text) elif td.text == 'Profit factor': - self.profit_factor = float(tds[index+1].text) + self.profit_factor = float(tds[index + 1].text) elif td.text == 'Expected payoff': - self.expected_payoff = float(tds[index+1].text) + self.expected_payoff = float(tds[index + 1].text) elif td.text == 'Absolute drawdown': - self.abs_drawdown = float(tds[index+1].text) + self.abs_drawdown = float(tds[index + 1].text) elif td.text == 'Maximal drawdown': - data, rate = self.get_data_and_rate(tds[index+1].text) + data, rate = self.get_data_and_rate(tds[index + 1].text) self.max_drawdown_rate = rate self.max_drawdown = data elif td.text == 'Relative drawdown': - data, rate = self.get_data_and_rate(tds[index+1].text) + data, rate = self.get_data_and_rate(tds[index + 1].text) self.relative_drawdown_rate = rate self.relative_drawdown = data elif td.text == 'Total trades': - self.total_trades = int(tds[index+1].text) + self.total_trades = int(tds[index + 1].text) elif td.text == 'Short positions (won %)': - data, rate = self.get_data_and_rate(tds[index+1].text) + data, rate = self.get_data_and_rate(tds[index + 1].text) self.short_positions_rate = rate self.short_positions = data elif td.text == 'Long positions (won %)': - data, rate = self.get_data_and_rate(tds[index+1].text) + data, rate = self.get_data_and_rate(tds[index + 1].text) self.long_positions_rate = rate self.long_positions = data elif td.text == 'Profit trades (% of total)': - data, rate = self.get_data_and_rate(tds[index+1].text) + data, rate = self.get_data_and_rate(tds[index + 1].text) self.profit_trades_rate = rate self.profit_trades = int(data) elif td.text == 'Loss trades (% of total)': - data, rate = self.get_data_and_rate(tds[index+1].text) + data, rate = self.get_data_and_rate(tds[index + 1].text) self.loss_trades_rate = rate self.loss_trades = int(data) elif td.text == 'Largest': - if tds[index+1].text == 'profit trade': - self.largest_profit_trade = float(tds[index+2].text) - if tds[index+3].text == 'loss trade': - self.largest_loss_trade = float(tds[index+4].text) + if tds[index + 1].text == 'profit trade': + self.largest_profit_trade = float(tds[index + 2].text) + if tds[index + 3].text == 'loss trade': + self.largest_loss_trade = float(tds[index + 4].text) elif td.text == 'Average': - if tds[index+1].text == 'profit trade': - self.average_profit_trade = float(tds[index+2].text) - elif tds[index+1].text == 'consecutive wins': - self.ave_consecutive_wins = int(tds[index+2].text) - if tds[index+3].text == 'loss trade': - self.average_loss_trade = float(tds[index+4].text) - elif tds[index+3].text == 'consecutive losses': - self.ave_consecutive_losses = int(tds[index+4].text) + if tds[index + 1].text == 'profit trade': + self.average_profit_trade = float(tds[index + 2].text) + elif tds[index + 1].text == 'consecutive wins': + self.ave_consecutive_wins = int(tds[index + 2].text) + if tds[index + 3].text == 'loss trade': + self.average_loss_trade = float(tds[index + 4].text) + elif tds[index + 3].text == 'consecutive losses': + self.ave_consecutive_losses = int(tds[index + 4].text) elif td.text == 'Maximum': - if tds[index+1].text == 'consecutive wins (profit in money)': - token = self.split_to_tokens(tds[index+2].text) + if tds[index + 1].text == 'consecutive wins (profit in money)': + token = self.split_to_tokens(tds[index + 2].text) self.max_consecutive_wins_count = int(token[0]) self.max_consecutive_wins_profit = float(token[1]) - if tds[index+3].text == 'consecutive losses (loss in money)': - token = self.split_to_tokens(tds[index+4].text) + if tds[index + 3].text == 'consecutive losses (loss in money)': + token = self.split_to_tokens(tds[index + 4].text) self.max_consecutive_losses_count = int(token[0]) self.max_consecutive_losses_loss = float(token[1]) elif td.text == 'Maximal': - if tds[index+1].text == 'consecutive profit (count of wins)': - token = self.split_to_tokens(tds[index+2].text) + if tds[index + 1].text == 'consecutive profit (count of wins)': + token = self.split_to_tokens(tds[index + 2].text) self.max_consecutive_profit = float(token[0]) self.max_consecutive_profit_count = int(token[1]) - if tds[index+3].text == 'consecutive loss (count of losses)': - token = self.split_to_tokens(tds[index+4].text) + if tds[index + 3].text == 'consecutive loss (count of losses)': + token = self.split_to_tokens(tds[index + 4].text) self.max_consecutive_loss = float(token[0]) self.max_consecutive_loss_count = int(token[1]) - def get_data_and_rate(self, line): - import re - from exception import InvalidReportFormat + formatted_str = re.sub('(\(|\))', '', line) values = formatted_str.split(r' ') + rate = 0.0 + data = 0.0 if len(values) != 2: raise InvalidReportFormat('value of Maximal drawdown contains more than 2 values') - + for value in values: - if re.match('.*\%$',value): - rate = re.sub(r'%','',value) + if re.match('.*\%$', value): + rate = re.sub(r'%', '', value) rate = float(rate) else: - data = float(value) + data = float(value) return data, rate - + def split_to_tokens(self, line): ''' Notes: @@ -195,17 +201,16 @@ def split_to_tokens(self, line): e.g. 1 (123.45) => (1, 123.45) 123.45 (1) => (123.45, 1) ''' - import re - from exception import InvalidReportFormat formatted_str = re.sub('(\(|\))', '', line) values = formatted_str.split(r' ') if len(values) != 2: raise InvalidReportFormat('value of Maximal drawdown contains more than 2 values') - + return values + class ShortReport(BaseReport): """ @todo: implement __hash__ and __eq__ to do set operation @@ -221,12 +226,12 @@ class ShortReport(BaseReport): max_drawdown_rate(float): max drawdown rate of deposit initial_deposit(int): initial deposit of backtest of optimization """ - + def __init__(self, back_test, **kwargs): super(ShortReport, self).__init__(back_test) result = kwargs.copy() - self.param = result.pop('param') + self.param = result.pop('param') self.profit = result.pop('profit') self.total_trades = result.pop('total_trades') self.profit_factor = result.pop('profit_factor') @@ -235,13 +240,14 @@ def __init__(self, back_test, **kwargs): self.max_drawdown_rate = result.pop('max_drawdown_rate') self.initial_deposit = result.pop('initial_deposit') + class OptimizationReport(): """ Note: this class has short reports """ reports = [] - + def _is_valid_format(self, raw_html): from bs4 import BeautifulSoup @@ -249,7 +255,7 @@ def _is_valid_format(self, raw_html): b_soup = BeautifulSoup(raw_html) report_titles = b_soup.find_all(has_divtag_with_style) - + for title in report_titles: if title.text == 'Optimization Report': if self._get_initial_deposit(raw_html) != 0: @@ -271,13 +277,13 @@ def _get_initial_deposit(self, raw_html): initial_deposit = float(tds[1].text) return initial_deposit - + def _get_param_from_text(self, text): ''' Note: create param dict from text in td title attribute in optimization report. Args: - text(string): + text(string): e.g.: x=2; y=0.2; z=true; Returns: param(dict): ea param names and values. @@ -291,11 +297,11 @@ def _get_param_from_text(self, text): name_value = p.split(r'=') name = name_value[0] value = name_value[1] - + param[name] = value - + return param - + def _get_results(self, backtest, raw_html): from bs4 import BeautifulSoup results = [] @@ -307,7 +313,7 @@ def _get_results(self, backtest, raw_html): # delete first tr, because it is category name trs.pop(0) - + for tr in trs: tds = tr.find_all('td') param = None @@ -317,7 +323,7 @@ def _get_results(self, backtest, raw_html): expected_payoff = None max_drawdown = None max_drawdown_rate = None - + for i, td in enumerate(tds): if i == 0: param_raw_text = td.attrs.pop('title') @@ -334,7 +340,7 @@ def _get_results(self, backtest, raw_html): max_drawdown = float(td.text) elif i == 6: max_drawdown_rate = float(td.text) - + short_report = ShortReport(backtest, param=param, profit=profit, @@ -348,8 +354,7 @@ def _get_results(self, backtest, raw_html): return results def __init__(self, backtest, alias=DEFAULT_MT4_NAME): - from exception import InvalidReportFormat - + report_file = get_report_abs_path(backtest.ea_name, alias=alias) with open(report_file, 'r') as fp: @@ -357,14 +362,15 @@ def __init__(self, backtest, alias=DEFAULT_MT4_NAME): if self._is_valid_format(raw_html): try: - self.results = self._get_results(backtest, raw_html) + self.results = self._get_results(backtest, raw_html) except KeyError: err_msg = 'optimization report seems invlid format' logging.error(err_msg) raise else: raise InvalidReportFormat(report_file, r'"Optimization Report" not found in html') - + + def get_report_abs_path(ea_name, alias=DEFAULT_MT4_NAME): import os mt4 = get_mt4(alias=alias) From 89b0ee86752e6c0a672c0e3cd57e1ef7de9524e6 Mon Sep 17 00:00:00 2001 From: Victor Gabriel Leandro Alves Date: Tue, 1 Dec 2015 11:37:49 -0200 Subject: [PATCH 2/2] fix: Added missing TestPeriod parameter on Backtest Class and *.conf file generator. --- metatrader/backtest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metatrader/backtest.py b/metatrader/backtest.py index d12675a..b6e261d 100644 --- a/metatrader/backtest.py +++ b/metatrader/backtest.py @@ -34,6 +34,7 @@ def __init__(self, ea_name, param, symbol, period, from_date, to_date, model=0, self.ea_name = ea_name self.param = param self.symbol = symbol + self.period = period self.from_date = from_date self.to_date = to_date self.model = model @@ -81,6 +82,7 @@ def _create_conf(self, alias=DEFAULT_MT4_NAME): fp.write('TestExpert=%s\n' % self.ea_name) fp.write('TestExpertParameters=%s.set\n' % self.ea_name) fp.write('TestSymbol=%s\n' % self.symbol) + fp.write('TestPeriod=%s\n' % self.period) fp.write('TestModel=%s\n' % self.model) fp.write('TestSpread=%s\n' % self.spread) fp.write('TestOptimization=%s\n' % str(self.optimization).lower())