Merge pull request #2994 from Fredrik81/hyperopt-table
Added dynamic print table function to hyperopt
This commit is contained in:
commit
57523d58df
@ -51,7 +51,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
Hyperopt.print_result_table(config, trials, total_epochs,
|
Hyperopt.print_result_table(config, trials, total_epochs,
|
||||||
not filteroptions['only_best'], print_colorized)
|
not filteroptions['only_best'], print_colorized, 0)
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
print('User interrupted..')
|
print('User interrupted..')
|
||||||
|
|
||||||
|
@ -22,7 +22,7 @@ from colorama import init as colorama_init
|
|||||||
from joblib import (Parallel, cpu_count, delayed, dump, load,
|
from joblib import (Parallel, cpu_count, delayed, dump, load,
|
||||||
wrap_non_picklable_objects)
|
wrap_non_picklable_objects)
|
||||||
from pandas import DataFrame, json_normalize, isna
|
from pandas import DataFrame, json_normalize, isna
|
||||||
from tabulate import tabulate
|
import tabulate
|
||||||
|
|
||||||
from freqtrade.data.converter import trim_dataframe
|
from freqtrade.data.converter import trim_dataframe
|
||||||
from freqtrade.data.history import get_timerange
|
from freqtrade.data.history import get_timerange
|
||||||
@ -117,6 +117,7 @@ class Hyperopt:
|
|||||||
self.config['ask_strategy']['use_sell_signal'] = True
|
self.config['ask_strategy']['use_sell_signal'] = True
|
||||||
|
|
||||||
self.print_all = self.config.get('print_all', False)
|
self.print_all = self.config.get('print_all', False)
|
||||||
|
self.hyperopt_table_header = 0
|
||||||
self.print_colorized = self.config.get('print_colorized', False)
|
self.print_colorized = self.config.get('print_colorized', False)
|
||||||
self.print_json = self.config.get('print_json', False)
|
self.print_json = self.config.get('print_json', False)
|
||||||
|
|
||||||
@ -154,7 +155,7 @@ class Hyperopt:
|
|||||||
"""
|
"""
|
||||||
num_trials = len(self.trials)
|
num_trials = len(self.trials)
|
||||||
if num_trials > self.num_trials_saved:
|
if num_trials > self.num_trials_saved:
|
||||||
logger.info(f"Saving {num_trials} {plural(num_trials, 'epoch')}.")
|
logger.debug(f"Saving {num_trials} {plural(num_trials, 'epoch')}.")
|
||||||
dump(self.trials, self.trials_file)
|
dump(self.trials, self.trials_file)
|
||||||
self.num_trials_saved = num_trials
|
self.num_trials_saved = num_trials
|
||||||
if final:
|
if final:
|
||||||
@ -273,8 +274,10 @@ class Hyperopt:
|
|||||||
if not self.print_all:
|
if not self.print_all:
|
||||||
# Separate the results explanation string from dots
|
# Separate the results explanation string from dots
|
||||||
print("\n")
|
print("\n")
|
||||||
self.print_results_explanation(results, self.total_epochs, self.print_all,
|
self.print_result_table(self.config, results, self.total_epochs,
|
||||||
self.print_colorized)
|
self.print_all, self.print_colorized,
|
||||||
|
self.hyperopt_table_header)
|
||||||
|
self.hyperopt_table_header = 2
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def print_results_explanation(results, total_epochs, highlight_best: bool,
|
def print_results_explanation(results, total_epochs, highlight_best: bool,
|
||||||
@ -300,13 +303,15 @@ class Hyperopt:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def print_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool,
|
def print_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool,
|
||||||
print_colorized: bool) -> None:
|
print_colorized: bool, remove_header: int) -> None:
|
||||||
"""
|
"""
|
||||||
Log result table
|
Log result table
|
||||||
"""
|
"""
|
||||||
if not results:
|
if not results:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
tabulate.PRESERVE_WHITESPACE = True
|
||||||
|
|
||||||
trials = json_normalize(results, max_level=1)
|
trials = json_normalize(results, max_level=1)
|
||||||
trials['Best'] = ''
|
trials['Best'] = ''
|
||||||
trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
|
trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
|
||||||
@ -318,35 +323,63 @@ class Hyperopt:
|
|||||||
trials['is_profit'] = False
|
trials['is_profit'] = False
|
||||||
trials.loc[trials['is_initial_point'], 'Best'] = '*'
|
trials.loc[trials['is_initial_point'], 'Best'] = '*'
|
||||||
trials.loc[trials['is_best'], 'Best'] = 'Best'
|
trials.loc[trials['is_best'], 'Best'] = 'Best'
|
||||||
trials['Objective'] = trials['Objective'].astype(str)
|
|
||||||
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
|
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
|
||||||
trials['Trades'] = trials['Trades'].astype(str)
|
trials['Trades'] = trials['Trades'].astype(str)
|
||||||
|
|
||||||
trials['Epoch'] = trials['Epoch'].apply(
|
trials['Epoch'] = trials['Epoch'].apply(
|
||||||
lambda x: "{}/{}".format(x, total_epochs))
|
lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs)
|
||||||
|
)
|
||||||
trials['Avg profit'] = trials['Avg profit'].apply(
|
trials['Avg profit'] = trials['Avg profit'].apply(
|
||||||
lambda x: '{:,.2f}%'.format(x) if not isna(x) else x)
|
lambda x: ('{:,.2f}%'.format(x)).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
|
||||||
trials['Profit'] = trials['Profit'].apply(
|
)
|
||||||
lambda x: '{:,.2f}%'.format(x) if not isna(x) else x)
|
|
||||||
trials['Total profit'] = trials['Total profit'].apply(
|
|
||||||
lambda x: '{: 11.8f} '.format(x) + config['stake_currency'] if not isna(x) else x)
|
|
||||||
trials['Avg duration'] = trials['Avg duration'].apply(
|
trials['Avg duration'] = trials['Avg duration'].apply(
|
||||||
lambda x: '{:,.1f}m'.format(x) if not isna(x) else x)
|
lambda x: ('{:,.1f} m'.format(x)).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
|
||||||
|
)
|
||||||
|
trials['Objective'] = trials['Objective'].apply(
|
||||||
|
lambda x: '{:,.5f}'.format(x).rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
|
||||||
|
)
|
||||||
|
|
||||||
|
trials['Profit'] = trials.apply(
|
||||||
|
lambda x: '{:,.8f} {} {}'.format(
|
||||||
|
x['Total profit'], config['stake_currency'],
|
||||||
|
'({:,.2f}%)'.format(x['Profit']).rjust(10, ' ')
|
||||||
|
).rjust(25+len(config['stake_currency']))
|
||||||
|
if x['Total profit'] != 0.0 else '--'.rjust(25+len(config['stake_currency'])),
|
||||||
|
axis=1
|
||||||
|
)
|
||||||
|
trials = trials.drop(columns=['Total profit'])
|
||||||
|
|
||||||
if print_colorized:
|
if print_colorized:
|
||||||
for i in range(len(trials)):
|
for i in range(len(trials)):
|
||||||
if trials.loc[i]['is_profit']:
|
if trials.loc[i]['is_profit']:
|
||||||
for z in range(len(trials.loc[i])-3):
|
for j in range(len(trials.loc[i])-3):
|
||||||
trials.iat[i, z] = "{}{}{}".format(Fore.GREEN,
|
trials.iat[i, j] = "{}{}{}".format(Fore.GREEN,
|
||||||
str(trials.loc[i][z]), Fore.RESET)
|
str(trials.loc[i][j]), Fore.RESET)
|
||||||
if trials.loc[i]['is_best'] and highlight_best:
|
if trials.loc[i]['is_best'] and highlight_best:
|
||||||
for z in range(len(trials.loc[i])-3):
|
for j in range(len(trials.loc[i])-3):
|
||||||
trials.iat[i, z] = "{}{}{}".format(Style.BRIGHT,
|
trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT,
|
||||||
str(trials.loc[i][z]), Style.RESET_ALL)
|
str(trials.loc[i][j]), Style.RESET_ALL)
|
||||||
|
|
||||||
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
|
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
|
||||||
|
if remove_header > 0:
|
||||||
|
table = tabulate.tabulate(
|
||||||
|
trials.to_dict(orient='list'), tablefmt='orgtbl',
|
||||||
|
headers='keys', stralign="right"
|
||||||
|
)
|
||||||
|
|
||||||
print(tabulate(trials.to_dict(orient='list'), headers='keys', tablefmt='psql',
|
table = table.split("\n", remove_header)[remove_header]
|
||||||
stralign="right"))
|
elif remove_header < 0:
|
||||||
|
table = tabulate.tabulate(
|
||||||
|
trials.to_dict(orient='list'), tablefmt='psql',
|
||||||
|
headers='keys', stralign="right"
|
||||||
|
)
|
||||||
|
table = "\n".join(table.split("\n")[0:remove_header])
|
||||||
|
else:
|
||||||
|
table = tabulate.tabulate(
|
||||||
|
trials.to_dict(orient='list'), tablefmt='psql',
|
||||||
|
headers='keys', stralign="right"
|
||||||
|
)
|
||||||
|
print(table)
|
||||||
|
|
||||||
def has_space(self, space: str) -> bool:
|
def has_space(self, space: str) -> bool:
|
||||||
"""
|
"""
|
||||||
@ -534,7 +567,7 @@ class Hyperopt:
|
|||||||
def start(self) -> None:
|
def start(self) -> None:
|
||||||
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
|
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
|
||||||
logger.info(f"Using optimizer random state: {self.random_state}")
|
logger.info(f"Using optimizer random state: {self.random_state}")
|
||||||
|
self.hyperopt_table_header = -1
|
||||||
data, timerange = self.backtesting.load_bt_data()
|
data, timerange = self.backtesting.load_bt_data()
|
||||||
|
|
||||||
preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
|
preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
|
||||||
|
@ -426,17 +426,27 @@ def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results)
|
|||||||
def test_log_results_if_loss_improves(hyperopt, capsys) -> None:
|
def test_log_results_if_loss_improves(hyperopt, capsys) -> None:
|
||||||
hyperopt.current_best_loss = 2
|
hyperopt.current_best_loss = 2
|
||||||
hyperopt.total_epochs = 2
|
hyperopt.total_epochs = 2
|
||||||
|
|
||||||
hyperopt.print_results(
|
hyperopt.print_results(
|
||||||
{
|
{
|
||||||
'is_best': True,
|
|
||||||
'loss': 1,
|
'loss': 1,
|
||||||
|
'results_metrics':
|
||||||
|
{
|
||||||
|
'trade_count': 1,
|
||||||
|
'avg_profit': 0.1,
|
||||||
|
'total_profit': 0.001,
|
||||||
|
'profit': 1.0,
|
||||||
|
'duration': 20.0
|
||||||
|
},
|
||||||
|
'total_profit': 0,
|
||||||
'current_epoch': 2, # This starts from 1 (in a human-friendly manner)
|
'current_epoch': 2, # This starts from 1 (in a human-friendly manner)
|
||||||
'results_explanation': 'foo.',
|
'is_initial_point': False,
|
||||||
'is_initial_point': False
|
'is_best': True
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
out, err = capsys.readouterr()
|
out, err = capsys.readouterr()
|
||||||
assert ' 2/2: foo. Objective: 1.00000' in out
|
assert all(x in out
|
||||||
|
for x in ["Best", "2/2", " 1", "0.10%", "0.00100000 BTC (1.00%)", "20.0 m"])
|
||||||
|
|
||||||
|
|
||||||
def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None:
|
def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None:
|
||||||
@ -458,13 +468,11 @@ def test_save_trials_saves_trials(mocker, hyperopt, testdatadir, caplog) -> None
|
|||||||
|
|
||||||
hyperopt.trials = trials
|
hyperopt.trials = trials
|
||||||
hyperopt.save_trials(final=True)
|
hyperopt.save_trials(final=True)
|
||||||
assert log_has("Saving 1 epoch.", caplog)
|
|
||||||
assert log_has(f"1 epoch saved to '{trials_file}'.", caplog)
|
assert log_has(f"1 epoch saved to '{trials_file}'.", caplog)
|
||||||
mock_dump.assert_called_once()
|
mock_dump.assert_called_once()
|
||||||
|
|
||||||
hyperopt.trials = trials + trials
|
hyperopt.trials = trials + trials
|
||||||
hyperopt.save_trials(final=True)
|
hyperopt.save_trials(final=True)
|
||||||
assert log_has("Saving 2 epochs.", caplog)
|
|
||||||
assert log_has(f"2 epochs saved to '{trials_file}'.", caplog)
|
assert log_has(f"2 epochs saved to '{trials_file}'.", caplog)
|
||||||
|
|
||||||
|
|
||||||
@ -502,8 +510,18 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None:
|
|||||||
|
|
||||||
parallel = mocker.patch(
|
parallel = mocker.patch(
|
||||||
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
||||||
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result',
|
MagicMock(return_value=[{
|
||||||
'params': {'buy': {}, 'sell': {}, 'roi': {}, 'stoploss': 0.0}}])
|
'loss': 1, 'results_explanation': 'foo result',
|
||||||
|
'params': {'buy': {}, 'sell': {}, 'roi': {}, 'stoploss': 0.0},
|
||||||
|
'results_metrics':
|
||||||
|
{
|
||||||
|
'trade_count': 1,
|
||||||
|
'avg_profit': 0.1,
|
||||||
|
'total_profit': 0.001,
|
||||||
|
'profit': 1.0,
|
||||||
|
'duration': 20.0
|
||||||
|
},
|
||||||
|
}])
|
||||||
)
|
)
|
||||||
patch_exchange(mocker)
|
patch_exchange(mocker)
|
||||||
# Co-test loading ticker-interval from strategy
|
# Co-test loading ticker-interval from strategy
|
||||||
@ -797,11 +815,23 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
|
|||||||
|
|
||||||
parallel = mocker.patch(
|
parallel = mocker.patch(
|
||||||
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
||||||
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {},
|
MagicMock(return_value=[{
|
||||||
'params_details': {'buy': {'mfi-value': None},
|
'loss': 1, 'results_explanation': 'foo result', 'params': {},
|
||||||
'sell': {'sell-mfi-value': None},
|
'params_details': {
|
||||||
'roi': {}, 'stoploss': {'stoploss': None},
|
'buy': {'mfi-value': None},
|
||||||
'trailing': {'trailing_stop': None}}}])
|
'sell': {'sell-mfi-value': None},
|
||||||
|
'roi': {}, 'stoploss': {'stoploss': None},
|
||||||
|
'trailing': {'trailing_stop': None}
|
||||||
|
},
|
||||||
|
'results_metrics':
|
||||||
|
{
|
||||||
|
'trade_count': 1,
|
||||||
|
'avg_profit': 0.1,
|
||||||
|
'total_profit': 0.001,
|
||||||
|
'profit': 1.0,
|
||||||
|
'duration': 20.0
|
||||||
|
}
|
||||||
|
}])
|
||||||
)
|
)
|
||||||
patch_exchange(mocker)
|
patch_exchange(mocker)
|
||||||
|
|
||||||
@ -823,7 +853,11 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
|
|||||||
parallel.assert_called_once()
|
parallel.assert_called_once()
|
||||||
|
|
||||||
out, err = capsys.readouterr()
|
out, err = capsys.readouterr()
|
||||||
assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null,"trailing_stop":null}' in out # noqa: E501
|
result_str = (
|
||||||
|
'{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi"'
|
||||||
|
':{},"stoploss":null,"trailing_stop":null}'
|
||||||
|
)
|
||||||
|
assert result_str in out # noqa: E501
|
||||||
assert dumper.called
|
assert dumper.called
|
||||||
# Should be called twice, once for tickerdata, once to save evaluations
|
# Should be called twice, once for tickerdata, once to save evaluations
|
||||||
assert dumper.call_count == 2
|
assert dumper.call_count == 2
|
||||||
@ -840,10 +874,22 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None
|
|||||||
|
|
||||||
parallel = mocker.patch(
|
parallel = mocker.patch(
|
||||||
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
||||||
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {},
|
MagicMock(return_value=[{
|
||||||
'params_details': {'buy': {'mfi-value': None},
|
'loss': 1, 'results_explanation': 'foo result', 'params': {},
|
||||||
'sell': {'sell-mfi-value': None},
|
'params_details': {
|
||||||
'roi': {}, 'stoploss': {'stoploss': None}}}])
|
'buy': {'mfi-value': None},
|
||||||
|
'sell': {'sell-mfi-value': None},
|
||||||
|
'roi': {}, 'stoploss': {'stoploss': None}
|
||||||
|
},
|
||||||
|
'results_metrics':
|
||||||
|
{
|
||||||
|
'trade_count': 1,
|
||||||
|
'avg_profit': 0.1,
|
||||||
|
'total_profit': 0.001,
|
||||||
|
'profit': 1.0,
|
||||||
|
'duration': 20.0
|
||||||
|
}
|
||||||
|
}])
|
||||||
)
|
)
|
||||||
patch_exchange(mocker)
|
patch_exchange(mocker)
|
||||||
|
|
||||||
@ -882,8 +928,18 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) ->
|
|||||||
|
|
||||||
parallel = mocker.patch(
|
parallel = mocker.patch(
|
||||||
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
||||||
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {},
|
MagicMock(return_value=[{
|
||||||
'params_details': {'roi': {}, 'stoploss': {'stoploss': None}}}])
|
'loss': 1, 'results_explanation': 'foo result', 'params': {},
|
||||||
|
'params_details': {'roi': {}, 'stoploss': {'stoploss': None}},
|
||||||
|
'results_metrics':
|
||||||
|
{
|
||||||
|
'trade_count': 1,
|
||||||
|
'avg_profit': 0.1,
|
||||||
|
'total_profit': 0.001,
|
||||||
|
'profit': 1.0,
|
||||||
|
'duration': 20.0
|
||||||
|
}
|
||||||
|
}])
|
||||||
)
|
)
|
||||||
patch_exchange(mocker)
|
patch_exchange(mocker)
|
||||||
|
|
||||||
@ -923,7 +979,16 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys)
|
|||||||
parallel = mocker.patch(
|
parallel = mocker.patch(
|
||||||
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
||||||
MagicMock(return_value=[{
|
MagicMock(return_value=[{
|
||||||
'loss': 1, 'results_explanation': 'foo result', 'params': {'stoploss': 0.0}}])
|
'loss': 1, 'results_explanation': 'foo result', 'params': {'stoploss': 0.0},
|
||||||
|
'results_metrics':
|
||||||
|
{
|
||||||
|
'trade_count': 1,
|
||||||
|
'avg_profit': 0.1,
|
||||||
|
'total_profit': 0.001,
|
||||||
|
'profit': 1.0,
|
||||||
|
'duration': 20.0
|
||||||
|
}
|
||||||
|
}])
|
||||||
)
|
)
|
||||||
patch_exchange(mocker)
|
patch_exchange(mocker)
|
||||||
|
|
||||||
@ -1001,7 +1066,17 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None:
|
|||||||
|
|
||||||
parallel = mocker.patch(
|
parallel = mocker.patch(
|
||||||
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
||||||
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}])
|
MagicMock(return_value=[{
|
||||||
|
'loss': 1, 'results_explanation': 'foo result', 'params': {},
|
||||||
|
'results_metrics':
|
||||||
|
{
|
||||||
|
'trade_count': 1,
|
||||||
|
'avg_profit': 0.1,
|
||||||
|
'total_profit': 0.001,
|
||||||
|
'profit': 1.0,
|
||||||
|
'duration': 20.0
|
||||||
|
}
|
||||||
|
}])
|
||||||
)
|
)
|
||||||
patch_exchange(mocker)
|
patch_exchange(mocker)
|
||||||
|
|
||||||
@ -1048,7 +1123,17 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None
|
|||||||
|
|
||||||
parallel = mocker.patch(
|
parallel = mocker.patch(
|
||||||
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
|
||||||
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}])
|
MagicMock(return_value=[{
|
||||||
|
'loss': 1, 'results_explanation': 'foo result', 'params': {},
|
||||||
|
'results_metrics':
|
||||||
|
{
|
||||||
|
'trade_count': 1,
|
||||||
|
'avg_profit': 0.1,
|
||||||
|
'total_profit': 0.001,
|
||||||
|
'profit': 1.0,
|
||||||
|
'duration': 20.0
|
||||||
|
}
|
||||||
|
}])
|
||||||
)
|
)
|
||||||
patch_exchange(mocker)
|
patch_exchange(mocker)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user