adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4930 from freqtrade/hyperopt_memory

Browse Source 
Hyperopt memory problems
This commit is contained in:
Matthias
2021-05-15 07:12:57 +02:00
committed by GitHub
parent 3aaf06a3e2 5e73195b30
commit a6c644161d
7 changed files with 139 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
freqtrade/optimize/hyperopt.py
View File

@@ -9,11 +9,11 @@ import random
import warnings
from datetime import datetime, timezone
from math import ceil
from operator import itemgetter
from pathlib import Path
from typing import Any, Dict, List, Optional
import progressbar
import rapidjson
from colorama import Fore, Style
from colorama import init as colorama_init
from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
@@ -86,7 +86,7 @@ class Hyperopt:
time_now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
strategy = str(self.config['strategy'])
self.results_file: Path = (self.config['user_data_dir'] / 'hyperopt_results' /
f'strategy_{strategy}_hyperopt_results_{time_now}.pickle')
f'strategy_{strategy}_{time_now}.fthypt')
self.data_pickle_file = (self.config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_tickerdata.pkl')
self.total_epochs = config.get('epochs', 0)
@@ -96,9 +96,7 @@ class Hyperopt:
self.clean_hyperopt()
self.num_epochs_saved = 0
# Previous evaluations
self.epochs: List = []
self.current_best_epoch: Optional[Dict[str, Any]] = None
# Populate functions here (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
@@ -156,21 +154,24 @@ class Hyperopt:
# and the values are taken from the list of parameters.
return {d.name: v for d, v in zip(dimensions, raw_params)}
def _save_results(self) -> None:
def _save_result(self, epoch: Dict) -> None:
"""
Save hyperopt results to file
Store one line per epoch.
While not a valid json object - this allows appending easily.
:param epoch: result dictionary for this epoch.
"""
num_epochs = len(self.epochs)
if num_epochs > self.num_epochs_saved:
logger.debug(f"Saving {num_epochs} {plural(num_epochs, 'epoch')}.")
dump(self.epochs, self.results_file)
self.num_epochs_saved = num_epochs
logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
f"saved to '{self.results_file}'.")
# Store hyperopt filename
latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN)
file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)},
log=False)
with self.results_file.open('a') as f:
rapidjson.dump(epoch, f, default=str, number_mode=rapidjson.NM_NATIVE)
f.write("\n")
self.num_epochs_saved += 1
logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
f"saved to '{self.results_file}'.")
# Store hyperopt filename
latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN)
file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)},
log=False)
def _get_params_details(self, params: Dict) -> Dict:
"""
@@ -268,7 +269,7 @@ class Hyperopt:
self.backtesting.strategy.trailing_only_offset_is_reached = \
d['trailing_only_offset_is_reached']
processed = load(self.data_pickle_file)
processed = load(self.data_pickle_file, mmap_mode='r+')
bt_results = self.backtesting.backtest(
processed=processed,
@@ -343,12 +344,7 @@ class Hyperopt:
def _set_random_state(self, random_state: Optional[int]) -> int:
return random_state or random.randint(1, 2**16 - 1)
def start(self) -> None:
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
logger.info(f"Using optimizer random state: {self.random_state}")
self.hyperopt_table_header = -1
# Initialize spaces ...
self.init_spaces()
def prepare_hyperopt_data(self) -> None:
data, timerange = self.backtesting.load_bt_data()
logger.info("Dataload complete. Calculating indicators")
preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
@@ -365,6 +361,15 @@ class Hyperopt:
dump(preprocessed, self.data_pickle_file)
def start(self) -> None:
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
logger.info(f"Using optimizer random state: {self.random_state}")
self.hyperopt_table_header = -1
# Initialize spaces ...
self.init_spaces()
self.prepare_hyperopt_data()
# We don't need exchange instance anymore while running hyperopt
self.backtesting.exchange.close()
self.backtesting.exchange._api = None # type: ignore
@@ -442,25 +447,21 @@ class Hyperopt:
if is_best:
self.current_best_loss = val['loss']
self.epochs.append(val)
self.current_best_epoch = val
# Save results after each best epoch and every 100 epochs
if is_best or current % 100 == 0:
self._save_results()
self._save_result(val)
pbar.update(current)
except KeyboardInterrupt:
print('User interrupted..')
self._save_results()
logger.info(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
f"saved to '{self.results_file}'.")
if self.epochs:
sorted_epochs = sorted(self.epochs, key=itemgetter('loss'))
best_epoch = sorted_epochs[0]
HyperoptTools.print_epoch_details(best_epoch, self.total_epochs, self.print_json)
if self.current_best_epoch:
HyperoptTools.print_epoch_details(self.current_best_epoch, self.total_epochs,
self.print_json)
else:
# This is printed when Ctrl+C is pressed quickly, before first epochs have
# a chance to be evaluated.

27
freqtrade/optimize/hyperopt_tools.py
View File

@@ -31,15 +31,27 @@ class HyperoptTools():
else:
return any(s in config['spaces'] for s in [space, 'all', 'default'])
@staticmethod
def _read_results_pickle(results_file: Path) -> List:
"""
Read hyperopt results from pickle file
LEGACY method - new files are written as json and cannot be read with this method.
"""
from joblib import load
logger.info(f"Reading pickled epochs from '{results_file}'")
data = load(results_file)
return data
@staticmethod
def _read_results(results_file: Path) -> List:
"""
Read hyperopt results from file
"""
from joblib import load
logger.info("Reading epochs from '%s'", results_file)
data = load(results_file)
import rapidjson
logger.info(f"Reading epochs from '{results_file}'")
with results_file.open('r') as f:
data = [rapidjson.loads(line) for line in f]
return data
@staticmethod
@@ -49,7 +61,10 @@ class HyperoptTools():
"""
epochs: List = []
if results_file.is_file() and results_file.stat().st_size > 0:
epochs = HyperoptTools._read_results(results_file)
if results_file.suffix == '.pickle':
epochs = HyperoptTools._read_results_pickle(results_file)
else:
epochs = HyperoptTools._read_results(results_file)
# Detection of some old format, without 'is_best' field saved
if epochs[0].get('is_best') is None:
raise OperationalException(
@@ -167,7 +182,7 @@ class HyperoptTools():
@staticmethod
def is_best_loss(results, current_best_loss: float) -> bool:
return results['loss'] < current_best_loss
return bool(results['loss'] < current_best_loss)
@staticmethod
def format_results_explanation_string(results_metrics: Dict, stake_currency: str) -> str:

16
freqtrade/optimize/optimize_reports.py
View File

@@ -313,9 +313,9 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0,
'profit_total': results['profit_abs'].sum() / starting_balance,
'profit_total_abs': results['profit_abs'].sum(),
'backtest_start': min_date,
'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
'backtest_start_ts': int(min_date.timestamp() * 1000),
'backtest_end': max_date,
'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT),
'backtest_end_ts': int(max_date.timestamp() * 1000),
'backtest_days': backtest_days,
@@ -362,9 +362,9 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
strat_stats.update({
'max_drawdown': max_drawdown,
'max_drawdown_abs': drawdown_abs,
'drawdown_start': drawdown_start,
'drawdown_start': drawdown_start.strftime(DATETIME_PRINT_FORMAT),
'drawdown_start_ts': drawdown_start.timestamp() * 1000,
'drawdown_end': drawdown_end,
'drawdown_end': drawdown_end.strftime(DATETIME_PRINT_FORMAT),
'drawdown_end_ts': drawdown_end.timestamp() * 1000,
'max_drawdown_low': low_val,
@@ -497,8 +497,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
best_trade = max(strat_results['trades'], key=lambda x: x['profit_ratio'])
worst_trade = min(strat_results['trades'], key=lambda x: x['profit_ratio'])
metrics = [
('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)),
('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)),
('Backtesting from', strat_results['backtest_start']),
('Backtesting to', strat_results['backtest_end']),
('Max open trades', strat_results['max_open_trades']),
('', ''), # Empty line to improve readability
('Total trades', strat_results['total_trades']),
@@ -546,8 +546,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
strat_results['stake_currency'])),
('Drawdown low', round_coin_value(strat_results['max_drawdown_low'],
strat_results['stake_currency'])),
('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)),
('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)),
('Drawdown Start', strat_results['drawdown_start']),
('Drawdown End', strat_results['drawdown_end']),
('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"),
]