From f4c7edf551442f369c5f451d5cd66000f196d728 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 14 Dec 2019 02:12:16 +0300 Subject: [PATCH 01/19] No args for backtest(), use arguments --- freqtrade/optimize/backtesting.py | 53 +++++++++++++++---------------- freqtrade/optimize/hyperopt.py | 14 ++++---- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1070f2481..92129b324 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -279,30 +279,28 @@ class Backtesting: return bt_res return None - def backtest(self, args: Dict) -> DataFrame: + def backtest(self, processed: Dict, stake_amount: int, + start_date, end_date, + max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame: """ - Implements backtesting functionality + Implement backtesting functionality NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. Of course try to not have ugly code. By some accessor are sometime slower than functions. - Avoid, logging on this method + Avoid extensive logging in this method and functions it calls. - :param args: a dict containing: - stake_amount: btc amount to use for each trade - processed: a processed dictionary with format {pair, data} - max_open_trades: maximum number of concurrent trades (default: 0, disabled) - position_stacking: do we allow position stacking? (default: False) - :return: DataFrame + :param processed: a processed dictionary with format {pair, data} + :param stake_amount: amount to use for each trade + :param start_date: backtesting timerange start datetime + :param end_date: backtesting timerange end datetime + :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited + :param position_stacking: do we allow position stacking? + :return: DataFrame with trades (results of backtesting) """ - # Arguments are long and noisy, so this is commented out. - # Uncomment if you need to debug the backtest() method. - # logger.debug(f"Start backtest, args: {args}") - processed = args['processed'] - stake_amount = args['stake_amount'] - max_open_trades = args.get('max_open_trades', 0) - position_stacking = args.get('position_stacking', False) - start_date = args['start_date'] - end_date = args['end_date'] + logger.debug(f"Run backtest, stake_amount: {stake_amount}, " + f"start_date: {start_date}, end_date: {end_date}, " + f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}" + ) trades = [] trade_count_lock: Dict = {} @@ -369,18 +367,21 @@ class Backtesting: def start(self) -> None: """ - Run a backtesting end-to-end + Run backtesting end-to-end :return: None """ data: Dict[str, Any] = {} + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + # Use max_open_trades in backtesting, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): max_open_trades = self.config['max_open_trades'] else: logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 + position_stacking = self.config.get('position_stacking', False) data, timerange = self.load_bt_data() @@ -403,14 +404,12 @@ class Backtesting: ) # Execute backtest and print results all_results[self.strategy.get_strategy_name()] = self.backtest( - { - 'stake_amount': self.config.get('stake_amount'), - 'processed': preprocessed, - 'max_open_trades': max_open_trades, - 'position_stacking': self.config.get('position_stacking', False), - 'start_date': min_date, - 'end_date': max_date, - } + processed=preprocessed, + stake_amount=self.config.get('stake_amount'), + start_date=min_date, + end_date=max_date, + max_open_trades=max_open_trades, + position_stacking=position_stacking, ) for strategy, results in all_results.items(): diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a8f1a71ef..525f491f3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -372,14 +372,12 @@ class Hyperopt: min_date, max_date = get_timerange(processed) backtesting_results = self.backtesting.backtest( - { - 'stake_amount': self.config['stake_amount'], - 'processed': processed, - 'max_open_trades': self.max_open_trades, - 'position_stacking': self.position_stacking, - 'start_date': min_date, - 'end_date': max_date, - } + processed=processed, + stake_amount=self.config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=self.max_open_trades, + position_stacking=self.position_stacking, ) return self._get_results_dict(backtesting_results, min_date, max_date, params_dict, params_details) From 52f0ed53109e1a4aa8429510321b4a6fe1f08660 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 14 Dec 2019 02:12:44 +0300 Subject: [PATCH 02/19] Adjust tests --- tests/optimize/test_backtest_detail.py | 12 ++--- tests/optimize/test_backtesting.py | 73 ++++++++++++-------------- 2 files changed, 38 insertions(+), 47 deletions(-) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 47cb9f353..bd2765430 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -382,13 +382,11 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: data_processed = {pair: frame.copy()} min_date, max_date = get_timerange({pair: frame}) results = backtesting.backtest( - { - 'stake_amount': default_conf['stake_amount'], - 'processed': data_processed, - 'max_open_trades': 10, - 'start_date': min_date, - 'end_date': max_date, - } + processed=data_processed, + stake_amount=default_conf['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=10, ) assert len(results) == len(data.trades) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 83d212e3d..acbc44e21 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -103,14 +103,12 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: min_date, max_date = get_timerange(processed) assert isinstance(processed, dict) results = backtesting.backtest( - { - 'stake_amount': config['stake_amount'], - 'processed': processed, - 'max_open_trades': 1, - 'position_stacking': False, - 'start_date': min_date, - 'end_date': max_date, - } + processed=processed, + stake_amount=config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=1, + position_stacking=False, ) # results :: assert len(results) == num_results @@ -132,7 +130,7 @@ def _load_pair_as_ticks(pair, tickfreq): # FIX: fixturize this? -def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record=None): +def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'): data = history.load_data(datadir=datadir, timeframe='1m', pairs=[pair]) data = trim_dictlist(data, -201) patch_exchange(mocker) @@ -140,13 +138,12 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record= processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timerange(processed) return { - 'stake_amount': conf['stake_amount'], 'processed': processed, - 'max_open_trades': 10, - 'position_stacking': False, - 'record': record, + 'stake_amount': conf['stake_amount'], 'start_date': min_date, 'end_date': max_date, + 'max_open_trades': 10, + 'position_stacking': False, } @@ -422,14 +419,12 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None: data_processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timerange(data_processed) results = backtesting.backtest( - { - 'stake_amount': default_conf['stake_amount'], - 'processed': data_processed, - 'max_open_trades': 10, - 'position_stacking': False, - 'start_date': min_date, - 'end_date': max_date, - } + processed=data_processed, + stake_amount=default_conf['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=10, + position_stacking=False, ) assert not results.empty assert len(results) == 2 @@ -478,14 +473,12 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) - processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timerange(processed) results = backtesting.backtest( - { - 'stake_amount': default_conf['stake_amount'], - 'processed': processed, - 'max_open_trades': 1, - 'position_stacking': False, - 'start_date': min_date, - 'end_date': max_date, - } + processed=processed, + stake_amount=default_conf['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=1, + position_stacking=False, ) assert not results.empty assert len(results) == 1 @@ -525,7 +518,7 @@ def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = fun # Override backtesting.strategy.advise_sell = fun # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) assert results.empty @@ -540,7 +533,7 @@ def test_backtest_only_sell(mocker, default_conf, testdatadir): backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = fun # Override backtesting.strategy.advise_sell = fun # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) assert results.empty @@ -553,7 +546,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = _trend_alternate # Override backtesting.strategy.advise_sell = _trend_alternate # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) backtesting._store_backtest_result("test_.json", results) # 200 candles in backtest data # won't buy on first (shifted by 1) @@ -598,15 +591,15 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timerange(data_processed) backtest_conf = { - 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, - 'max_open_trades': 3, - 'position_stacking': False, + 'stake_amount': default_conf['stake_amount'], 'start_date': min_date, 'end_date': max_date, + 'max_open_trades': 3, + 'position_stacking': False, } - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) # Make sure we have parallel trades assert len(evaluate_result_multi(results, '5m', 2)) > 0 @@ -614,14 +607,14 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) assert len(evaluate_result_multi(results, '5m', 3)) == 0 backtest_conf = { - 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, - 'max_open_trades': 1, - 'position_stacking': False, + 'stake_amount': default_conf['stake_amount'], 'start_date': min_date, 'end_date': max_date, + 'max_open_trades': 1, + 'position_stacking': False, } - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) assert len(evaluate_result_multi(results, '5m', 1)) == 0 From bd4dd8403b0b6223bff4ef494f28c15c506caf75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Jan 2020 12:48:13 +0100 Subject: [PATCH 03/19] Fix type-errors with stake_amount --- freqtrade/optimize/backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 92129b324..cdf74f65f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -279,7 +279,7 @@ class Backtesting: return bt_res return None - def backtest(self, processed: Dict, stake_amount: int, + def backtest(self, processed: Dict, stake_amount: float, start_date, end_date, max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame: """ @@ -300,7 +300,7 @@ class Backtesting: logger.debug(f"Run backtest, stake_amount: {stake_amount}, " f"start_date: {start_date}, end_date: {end_date}, " f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}" - ) + ) trades = [] trade_count_lock: Dict = {} @@ -405,7 +405,7 @@ class Backtesting: # Execute backtest and print results all_results[self.strategy.get_strategy_name()] = self.backtest( processed=preprocessed, - stake_amount=self.config.get('stake_amount'), + stake_amount=self.config['stake_amount'], start_date=min_date, end_date=max_date, max_open_trades=max_open_trades, From a3ac05cc16f3b9fd684fd0330f9b5e3191a34d4a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Jan 2020 13:38:13 +0100 Subject: [PATCH 04/19] Fix missed mock --- tests/test_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index e145a8470..7ca2cac5a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -665,11 +665,11 @@ def test_start_list_strategies(mocker, caplog, capsys): assert "DefaultStrategy" in captured.out -def test_start_test_pairlist(mocker, caplog, markets, tickers, default_conf, capsys): +def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): + patch_exchange(mocker, mock_markets=True) mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), exchange_has=MagicMock(return_value=True), - get_tickers=tickers + get_tickers=tickers, ) default_conf['pairlists'] = [ From 3f2542fcbcf185ac61668f1394c781da5c7ef421 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 10:44:42 +0100 Subject: [PATCH 05/19] Move edge-module out of __init__.py --- freqtrade/edge/__init__.py | 465 +---------------------------- freqtrade/edge/edge_positioning.py | 464 ++++++++++++++++++++++++++++ 2 files changed, 465 insertions(+), 464 deletions(-) create mode 100644 freqtrade/edge/edge_positioning.py diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 15883357b..d275a80e3 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -1,464 +1 @@ -# pragma pylint: disable=W0603 -""" Edge positioning package """ -import logging -from typing import Any, Dict, NamedTuple - -import arrow -import numpy as np -import utils_find_1st as utf1st -from pandas import DataFrame - -from freqtrade import constants -from freqtrade.configuration import TimeRange -from freqtrade.data import history -from freqtrade.exceptions import OperationalException -from freqtrade.strategy.interface import SellType - -logger = logging.getLogger(__name__) - - -class PairInfo(NamedTuple): - stoploss: float - winrate: float - risk_reward_ratio: float - required_risk_reward: float - expectancy: float - nb_trades: int - avg_trade_duration: float - - -class Edge: - """ - Calculates Win Rate, Risk Reward Ratio, Expectancy - against historical data for a give set of markets and a strategy - it then adjusts stoploss and position size accordingly - and force it into the strategy - Author: https://github.com/mishaker - """ - - config: Dict = {} - _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - - def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: - - self.config = config - self.exchange = exchange - self.strategy = strategy - - self.edge_config = self.config.get('edge', {}) - self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._final_pairs: list = [] - - # checking max_open_trades. it should be -1 as with Edge - # the number of trades is determined by position size - if self.config['max_open_trades'] != float('inf'): - logger.critical('max_open_trades should be -1 in config !') - - if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT: - raise OperationalException('Edge works only with unlimited stake amount') - - # Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future. - self._capital_percentage: float = self.edge_config.get( - 'capital_available_percentage', self.config['tradable_balance_ratio']) - self._allowed_risk: float = self.edge_config.get('allowed_risk') - self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) - self._last_updated: int = 0 # Timestamp of pairs last updated time - self._refresh_pairs = True - - self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) - self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) - self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) - - # calculating stoploss range - self._stoploss_range = np.arange( - self._stoploss_range_min, - self._stoploss_range_max, - self._stoploss_range_step - ) - - self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift( - days=-1 * self._since_number_of_days).format('YYYYMMDD')) - if config.get('fee'): - self.fee = config['fee'] - else: - self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0]) - - def calculate(self) -> bool: - pairs = self.config['exchange']['pair_whitelist'] - heartbeat = self.edge_config.get('process_throttle_secs') - - if (self._last_updated > 0) and ( - self._last_updated + heartbeat > arrow.utcnow().timestamp): - return False - - data: Dict[str, Any] = {} - logger.info('Using stake_currency: %s ...', self.config['stake_currency']) - logger.info('Using local backtesting data (using whitelist in given config) ...') - - if self._refresh_pairs: - history.refresh_data( - datadir=self.config['datadir'], - pairs=pairs, - exchange=self.exchange, - timeframe=self.strategy.ticker_interval, - timerange=self._timerange, - ) - - data = history.load_data( - datadir=self.config['datadir'], - pairs=pairs, - timeframe=self.strategy.ticker_interval, - timerange=self._timerange, - startup_candles=self.strategy.startup_candle_count, - ) - - if not data: - # Reinitializing cached pairs - self._cached_pairs = {} - logger.critical("No data found. Edge is stopped ...") - return False - - preprocessed = self.strategy.tickerdata_to_dataframe(data) - - # Print timeframe - min_date, max_date = history.get_timerange(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days) ...', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - - trades: list = [] - for pair, pair_data in preprocessed.items(): - # Sorting dataframe by date and reset index - pair_data = pair_data.sort_values(by=['date']) - pair_data = pair_data.reset_index(drop=True) - - ticker_data = self.strategy.advise_sell( - self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - - trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range) - - # If no trade found then exit - if len(trades) == 0: - logger.info("No trades found.") - return False - - # Fill missing, calculable columns, profit, duration , abs etc. - trades_df = self._fill_calculable_fields(DataFrame(trades)) - self._cached_pairs = self._process_expectancy(trades_df) - self._last_updated = arrow.utcnow().timestamp - - return True - - def stake_amount(self, pair: str, free_capital: float, - total_capital: float, capital_in_trade: float) -> float: - stoploss = self.stoploss(pair) - available_capital = (total_capital + capital_in_trade) * self._capital_percentage - allowed_capital_at_risk = available_capital * self._allowed_risk - max_position_size = abs(allowed_capital_at_risk / stoploss) - position_size = min(max_position_size, free_capital) - if pair in self._cached_pairs: - logger.info( - 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' - ' capital in trade: %s, free capital: %s, total capital: %s,' - ' stoploss: %s, available capital: %s.', - self._cached_pairs[pair].winrate, - self._cached_pairs[pair].expectancy, - position_size, pair, - capital_in_trade, free_capital, total_capital, - stoploss, available_capital - ) - return round(position_size, 15) - - def stoploss(self, pair: str) -> float: - if pair in self._cached_pairs: - return self._cached_pairs[pair].stoploss - else: - logger.warning('tried to access stoploss of a non-existing pair, ' - 'strategy stoploss is returned instead.') - return self.strategy.stoploss - - def adjust(self, pairs) -> list: - """ - Filters out and sorts "pairs" according to Edge calculated pairs - """ - final = [] - for pair, info in self._cached_pairs.items(): - if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ - info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ - pair in pairs: - final.append(pair) - - if self._final_pairs != final: - self._final_pairs = final - if self._final_pairs: - logger.info( - 'Minimum expectancy and minimum winrate are met only for %s,' - ' so other pairs are filtered out.', - self._final_pairs - ) - else: - logger.info( - 'Edge removed all pairs as no pair with minimum expectancy ' - 'and minimum winrate was found !' - ) - - return self._final_pairs - - def accepted_pairs(self) -> list: - """ - return a list of accepted pairs along with their winrate, expectancy and stoploss - """ - final = [] - for pair, info in self._cached_pairs.items(): - if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ - info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)): - final.append({ - 'Pair': pair, - 'Winrate': info.winrate, - 'Expectancy': info.expectancy, - 'Stoploss': info.stoploss, - }) - return final - - def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: - """ - The result frame contains a number of columns that are calculable - from other columns. These are left blank till all rows are added, - to be populated in single vector calls. - - Columns to be populated are: - - Profit - - trade duration - - profit abs - :param result Dataframe - :return: result Dataframe - """ - - # stake and fees - # stake = 0.015 - # 0.05% is 0.0005 - # fee = 0.001 - - # we set stake amount to an arbitrary amount. - # as it doesn't change the calculation. - # all returned values are relative. they are percentages. - stake = 0.015 - fee = self.fee - open_fee = fee / 2 - close_fee = fee / 2 - - result['trade_duration'] = result['close_time'] - result['open_time'] - - result['trade_duration'] = result['trade_duration'].map( - lambda x: int(x.total_seconds() / 60)) - - # Spends, Takes, Profit, Absolute Profit - - # Buy Price - result['buy_vol'] = stake / result['open_rate'] # How many target are we buying - result['buy_fee'] = stake * open_fee - result['buy_spend'] = stake + result['buy_fee'] # How much we're spending - - # Sell price - result['sell_sum'] = result['buy_vol'] * result['close_rate'] - result['sell_fee'] = result['sell_sum'] * close_fee - result['sell_take'] = result['sell_sum'] - result['sell_fee'] - - # profit_percent - result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend'] - - # Absolute profit - result['profit_abs'] = result['sell_take'] - result['buy_spend'] - - return result - - def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]: - """ - This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs - The calulation will be done per pair and per strategy. - """ - # Removing pairs having less than min_trades_number - min_trades_number = self.edge_config.get('min_trade_number', 10) - results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) - ################################### - - # Removing outliers (Only Pumps) from the dataset - # The method to detect outliers is to calculate standard deviation - # Then every value more than (standard deviation + 2*average) is out (pump) - # - # Removing Pumps - if self.edge_config.get('remove_pumps', False): - results = results.groupby(['pair', 'stoploss']).apply( - lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) - ########################################################################## - - # Removing trades having a duration more than X minutes (set in config) - max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440) - results = results[results.trade_duration < max_trade_duration] - ####################################################################### - - if results.empty: - return {} - - groupby_aggregator = { - 'profit_abs': [ - ('nb_trades', 'count'), # number of all trades - ('profit_sum', lambda x: x[x > 0].sum()), # cumulative profit of all winning trades - ('loss_sum', lambda x: abs(x[x < 0].sum())), # cumulative loss of all losing trades - ('nb_win_trades', lambda x: x[x > 0].count()) # number of winning trades - ], - 'trade_duration': [('avg_trade_duration', 'mean')] - } - - # Group by (pair and stoploss) by applying above aggregator - df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( - groupby_aggregator).reset_index(col_level=1) - - # Dropping level 0 as we don't need it - df.columns = df.columns.droplevel(0) - - # Calculating number of losing trades, average win and average loss - df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades'] - df['average_win'] = df['profit_sum'] / df['nb_win_trades'] - df['average_loss'] = df['loss_sum'] / df['nb_loss_trades'] - - # Win rate = number of profitable trades / number of trades - df['winrate'] = df['nb_win_trades'] / df['nb_trades'] - - # risk_reward_ratio = average win / average loss - df['risk_reward_ratio'] = df['average_win'] / df['average_loss'] - - # required_risk_reward = (1 / winrate) - 1 - df['required_risk_reward'] = (1 / df['winrate']) - 1 - - # expectancy = (risk_reward_ratio * winrate) - (lossrate) - df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate']) - - # sort by expectancy and stoploss - df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( - 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() - - final = {} - for x in df.itertuples(): - final[x.pair] = PairInfo( - x.stoploss, - x.winrate, - x.risk_reward_ratio, - x.required_risk_reward, - x.expectancy, - x.nb_trades, - x.avg_trade_duration - ) - - # Returning a list of pairs in order of "expectancy" - return final - - def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): - buy_column = ticker_data['buy'].values - sell_column = ticker_data['sell'].values - date_column = ticker_data['date'].values - ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values - - result: list = [] - for stoploss in stoploss_range: - result += self._detect_next_stop_or_sell_point( - buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair - ) - - return result - - def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column, - ohlc_columns, stoploss, pair): - """ - Iterate through ohlc_columns in order to find the next trade - Next trade opens from the first buy signal noticed to - The sell or stoploss signal after it. - It then cuts OHLC, buy_column, sell_column and date_column. - Cut from (the exit trade index) + 1. - - Author: https://github.com/mishaker - """ - - result: list = [] - start_point = 0 - - while True: - open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) - - # Return empty if we don't find trade entry (i.e. buy==1) or - # we find a buy but at the end of array - if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: - break - else: - # When a buy signal is seen, - # trade opens in reality on the next candle - open_trade_index += 1 - - stop_price_percentage = stoploss + 1 - open_price = ohlc_columns[open_trade_index, 0] - stop_price = (open_price * stop_price_percentage) - - # Searching for the index where stoploss is hit - stop_index = utf1st.find_1st( - ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller) - - # If we don't find it then we assume stop_index will be far in future (infinite number) - if stop_index == -1: - stop_index = float('inf') - - # Searching for the index where sell is hit - sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal) - - # If we don't find it then we assume sell_index will be far in future (infinite number) - if sell_index == -1: - sell_index = float('inf') - - # Check if we don't find any stop or sell point (in that case trade remains open) - # It is not interesting for Edge to consider it so we simply ignore the trade - # And stop iterating there is no more entry - if stop_index == sell_index == float('inf'): - break - - if stop_index <= sell_index: - exit_index = open_trade_index + stop_index - exit_type = SellType.STOP_LOSS - exit_price = stop_price - elif stop_index > sell_index: - # If exit is SELL then we exit at the next candle - exit_index = open_trade_index + sell_index + 1 - - # Check if we have the next candle - if len(ohlc_columns) - 1 < exit_index: - break - - exit_type = SellType.SELL_SIGNAL - exit_price = ohlc_columns[exit_index, 0] - - trade = {'pair': pair, - 'stoploss': stoploss, - 'profit_percent': '', - 'profit_abs': '', - 'open_time': date_column[open_trade_index], - 'close_time': date_column[exit_index], - 'open_index': start_point + open_trade_index, - 'close_index': start_point + exit_index, - 'trade_duration': '', - 'open_rate': round(open_price, 15), - 'close_rate': round(exit_price, 15), - 'exit_type': exit_type - } - - result.append(trade) - - # Giving a view of exit_index till the end of array - buy_column = buy_column[exit_index:] - sell_column = sell_column[exit_index:] - date_column = date_column[exit_index:] - ohlc_columns = ohlc_columns[exit_index:] - start_point += exit_index - - return result +from .edge_positioning import Edge, PairInfo # noqa: F401 diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py new file mode 100644 index 000000000..15883357b --- /dev/null +++ b/freqtrade/edge/edge_positioning.py @@ -0,0 +1,464 @@ +# pragma pylint: disable=W0603 +""" Edge positioning package """ +import logging +from typing import Any, Dict, NamedTuple + +import arrow +import numpy as np +import utils_find_1st as utf1st +from pandas import DataFrame + +from freqtrade import constants +from freqtrade.configuration import TimeRange +from freqtrade.data import history +from freqtrade.exceptions import OperationalException +from freqtrade.strategy.interface import SellType + +logger = logging.getLogger(__name__) + + +class PairInfo(NamedTuple): + stoploss: float + winrate: float + risk_reward_ratio: float + required_risk_reward: float + expectancy: float + nb_trades: int + avg_trade_duration: float + + +class Edge: + """ + Calculates Win Rate, Risk Reward Ratio, Expectancy + against historical data for a give set of markets and a strategy + it then adjusts stoploss and position size accordingly + and force it into the strategy + Author: https://github.com/mishaker + """ + + config: Dict = {} + _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + + def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: + + self.config = config + self.exchange = exchange + self.strategy = strategy + + self.edge_config = self.config.get('edge', {}) + self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + self._final_pairs: list = [] + + # checking max_open_trades. it should be -1 as with Edge + # the number of trades is determined by position size + if self.config['max_open_trades'] != float('inf'): + logger.critical('max_open_trades should be -1 in config !') + + if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT: + raise OperationalException('Edge works only with unlimited stake amount') + + # Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future. + self._capital_percentage: float = self.edge_config.get( + 'capital_available_percentage', self.config['tradable_balance_ratio']) + self._allowed_risk: float = self.edge_config.get('allowed_risk') + self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) + self._last_updated: int = 0 # Timestamp of pairs last updated time + self._refresh_pairs = True + + self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) + self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) + self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) + + # calculating stoploss range + self._stoploss_range = np.arange( + self._stoploss_range_min, + self._stoploss_range_max, + self._stoploss_range_step + ) + + self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift( + days=-1 * self._since_number_of_days).format('YYYYMMDD')) + if config.get('fee'): + self.fee = config['fee'] + else: + self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0]) + + def calculate(self) -> bool: + pairs = self.config['exchange']['pair_whitelist'] + heartbeat = self.edge_config.get('process_throttle_secs') + + if (self._last_updated > 0) and ( + self._last_updated + heartbeat > arrow.utcnow().timestamp): + return False + + data: Dict[str, Any] = {} + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using local backtesting data (using whitelist in given config) ...') + + if self._refresh_pairs: + history.refresh_data( + datadir=self.config['datadir'], + pairs=pairs, + exchange=self.exchange, + timeframe=self.strategy.ticker_interval, + timerange=self._timerange, + ) + + data = history.load_data( + datadir=self.config['datadir'], + pairs=pairs, + timeframe=self.strategy.ticker_interval, + timerange=self._timerange, + startup_candles=self.strategy.startup_candle_count, + ) + + if not data: + # Reinitializing cached pairs + self._cached_pairs = {} + logger.critical("No data found. Edge is stopped ...") + return False + + preprocessed = self.strategy.tickerdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = history.get_timerange(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days) ...', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + + trades: list = [] + for pair, pair_data in preprocessed.items(): + # Sorting dataframe by date and reset index + pair_data = pair_data.sort_values(by=['date']) + pair_data = pair_data.reset_index(drop=True) + + ticker_data = self.strategy.advise_sell( + self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() + + trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range) + + # If no trade found then exit + if len(trades) == 0: + logger.info("No trades found.") + return False + + # Fill missing, calculable columns, profit, duration , abs etc. + trades_df = self._fill_calculable_fields(DataFrame(trades)) + self._cached_pairs = self._process_expectancy(trades_df) + self._last_updated = arrow.utcnow().timestamp + + return True + + def stake_amount(self, pair: str, free_capital: float, + total_capital: float, capital_in_trade: float) -> float: + stoploss = self.stoploss(pair) + available_capital = (total_capital + capital_in_trade) * self._capital_percentage + allowed_capital_at_risk = available_capital * self._allowed_risk + max_position_size = abs(allowed_capital_at_risk / stoploss) + position_size = min(max_position_size, free_capital) + if pair in self._cached_pairs: + logger.info( + 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' + ' capital in trade: %s, free capital: %s, total capital: %s,' + ' stoploss: %s, available capital: %s.', + self._cached_pairs[pair].winrate, + self._cached_pairs[pair].expectancy, + position_size, pair, + capital_in_trade, free_capital, total_capital, + stoploss, available_capital + ) + return round(position_size, 15) + + def stoploss(self, pair: str) -> float: + if pair in self._cached_pairs: + return self._cached_pairs[pair].stoploss + else: + logger.warning('tried to access stoploss of a non-existing pair, ' + 'strategy stoploss is returned instead.') + return self.strategy.stoploss + + def adjust(self, pairs) -> list: + """ + Filters out and sorts "pairs" according to Edge calculated pairs + """ + final = [] + for pair, info in self._cached_pairs.items(): + if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ + info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ + pair in pairs: + final.append(pair) + + if self._final_pairs != final: + self._final_pairs = final + if self._final_pairs: + logger.info( + 'Minimum expectancy and minimum winrate are met only for %s,' + ' so other pairs are filtered out.', + self._final_pairs + ) + else: + logger.info( + 'Edge removed all pairs as no pair with minimum expectancy ' + 'and minimum winrate was found !' + ) + + return self._final_pairs + + def accepted_pairs(self) -> list: + """ + return a list of accepted pairs along with their winrate, expectancy and stoploss + """ + final = [] + for pair, info in self._cached_pairs.items(): + if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ + info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)): + final.append({ + 'Pair': pair, + 'Winrate': info.winrate, + 'Expectancy': info.expectancy, + 'Stoploss': info.stoploss, + }) + return final + + def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: + """ + The result frame contains a number of columns that are calculable + from other columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + :param result Dataframe + :return: result Dataframe + """ + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + # fee = 0.001 + + # we set stake amount to an arbitrary amount. + # as it doesn't change the calculation. + # all returned values are relative. they are percentages. + stake = 0.015 + fee = self.fee + open_fee = fee / 2 + close_fee = fee / 2 + + result['trade_duration'] = result['close_time'] - result['open_time'] + + result['trade_duration'] = result['trade_duration'].map( + lambda x: int(x.total_seconds() / 60)) + + # Spends, Takes, Profit, Absolute Profit + + # Buy Price + result['buy_vol'] = stake / result['open_rate'] # How many target are we buying + result['buy_fee'] = stake * open_fee + result['buy_spend'] = stake + result['buy_fee'] # How much we're spending + + # Sell price + result['sell_sum'] = result['buy_vol'] * result['close_rate'] + result['sell_fee'] = result['sell_sum'] * close_fee + result['sell_take'] = result['sell_sum'] - result['sell_fee'] + + # profit_percent + result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend'] + + # Absolute profit + result['profit_abs'] = result['sell_take'] - result['buy_spend'] + + return result + + def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]: + """ + This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs + The calulation will be done per pair and per strategy. + """ + # Removing pairs having less than min_trades_number + min_trades_number = self.edge_config.get('min_trade_number', 10) + results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) + ################################### + + # Removing outliers (Only Pumps) from the dataset + # The method to detect outliers is to calculate standard deviation + # Then every value more than (standard deviation + 2*average) is out (pump) + # + # Removing Pumps + if self.edge_config.get('remove_pumps', False): + results = results.groupby(['pair', 'stoploss']).apply( + lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) + ########################################################################## + + # Removing trades having a duration more than X minutes (set in config) + max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440) + results = results[results.trade_duration < max_trade_duration] + ####################################################################### + + if results.empty: + return {} + + groupby_aggregator = { + 'profit_abs': [ + ('nb_trades', 'count'), # number of all trades + ('profit_sum', lambda x: x[x > 0].sum()), # cumulative profit of all winning trades + ('loss_sum', lambda x: abs(x[x < 0].sum())), # cumulative loss of all losing trades + ('nb_win_trades', lambda x: x[x > 0].count()) # number of winning trades + ], + 'trade_duration': [('avg_trade_duration', 'mean')] + } + + # Group by (pair and stoploss) by applying above aggregator + df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( + groupby_aggregator).reset_index(col_level=1) + + # Dropping level 0 as we don't need it + df.columns = df.columns.droplevel(0) + + # Calculating number of losing trades, average win and average loss + df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades'] + df['average_win'] = df['profit_sum'] / df['nb_win_trades'] + df['average_loss'] = df['loss_sum'] / df['nb_loss_trades'] + + # Win rate = number of profitable trades / number of trades + df['winrate'] = df['nb_win_trades'] / df['nb_trades'] + + # risk_reward_ratio = average win / average loss + df['risk_reward_ratio'] = df['average_win'] / df['average_loss'] + + # required_risk_reward = (1 / winrate) - 1 + df['required_risk_reward'] = (1 / df['winrate']) - 1 + + # expectancy = (risk_reward_ratio * winrate) - (lossrate) + df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate']) + + # sort by expectancy and stoploss + df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( + 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() + + final = {} + for x in df.itertuples(): + final[x.pair] = PairInfo( + x.stoploss, + x.winrate, + x.risk_reward_ratio, + x.required_risk_reward, + x.expectancy, + x.nb_trades, + x.avg_trade_duration + ) + + # Returning a list of pairs in order of "expectancy" + return final + + def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): + buy_column = ticker_data['buy'].values + sell_column = ticker_data['sell'].values + date_column = ticker_data['date'].values + ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values + + result: list = [] + for stoploss in stoploss_range: + result += self._detect_next_stop_or_sell_point( + buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair + ) + + return result + + def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column, + ohlc_columns, stoploss, pair): + """ + Iterate through ohlc_columns in order to find the next trade + Next trade opens from the first buy signal noticed to + The sell or stoploss signal after it. + It then cuts OHLC, buy_column, sell_column and date_column. + Cut from (the exit trade index) + 1. + + Author: https://github.com/mishaker + """ + + result: list = [] + start_point = 0 + + while True: + open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) + + # Return empty if we don't find trade entry (i.e. buy==1) or + # we find a buy but at the end of array + if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: + break + else: + # When a buy signal is seen, + # trade opens in reality on the next candle + open_trade_index += 1 + + stop_price_percentage = stoploss + 1 + open_price = ohlc_columns[open_trade_index, 0] + stop_price = (open_price * stop_price_percentage) + + # Searching for the index where stoploss is hit + stop_index = utf1st.find_1st( + ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller) + + # If we don't find it then we assume stop_index will be far in future (infinite number) + if stop_index == -1: + stop_index = float('inf') + + # Searching for the index where sell is hit + sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal) + + # If we don't find it then we assume sell_index will be far in future (infinite number) + if sell_index == -1: + sell_index = float('inf') + + # Check if we don't find any stop or sell point (in that case trade remains open) + # It is not interesting for Edge to consider it so we simply ignore the trade + # And stop iterating there is no more entry + if stop_index == sell_index == float('inf'): + break + + if stop_index <= sell_index: + exit_index = open_trade_index + stop_index + exit_type = SellType.STOP_LOSS + exit_price = stop_price + elif stop_index > sell_index: + # If exit is SELL then we exit at the next candle + exit_index = open_trade_index + sell_index + 1 + + # Check if we have the next candle + if len(ohlc_columns) - 1 < exit_index: + break + + exit_type = SellType.SELL_SIGNAL + exit_price = ohlc_columns[exit_index, 0] + + trade = {'pair': pair, + 'stoploss': stoploss, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': date_column[open_trade_index], + 'close_time': date_column[exit_index], + 'open_index': start_point + open_trade_index, + 'close_index': start_point + exit_index, + 'trade_duration': '', + 'open_rate': round(open_price, 15), + 'close_rate': round(exit_price, 15), + 'exit_type': exit_type + } + + result.append(trade) + + # Giving a view of exit_index till the end of array + buy_column = buy_column[exit_index:] + sell_column = sell_column[exit_index:] + date_column = date_column[exit_index:] + ohlc_columns = ohlc_columns[exit_index:] + start_point += exit_index + + return result From 80ed1c3e146da9c3f36f978ceae0df250acc4131 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 10:51:09 +0100 Subject: [PATCH 06/19] Move utils to commands --- freqtrade/commands/__init__.py | 0 freqtrade/{ => commands}/utils.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 freqtrade/commands/__init__.py rename freqtrade/{ => commands}/utils.py (100%) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/utils.py b/freqtrade/commands/utils.py similarity index 100% rename from freqtrade/utils.py rename to freqtrade/commands/utils.py From 6e852804671efa0546341b5db5f35b79bc9e9278 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 10:55:39 +0100 Subject: [PATCH 07/19] Adjust imports --- freqtrade/commands/__init__.py | 6 ++++++ freqtrade/configuration/arguments.py | 12 +++++------ freqtrade/optimize/__init__.py | 2 +- freqtrade/plot/plot_utils.py | 2 +- tests/test_utils.py | 30 ++++++++++++++-------------- 5 files changed, 29 insertions(+), 23 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index e69de29bb..0c31048f2 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -0,0 +1,6 @@ +from .utils import (setup_utils_configuration, start_create_userdir, # noqa: F401 + start_download_data, start_hyperopt_list, + start_hyperopt_show, start_list_exchanges, + start_list_markets, start_list_strategies, + start_list_timeframes, start_new_hyperopt, + start_new_strategy, start_test_pairlist, start_trading) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index b2197619d..3c71fa548 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -131,12 +131,12 @@ class Arguments: self._build_args(optionlist=['version'], parser=self.parser) from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge - from freqtrade.utils import (start_create_userdir, start_download_data, - start_hyperopt_list, start_hyperopt_show, - start_list_exchanges, start_list_markets, - start_list_strategies, start_new_hyperopt, - start_new_strategy, start_list_timeframes, - start_test_pairlist, start_trading) + from freqtrade.commands import (start_create_userdir, start_download_data, + start_hyperopt_list, start_hyperopt_show, + start_list_exchanges, start_list_markets, + start_list_strategies, start_new_hyperopt, + start_new_strategy, start_list_timeframes, + start_test_pairlist, start_trading) from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit subparsers = self.parser.add_subparsers(dest='command', diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 34760372f..98b521d4e 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -4,7 +4,7 @@ from typing import Any, Dict from freqtrade import constants from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.state import RunMode -from freqtrade.utils import setup_utils_configuration +from freqtrade.commands import setup_utils_configuration logger = logging.getLogger(__name__) diff --git a/freqtrade/plot/plot_utils.py b/freqtrade/plot/plot_utils.py index 9eff08396..b2f16dff0 100644 --- a/freqtrade/plot/plot_utils.py +++ b/freqtrade/plot/plot_utils.py @@ -2,7 +2,7 @@ from typing import Any, Dict from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.utils import setup_utils_configuration +from freqtrade.commands import setup_utils_configuration def validate_plot_args(args: Dict[str, Any]): diff --git a/tests/test_utils.py b/tests/test_utils.py index 7ca2cac5a..91009ec42 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,15 +4,15 @@ from unittest.mock import MagicMock, PropertyMock import pytest +from freqtrade.commands import (setup_utils_configuration, + start_create_userdir, start_download_data, + start_hyperopt_list, start_hyperopt_show, + start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, + start_new_hyperopt, start_new_strategy, + start_test_pairlist, start_trading) from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.utils import (setup_utils_configuration, start_create_userdir, - start_download_data, start_hyperopt_list, - start_hyperopt_show, start_list_exchanges, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_hyperopt, - start_new_strategy, start_test_pairlist, - start_trading) from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -451,8 +451,8 @@ def test_create_datadir(caplog, mocker): # Added assert here to analyze random test-failures ... assert len(caplog.record_tuples) == 0 - cud = mocker.patch("freqtrade.utils.create_userdata_dir", MagicMock()) - csf = mocker.patch("freqtrade.utils.copy_sample_files", MagicMock()) + cud = mocker.patch("freqtrade.commands.create_userdata_dir", MagicMock()) + csf = mocker.patch("freqtrade.commands.copy_sample_files", MagicMock()) args = [ "create-userdir", "--userdir", @@ -538,7 +538,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -556,7 +556,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker, id='binance') mocker.patch( @@ -574,7 +574,7 @@ def test_download_data_no_markets(mocker, caplog): def test_download_data_no_exchange(mocker, caplog): - mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -594,7 +594,7 @@ def test_download_data_no_pairs(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -613,9 +613,9 @@ def test_download_data_no_pairs(mocker, caplog): def test_download_data_trades(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_trades_data', + dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_trades_data', MagicMock(return_value=[])) - convert_mock = mocker.patch('freqtrade.utils.convert_trades_to_ohlcv', + convert_mock = mocker.patch('freqtrade.commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) mocker.patch( From 926bf07df1420c2e08dadbb53e8359a1ffa20900 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:08:58 +0100 Subject: [PATCH 08/19] Seperate a few commands into specific files --- freqtrade/commands/__init__.py | 16 +- freqtrade/commands/deploy_commands.py | 113 ++++++++ freqtrade/commands/hyperopt_commands.py | 114 ++++++++ freqtrade/commands/list_commands.py | 157 +++++++++++ freqtrade/commands/trade_commands.py | 25 ++ freqtrade/commands/utils.py | 356 ------------------------ 6 files changed, 419 insertions(+), 362 deletions(-) create mode 100644 freqtrade/commands/deploy_commands.py create mode 100644 freqtrade/commands/hyperopt_commands.py create mode 100644 freqtrade/commands/list_commands.py create mode 100644 freqtrade/commands/trade_commands.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 0c31048f2..4c57efeae 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,6 +1,10 @@ -from .utils import (setup_utils_configuration, start_create_userdir, # noqa: F401 - start_download_data, start_hyperopt_list, - start_hyperopt_show, start_list_exchanges, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_hyperopt, - start_new_strategy, start_test_pairlist, start_trading) +from .hyperopt_commands import (start_hyperopt_list, start_hyperopt_show) # noqa: 401 + +from .list_commands import (start_list_exchanges, # noqa: F401 + start_list_markets, start_list_strategies, + start_list_timeframes) +from .utils import setup_utils_configuration # noqa: F401 +from .utils import (start_download_data, # noqa: F401 + start_test_pairlist) +from .deploy_commands import (start_new_hyperopt, start_new_strategy, start_create_userdir) # noqa: F401 +from .trade_commands import start_trading # noqa: F401 diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py new file mode 100644 index 000000000..892e15db1 --- /dev/null +++ b/freqtrade/commands/deploy_commands.py @@ -0,0 +1,113 @@ +import logging +import sys +from pathlib import Path +from typing import Any, Dict + +from freqtrade.configuration.directory_operations import (copy_sample_files, + create_userdata_dir) +from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY +from freqtrade.exceptions import OperationalException +from freqtrade.misc import render_template +from freqtrade.state import RunMode + +from .utils import setup_utils_configuration + +logger = logging.getLogger(__name__) + + +def start_create_userdir(args: Dict[str, Any]) -> None: + """ + Create "user_data" directory to contain user data strategies, hyperopt, ...) + :param args: Cli args from Arguments() + :return: None + """ + if "user_data_dir" in args and args["user_data_dir"]: + userdir = create_userdata_dir(args["user_data_dir"], create_dir=True) + copy_sample_files(userdir, overwrite=args["reset"]) + else: + logger.warning("`create-userdir` requires --userdir to be set.") + sys.exit(1) + + +def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str): + """ + Deploy new strategy from template to strategy_path + """ + indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",) + buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",) + sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",) + plot_config = render_template(templatefile=f"subtemplates/plot_config_{subtemplate}.j2",) + + strategy_text = render_template(templatefile='base_strategy.py.j2', + arguments={"strategy": strategy_name, + "indicators": indicators, + "buy_trend": buy_trend, + "sell_trend": sell_trend, + "plot_config": plot_config, + }) + + logger.info(f"Writing strategy to `{strategy_path}`.") + strategy_path.write_text(strategy_text) + + +def start_new_strategy(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if "strategy" in args and args["strategy"]: + if args["strategy"] == "DefaultStrategy": + raise OperationalException("DefaultStrategy is not allowed as name.") + + new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py") + + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another Strategy Name.") + + deploy_new_strategy(args['strategy'], new_path, args['template']) + + else: + raise OperationalException("`new-strategy` requires --strategy to be set.") + + +def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str): + """ + Deploys a new hyperopt template to hyperopt_path + """ + buy_guards = render_template( + templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",) + sell_guards = render_template( + templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",) + buy_space = render_template( + templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",) + sell_space = render_template( + templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",) + + strategy_text = render_template(templatefile='base_hyperopt.py.j2', + arguments={"hyperopt": hyperopt_name, + "buy_guards": buy_guards, + "sell_guards": sell_guards, + "buy_space": buy_space, + "sell_space": sell_space, + }) + + logger.info(f"Writing hyperopt to `{hyperopt_path}`.") + hyperopt_path.write_text(strategy_text) + + +def start_new_hyperopt(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if "hyperopt" in args and args["hyperopt"]: + if args["hyperopt"] == "DefaultHyperopt": + raise OperationalException("DefaultHyperopt is not allowed as name.") + + new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py") + + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another Strategy Name.") + deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) + else: + raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py new file mode 100644 index 000000000..db3f69050 --- /dev/null +++ b/freqtrade/commands/hyperopt_commands.py @@ -0,0 +1,114 @@ +import logging +from operator import itemgetter +from typing import Any, Dict, List + +from colorama import init as colorama_init + +from .utils import setup_utils_configuration +from freqtrade.exceptions import OperationalException +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_hyperopt_list(args: Dict[str, Any]) -> None: + """ + List hyperopt epochs previously evaluated + """ + from freqtrade.optimize.hyperopt import Hyperopt + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + only_best = config.get('hyperopt_list_best', False) + only_profitable = config.get('hyperopt_list_profitable', False) + print_colorized = config.get('print_colorized', False) + print_json = config.get('print_json', False) + no_details = config.get('hyperopt_list_no_details', False) + no_header = False + + trials_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + + # Previous evaluations + trials = Hyperopt.load_previous_results(trials_file) + total_epochs = len(trials) + + trials = _hyperopt_filter_trials(trials, only_best, only_profitable) + + # TODO: fetch the interval for epochs to print from the cli option + epoch_start, epoch_stop = 0, None + + if print_colorized: + colorama_init(autoreset=True) + + try: + # Human-friendly indexes used here (starting from 1) + for val in trials[epoch_start:epoch_stop]: + Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized) + + except KeyboardInterrupt: + print('User interrupted..') + + if trials and not no_details: + sorted_trials = sorted(trials, key=itemgetter('loss')) + results = sorted_trials[0] + Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header) + + +def start_hyperopt_show(args: Dict[str, Any]) -> None: + """ + Show details of a hyperopt epoch previously evaluated + """ + from freqtrade.optimize.hyperopt import Hyperopt + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + only_best = config.get('hyperopt_list_best', False) + only_profitable = config.get('hyperopt_list_profitable', False) + no_header = config.get('hyperopt_show_no_header', False) + + trials_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + + # Previous evaluations + trials = Hyperopt.load_previous_results(trials_file) + total_epochs = len(trials) + + trials = _hyperopt_filter_trials(trials, only_best, only_profitable) + trials_epochs = len(trials) + + n = config.get('hyperopt_show_index', -1) + if n > trials_epochs: + raise OperationalException( + f"The index of the epoch to show should be less than {trials_epochs + 1}.") + if n < -trials_epochs: + raise OperationalException( + f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") + + # Translate epoch index from human-readable format to pythonic + if n > 0: + n -= 1 + + print_json = config.get('print_json', False) + + if trials: + val = trials[n] + Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, + header_str="Epoch details") + + +def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List: + """ + Filter our items from the list of hyperopt results + """ + if only_best: + trials = [x for x in trials if x['is_best']] + if only_profitable: + trials = [x for x in trials if x['results_metrics']['profit'] > 0] + + logger.info(f"{len(trials)} " + + ("best " if only_best else "") + + ("profitable " if only_profitable else "") + + "epochs found.") + + return trials diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py new file mode 100644 index 000000000..90efaf66f --- /dev/null +++ b/freqtrade/commands/list_commands.py @@ -0,0 +1,157 @@ +import csv +import logging +import sys +from collections import OrderedDict +from pathlib import Path +from typing import Any, Dict + +import rapidjson +from tabulate import tabulate + +from freqtrade.constants import USERPATH_STRATEGY +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import (available_exchanges, ccxt_exchanges, + market_is_active, symbol_is_pair) +from freqtrade.misc import plural +from freqtrade.resolvers import ExchangeResolver, StrategyResolver +from freqtrade.state import RunMode + +from .utils import setup_utils_configuration + +logger = logging.getLogger(__name__) + + +def start_list_exchanges(args: Dict[str, Any]) -> None: + """ + Print available exchanges + :param args: Cli args from Arguments() + :return: None + """ + exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges() + if args['print_one_column']: + print('\n'.join(exchanges)) + else: + if args['list_exchanges_all']: + print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}") + else: + print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") + + +def start_list_strategies(args: Dict[str, Any]) -> None: + """ + Print Strategies available in a directory + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY)) + strategies = StrategyResolver.search_all_objects(directory) + # Sort alphabetically + strategies = sorted(strategies, key=lambda x: x['name']) + strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies] + + if args['print_one_column']: + print('\n'.join([s['name'] for s in strategies])) + else: + print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) + + +def start_list_timeframes(args: Dict[str, Any]) -> None: + """ + Print ticker intervals (timeframes) available on Exchange + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + # Do not use ticker_interval set in the config + config['ticker_interval'] = None + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + if args['print_one_column']: + print('\n'.join(exchange.timeframes)) + else: + print(f"Timeframes available for the exchange `{exchange.name}`: " + f"{', '.join(exchange.timeframes)}") + + +def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: + """ + Print pairs/markets on the exchange + :param args: Cli args from Arguments() + :param pairs_only: if True print only pairs, otherwise print all instruments (markets) + :return: None + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + # By default only active pairs/markets are to be shown + active_only = not args.get('list_pairs_all', False) + + base_currencies = args.get('base_currencies', []) + quote_currencies = args.get('quote_currencies', []) + + try: + pairs = exchange.get_markets(base_currencies=base_currencies, + quote_currencies=quote_currencies, + pairs_only=pairs_only, + active_only=active_only) + # Sort the pairs/markets by symbol + pairs = OrderedDict(sorted(pairs.items())) + except Exception as e: + raise OperationalException(f"Cannot get markets. Reason: {e}") from e + + else: + summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") + + ("active " if active_only else "") + + (plural(len(pairs), "pair" if pairs_only else "market")) + + (f" with {', '.join(base_currencies)} as base " + f"{plural(len(base_currencies), 'currency', 'currencies')}" + if base_currencies else "") + + (" and" if base_currencies and quote_currencies else "") + + (f" with {', '.join(quote_currencies)} as quote " + f"{plural(len(quote_currencies), 'currency', 'currencies')}" + if quote_currencies else "")) + + headers = ["Id", "Symbol", "Base", "Quote", "Active", + *(['Is pair'] if not pairs_only else [])] + + tabular_data = [] + for _, v in pairs.items(): + tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], + 'Base': v['base'], 'Quote': v['quote'], + 'Active': market_is_active(v), + **({'Is pair': symbol_is_pair(v['symbol'])} + if not pairs_only else {})}) + + if (args.get('print_one_column', False) or + args.get('list_pairs_print_json', False) or + args.get('print_csv', False)): + # Print summary string in the log in case of machine-readable + # regular formats. + logger.info(f"{summary_str}.") + else: + # Print empty string separating leading logs and output in case of + # human-readable formats. + print() + + if len(pairs): + if args.get('print_list', False): + # print data as a list, with human-readable summary + print(f"{summary_str}: {', '.join(pairs.keys())}.") + elif args.get('print_one_column', False): + print('\n'.join(pairs.keys())) + elif args.get('list_pairs_print_json', False): + print(rapidjson.dumps(list(pairs.keys()), default=str)) + elif args.get('print_csv', False): + writer = csv.DictWriter(sys.stdout, fieldnames=headers) + writer.writeheader() + writer.writerows(tabular_data) + else: + # print data as a table, with the human-readable summary + print(f"{summary_str}:") + print(tabulate(tabular_data, headers='keys', tablefmt='pipe')) + elif not (args.get('print_one_column', False) or + args.get('list_pairs_print_json', False) or + args.get('print_csv', False)): + print(f"{summary_str}.") diff --git a/freqtrade/commands/trade_commands.py b/freqtrade/commands/trade_commands.py new file mode 100644 index 000000000..2c0c4c9c1 --- /dev/null +++ b/freqtrade/commands/trade_commands.py @@ -0,0 +1,25 @@ +import logging + +from typing import Any, Dict + + +logger = logging.getLogger(__name__) + + +def start_trading(args: Dict[str, Any]) -> int: + """ + Main entry point for trading mode + """ + from freqtrade.worker import Worker + # Load and run worker + worker = None + try: + worker = Worker(args) + worker.run() + except KeyboardInterrupt: + logger.info('SIGINT received, aborting ...') + finally: + if worker: + logger.info("worker found ... calling exit") + worker.exit() + return 0 diff --git a/freqtrade/commands/utils.py b/freqtrade/commands/utils.py index 2f7b2d717..ba9173cf2 100644 --- a/freqtrade/commands/utils.py +++ b/freqtrade/commands/utils.py @@ -46,139 +46,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str return config -def start_trading(args: Dict[str, Any]) -> int: - """ - Main entry point for trading mode - """ - from freqtrade.worker import Worker - # Load and run worker - worker = None - try: - worker = Worker(args) - worker.run() - except KeyboardInterrupt: - logger.info('SIGINT received, aborting ...') - finally: - if worker: - logger.info("worker found ... calling exit") - worker.exit() - return 0 - - -def start_list_exchanges(args: Dict[str, Any]) -> None: - """ - Print available exchanges - :param args: Cli args from Arguments() - :return: None - """ - exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges() - if args['print_one_column']: - print('\n'.join(exchanges)) - else: - if args['list_exchanges_all']: - print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}") - else: - print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") - - -def start_create_userdir(args: Dict[str, Any]) -> None: - """ - Create "user_data" directory to contain user data strategies, hyperopt, ...) - :param args: Cli args from Arguments() - :return: None - """ - if "user_data_dir" in args and args["user_data_dir"]: - userdir = create_userdata_dir(args["user_data_dir"], create_dir=True) - copy_sample_files(userdir, overwrite=args["reset"]) - else: - logger.warning("`create-userdir` requires --userdir to be set.") - sys.exit(1) - - -def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str): - """ - Deploy new strategy from template to strategy_path - """ - indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",) - buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",) - sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",) - plot_config = render_template(templatefile=f"subtemplates/plot_config_{subtemplate}.j2",) - - strategy_text = render_template(templatefile='base_strategy.py.j2', - arguments={"strategy": strategy_name, - "indicators": indicators, - "buy_trend": buy_trend, - "sell_trend": sell_trend, - "plot_config": plot_config, - }) - - logger.info(f"Writing strategy to `{strategy_path}`.") - strategy_path.write_text(strategy_text) - - -def start_new_strategy(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if "strategy" in args and args["strategy"]: - if args["strategy"] == "DefaultStrategy": - raise OperationalException("DefaultStrategy is not allowed as name.") - - new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py") - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Strategy Name.") - - deploy_new_strategy(args['strategy'], new_path, args['template']) - - else: - raise OperationalException("`new-strategy` requires --strategy to be set.") - - -def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str): - """ - Deploys a new hyperopt template to hyperopt_path - """ - buy_guards = render_template( - templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",) - sell_guards = render_template( - templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",) - buy_space = render_template( - templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",) - sell_space = render_template( - templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",) - - strategy_text = render_template(templatefile='base_hyperopt.py.j2', - arguments={"hyperopt": hyperopt_name, - "buy_guards": buy_guards, - "sell_guards": sell_guards, - "buy_space": buy_space, - "sell_space": sell_space, - }) - - logger.info(f"Writing hyperopt to `{hyperopt_path}`.") - hyperopt_path.write_text(strategy_text) - - -def start_new_hyperopt(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if "hyperopt" in args and args["hyperopt"]: - if args["hyperopt"] == "DefaultHyperopt": - raise OperationalException("DefaultHyperopt is not allowed as name.") - - new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py") - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Strategy Name.") - deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) - else: - raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") - - def start_download_data(args: Dict[str, Any]) -> None: """ Download data (former download_backtest_data.py script) @@ -227,126 +94,6 @@ def start_download_data(args: Dict[str, Any]) -> None: f"on exchange {exchange.name}.") -def start_list_strategies(args: Dict[str, Any]) -> None: - """ - Print Strategies available in a directory - """ - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY)) - strategies = StrategyResolver.search_all_objects(directory) - # Sort alphabetically - strategies = sorted(strategies, key=lambda x: x['name']) - strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies] - - if args['print_one_column']: - print('\n'.join([s['name'] for s in strategies])) - else: - print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) - - -def start_list_timeframes(args: Dict[str, Any]) -> None: - """ - Print ticker intervals (timeframes) available on Exchange - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - # Do not use ticker_interval set in the config - config['ticker_interval'] = None - - # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) - - if args['print_one_column']: - print('\n'.join(exchange.timeframes)) - else: - print(f"Timeframes available for the exchange `{exchange.name}`: " - f"{', '.join(exchange.timeframes)}") - - -def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: - """ - Print pairs/markets on the exchange - :param args: Cli args from Arguments() - :param pairs_only: if True print only pairs, otherwise print all instruments (markets) - :return: None - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) - - # By default only active pairs/markets are to be shown - active_only = not args.get('list_pairs_all', False) - - base_currencies = args.get('base_currencies', []) - quote_currencies = args.get('quote_currencies', []) - - try: - pairs = exchange.get_markets(base_currencies=base_currencies, - quote_currencies=quote_currencies, - pairs_only=pairs_only, - active_only=active_only) - # Sort the pairs/markets by symbol - pairs = OrderedDict(sorted(pairs.items())) - except Exception as e: - raise OperationalException(f"Cannot get markets. Reason: {e}") from e - - else: - summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") + - ("active " if active_only else "") + - (plural(len(pairs), "pair" if pairs_only else "market")) + - (f" with {', '.join(base_currencies)} as base " - f"{plural(len(base_currencies), 'currency', 'currencies')}" - if base_currencies else "") + - (" and" if base_currencies and quote_currencies else "") + - (f" with {', '.join(quote_currencies)} as quote " - f"{plural(len(quote_currencies), 'currency', 'currencies')}" - if quote_currencies else "")) - - headers = ["Id", "Symbol", "Base", "Quote", "Active", - *(['Is pair'] if not pairs_only else [])] - - tabular_data = [] - for _, v in pairs.items(): - tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], - 'Base': v['base'], 'Quote': v['quote'], - 'Active': market_is_active(v), - **({'Is pair': symbol_is_pair(v['symbol'])} - if not pairs_only else {})}) - - if (args.get('print_one_column', False) or - args.get('list_pairs_print_json', False) or - args.get('print_csv', False)): - # Print summary string in the log in case of machine-readable - # regular formats. - logger.info(f"{summary_str}.") - else: - # Print empty string separating leading logs and output in case of - # human-readable formats. - print() - - if len(pairs): - if args.get('print_list', False): - # print data as a list, with human-readable summary - print(f"{summary_str}: {', '.join(pairs.keys())}.") - elif args.get('print_one_column', False): - print('\n'.join(pairs.keys())) - elif args.get('list_pairs_print_json', False): - print(rapidjson.dumps(list(pairs.keys()), default=str)) - elif args.get('print_csv', False): - writer = csv.DictWriter(sys.stdout, fieldnames=headers) - writer.writeheader() - writer.writerows(tabular_data) - else: - # print data as a table, with the human-readable summary - print(f"{summary_str}:") - print(tabulate(tabular_data, headers='keys', tablefmt='pipe')) - elif not (args.get('print_one_column', False) or - args.get('list_pairs_print_json', False) or - args.get('print_csv', False)): - print(f"{summary_str}.") - - def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration @@ -377,106 +124,3 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: print(rapidjson.dumps(list(pairlist), default=str)) else: print(pairlist) - - -def start_hyperopt_list(args: Dict[str, Any]) -> None: - """ - List hyperopt epochs previously evaluated - """ - from freqtrade.optimize.hyperopt import Hyperopt - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - only_best = config.get('hyperopt_list_best', False) - only_profitable = config.get('hyperopt_list_profitable', False) - print_colorized = config.get('print_colorized', False) - print_json = config.get('print_json', False) - no_details = config.get('hyperopt_list_no_details', False) - no_header = False - - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') - - # Previous evaluations - trials = Hyperopt.load_previous_results(trials_file) - total_epochs = len(trials) - - trials = _hyperopt_filter_trials(trials, only_best, only_profitable) - - # TODO: fetch the interval for epochs to print from the cli option - epoch_start, epoch_stop = 0, None - - if print_colorized: - colorama_init(autoreset=True) - - try: - # Human-friendly indexes used here (starting from 1) - for val in trials[epoch_start:epoch_stop]: - Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized) - - except KeyboardInterrupt: - print('User interrupted..') - - if trials and not no_details: - sorted_trials = sorted(trials, key=itemgetter('loss')) - results = sorted_trials[0] - Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header) - - -def start_hyperopt_show(args: Dict[str, Any]) -> None: - """ - Show details of a hyperopt epoch previously evaluated - """ - from freqtrade.optimize.hyperopt import Hyperopt - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - only_best = config.get('hyperopt_list_best', False) - only_profitable = config.get('hyperopt_list_profitable', False) - no_header = config.get('hyperopt_show_no_header', False) - - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') - - # Previous evaluations - trials = Hyperopt.load_previous_results(trials_file) - total_epochs = len(trials) - - trials = _hyperopt_filter_trials(trials, only_best, only_profitable) - trials_epochs = len(trials) - - n = config.get('hyperopt_show_index', -1) - if n > trials_epochs: - raise OperationalException( - f"The index of the epoch to show should be less than {trials_epochs + 1}.") - if n < -trials_epochs: - raise OperationalException( - f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") - - # Translate epoch index from human-readable format to pythonic - if n > 0: - n -= 1 - - print_json = config.get('print_json', False) - - if trials: - val = trials[n] - Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, - header_str="Epoch details") - - -def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List: - """ - Filter our items from the list of hyperopt results - """ - if only_best: - trials = [x for x in trials if x['is_best']] - if only_profitable: - trials = [x for x in trials if x['results_metrics']['profit'] > 0] - - logger.info(f"{len(trials)} " + - ("best " if only_best else "") + - ("profitable " if only_profitable else "") + - "epochs found.") - - return trials From 7e233041877d7c4976735e10269a2f3421c530f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:09:13 +0100 Subject: [PATCH 09/19] Adjust tests to new paths --- tests/test_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 91009ec42..c686d2117 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -451,8 +451,8 @@ def test_create_datadir(caplog, mocker): # Added assert here to analyze random test-failures ... assert len(caplog.record_tuples) == 0 - cud = mocker.patch("freqtrade.commands.create_userdata_dir", MagicMock()) - csf = mocker.patch("freqtrade.commands.copy_sample_files", MagicMock()) + cud = mocker.patch("freqtrade.commands.deploy_commands.create_userdata_dir", MagicMock()) + csf = mocker.patch("freqtrade.commands.deploy_commands.copy_sample_files", MagicMock()) args = [ "create-userdir", "--userdir", @@ -538,7 +538,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -556,7 +556,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker, id='binance') mocker.patch( @@ -574,7 +574,7 @@ def test_download_data_no_markets(mocker, caplog): def test_download_data_no_exchange(mocker, caplog): - mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -594,7 +594,7 @@ def test_download_data_no_pairs(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -613,9 +613,9 @@ def test_download_data_no_pairs(mocker, caplog): def test_download_data_trades(mocker, caplog): - dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_trades_data', + dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_trades_data', MagicMock(return_value=[])) - convert_mock = mocker.patch('freqtrade.commands.convert_trades_to_ohlcv', + convert_mock = mocker.patch('freqtrade.commands.utils.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) mocker.patch( From 70a0346b0a2b781dda480dc1a00da9215ac80eea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:17:26 +0100 Subject: [PATCH 10/19] Move data-stuff to data-commands --- freqtrade/commands/__init__.py | 16 +++---- freqtrade/commands/data_commands.py | 64 +++++++++++++++++++++++++ freqtrade/commands/utils.py | 73 ++--------------------------- tests/test_utils.py | 12 ++--- 4 files changed, 81 insertions(+), 84 deletions(-) create mode 100644 freqtrade/commands/data_commands.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 4c57efeae..5fa667ccc 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,10 +1,10 @@ -from .hyperopt_commands import (start_hyperopt_list, start_hyperopt_show) # noqa: 401 - -from .list_commands import (start_list_exchanges, # noqa: F401 +from freqtrade.commands.data_commands import start_download_data # noqa: F401 +from freqtrade.commands.deploy_commands import (start_create_userdir, # noqa: F401 + start_new_hyperopt, start_new_strategy) +from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, # noqa: F401 + start_hyperopt_show) +from freqtrade.commands.list_commands import (start_list_exchanges, # noqa: F401 start_list_markets, start_list_strategies, start_list_timeframes) -from .utils import setup_utils_configuration # noqa: F401 -from .utils import (start_download_data, # noqa: F401 - start_test_pairlist) -from .deploy_commands import (start_new_hyperopt, start_new_strategy, start_create_userdir) # noqa: F401 -from .trade_commands import start_trading # noqa: F401 +from freqtrade.commands.trade_commands import start_trading # noqa: F401 +from freqtrade.commands.utils import setup_utils_configuration, start_test_pairlist # noqa: F401 diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py new file mode 100644 index 000000000..b1c847d1d --- /dev/null +++ b/freqtrade/commands/data_commands.py @@ -0,0 +1,64 @@ +import logging +import sys +from typing import Any, Dict, List + +import arrow + +from freqtrade.configuration import TimeRange +from freqtrade.data.history import (convert_trades_to_ohlcv, + refresh_backtest_ohlcv_data, + refresh_backtest_trades_data) +from freqtrade.exceptions import OperationalException +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode +from .utils import setup_utils_configuration + +logger = logging.getLogger(__name__) + + +def start_download_data(args: Dict[str, Any]) -> None: + """ + Download data (former download_backtest_data.py script) + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + timerange = TimeRange() + if 'days' in config: + time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + + logger.info(f'About to download pairs: {config["pairs"]}, ' + f'intervals: {config["timeframes"]} to {config["datadir"]}') + + pairs_not_available: List[str] = [] + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) + try: + + if config.get('download_trades'): + pairs_not_available = refresh_backtest_trades_data( + exchange, pairs=config["pairs"], datadir=config['datadir'], + timerange=timerange, erase=config.get("erase")) + + # Convert downloaded trade data to different timeframes + convert_trades_to_ohlcv( + pairs=config["pairs"], timeframes=config["timeframes"], + datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) + else: + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=config["pairs"], timeframes=config["timeframes"], + datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) + + except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + + finally: + if pairs_not_available: + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {exchange.name}.") diff --git a/freqtrade/commands/utils.py b/freqtrade/commands/utils.py index ba9173cf2..a597d5335 100644 --- a/freqtrade/commands/utils.py +++ b/freqtrade/commands/utils.py @@ -1,30 +1,11 @@ -import csv import logging -import sys -from collections import OrderedDict -from operator import itemgetter -from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict -import arrow import rapidjson -from colorama import init as colorama_init -from tabulate import tabulate -from freqtrade.configuration import (Configuration, TimeRange, - remove_credentials, +from freqtrade.configuration import (Configuration, remove_credentials, validate_config_consistency) -from freqtrade.configuration.directory_operations import (copy_sample_files, - create_userdata_dir) -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY -from freqtrade.data.history import (convert_trades_to_ohlcv, - refresh_backtest_ohlcv_data, - refresh_backtest_trades_data) -from freqtrade.exceptions import OperationalException -from freqtrade.exchange import (available_exchanges, ccxt_exchanges, - market_is_active, symbol_is_pair) -from freqtrade.misc import plural, render_template -from freqtrade.resolvers import ExchangeResolver, StrategyResolver +from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -46,54 +27,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str return config -def start_download_data(args: Dict[str, Any]) -> None: - """ - Download data (former download_backtest_data.py script) - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - timerange = TimeRange() - if 'days' in config: - time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") - timerange = TimeRange.parse_timerange(f'{time_since}-') - - if 'pairs' not in config: - raise OperationalException( - "Downloading data requires a list of pairs. " - "Please check the documentation on how to configure this.") - - logger.info(f'About to download pairs: {config["pairs"]}, ' - f'intervals: {config["timeframes"]} to {config["datadir"]}') - - pairs_not_available: List[str] = [] - - # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) - try: - - if config.get('download_trades'): - pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=config["pairs"], datadir=config['datadir'], - timerange=timerange, erase=config.get("erase")) - - # Convert downloaded trade data to different timeframes - convert_trades_to_ohlcv( - pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) - else: - pairs_not_available = refresh_backtest_ohlcv_data( - exchange, pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) - - except KeyboardInterrupt: - sys.exit("SIGINT received, aborting ...") - - finally: - if pairs_not_available: - logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {exchange.name}.") - - def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration diff --git a/tests/test_utils.py b/tests/test_utils.py index c686d2117..1328e3981 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -538,7 +538,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -556,7 +556,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker, id='binance') mocker.patch( @@ -574,7 +574,7 @@ def test_download_data_no_markets(mocker, caplog): def test_download_data_no_exchange(mocker, caplog): - mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -594,7 +594,7 @@ def test_download_data_no_pairs(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -613,9 +613,9 @@ def test_download_data_no_pairs(mocker, caplog): def test_download_data_trades(mocker, caplog): - dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_trades_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_trades_data', MagicMock(return_value=[])) - convert_mock = mocker.patch('freqtrade.commands.utils.convert_trades_to_ohlcv', + convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) mocker.patch( From b254bdfea31c95b237b2642c6b55050d1e727b45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:20:34 +0100 Subject: [PATCH 11/19] Move plot_utils to plot_commands --- freqtrade/commands/__init__.py | 1 + freqtrade/{plot/plot_utils.py => commands/plot_commands.py} | 2 +- freqtrade/configuration/arguments.py | 2 +- tests/test_plotting.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) rename freqtrade/{plot/plot_utils.py => commands/plot_commands.py} (94%) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 5fa667ccc..45088e8a3 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -6,5 +6,6 @@ from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, # noqa: from freqtrade.commands.list_commands import (start_list_exchanges, # noqa: F401 start_list_markets, start_list_strategies, start_list_timeframes) +from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit # noqa: F401 from freqtrade.commands.trade_commands import start_trading # noqa: F401 from freqtrade.commands.utils import setup_utils_configuration, start_test_pairlist # noqa: F401 diff --git a/freqtrade/plot/plot_utils.py b/freqtrade/commands/plot_commands.py similarity index 94% rename from freqtrade/plot/plot_utils.py rename to freqtrade/commands/plot_commands.py index b2f16dff0..4c68cc3c5 100644 --- a/freqtrade/plot/plot_utils.py +++ b/freqtrade/commands/plot_commands.py @@ -2,7 +2,7 @@ from typing import Any, Dict from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.commands import setup_utils_configuration +from freqtrade.commands.utils import setup_utils_configuration def validate_plot_args(args: Dict[str, Any]): diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 3c71fa548..c61e4ec13 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -136,8 +136,8 @@ class Arguments: start_list_exchanges, start_list_markets, start_list_strategies, start_new_hyperopt, start_new_strategy, start_list_timeframes, + start_plot_dataframe, start_plot_profit, start_test_pairlist, start_trading) - from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 78c01eb97..e7ec4ce46 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -11,7 +11,7 @@ from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data from freqtrade.exceptions import OperationalException -from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit +from freqtrade.commands import start_plot_dataframe, start_plot_profit from freqtrade.plot.plotting import (add_indicators, add_profit, create_plotconfig, generate_candlestick_graph, From e033df6a2fce1d50333e70e599cd7dfde187b162 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:32:52 +0100 Subject: [PATCH 12/19] Move optimize_commands to commands module --- .../__init__.py => commands/optimize_commands.py} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename freqtrade/{optimize/__init__.py => commands/optimize_commands.py} (89%) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/commands/optimize_commands.py similarity index 89% rename from freqtrade/optimize/__init__.py rename to freqtrade/commands/optimize_commands.py index 98b521d4e..e635e61fd 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/commands/optimize_commands.py @@ -2,14 +2,14 @@ import logging from typing import Any, Dict from freqtrade import constants +from freqtrade.commands.utils import setup_utils_configuration from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.state import RunMode -from freqtrade.commands import setup_utils_configuration logger = logging.getLogger(__name__) -def setup_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: +def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: """ Prepare the configuration for the Hyperopt module :param args: Cli args from Arguments() @@ -35,7 +35,7 @@ def start_backtesting(args: Dict[str, Any]) -> None: from freqtrade.optimize.backtesting import Backtesting # Initialize configuration - config = setup_configuration(args, RunMode.BACKTEST) + config = setup_optimize_configuration(args, RunMode.BACKTEST) logger.info('Starting freqtrade in Backtesting mode') @@ -58,7 +58,7 @@ def start_hyperopt(args: Dict[str, Any]) -> None: raise OperationalException( f"{e}. Please ensure that the hyperopt dependencies are installed.") from e # Initialize configuration - config = setup_configuration(args, RunMode.HYPEROPT) + config = setup_optimize_configuration(args, RunMode.HYPEROPT) logger.info('Starting freqtrade in Hyperopt mode') @@ -94,7 +94,7 @@ def start_edge(args: Dict[str, Any]) -> None: """ from freqtrade.optimize.edge_cli import EdgeCli # Initialize configuration - config = setup_configuration(args, RunMode.EDGE) + config = setup_optimize_configuration(args, RunMode.EDGE) logger.info('Starting freqtrade in Edge mode') # Initialize Edge object From f347e5934a6be5d515c983daac4755ee3a8ec9ca Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:33:13 +0100 Subject: [PATCH 13/19] Small adjustments for moved commands --- freqtrade/commands/__init__.py | 30 ++++++++++++++++++---------- freqtrade/configuration/arguments.py | 2 +- freqtrade/optimize/__init__.py | 0 tests/optimize/test_backtesting.py | 12 +++++------ tests/optimize/test_edge_cli.py | 8 ++++---- tests/optimize/test_hyperopt.py | 8 ++++---- tests/test_main.py | 4 ++-- 7 files changed, 36 insertions(+), 28 deletions(-) create mode 100644 freqtrade/optimize/__init__.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 45088e8a3..8339486fb 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,11 +1,19 @@ -from freqtrade.commands.data_commands import start_download_data # noqa: F401 -from freqtrade.commands.deploy_commands import (start_create_userdir, # noqa: F401 - start_new_hyperopt, start_new_strategy) -from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, # noqa: F401 - start_hyperopt_show) -from freqtrade.commands.list_commands import (start_list_exchanges, # noqa: F401 - start_list_markets, start_list_strategies, - start_list_timeframes) -from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit # noqa: F401 -from freqtrade.commands.trade_commands import start_trading # noqa: F401 -from freqtrade.commands.utils import setup_utils_configuration, start_test_pairlist # noqa: F401 +# flake8: noqa: F401 + +from freqtrade.commands.data_commands import start_download_data +from freqtrade.commands.deploy_commands import (start_create_userdir, + start_new_hyperopt, + start_new_strategy) +from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, + start_hyperopt_show) +from freqtrade.commands.list_commands import (start_list_exchanges, + start_list_markets, + start_list_strategies, + start_list_timeframes) +from freqtrade.commands.optimize_commands import (start_backtesting, + start_edge, start_hyperopt) +from freqtrade.commands.plot_commands import (start_plot_dataframe, + start_plot_profit) +from freqtrade.commands.trade_commands import start_trading +from freqtrade.commands.utils import (setup_utils_configuration, + start_test_pairlist) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index c61e4ec13..21800e821 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -130,13 +130,13 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge from freqtrade.commands import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, start_list_strategies, start_new_hyperopt, start_new_strategy, start_list_timeframes, start_plot_dataframe, start_plot_profit, + start_backtesting, start_hyperopt, start_edge, start_test_pairlist, start_trading) subparsers = self.parser.add_subparsers(dest='command', diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index acbc44e21..07872da57 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -12,13 +12,13 @@ from arrow import Arrow from freqtrade import constants from freqtrade.configuration import TimeRange +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting from freqtrade.data import history from freqtrade.data.btanalysis import evaluate_result_multi from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.optimize import setup_configuration, start_backtesting from freqtrade.optimize.backtesting import Backtesting from freqtrade.state import RunMode from freqtrade.strategy.default_strategy import DefaultStrategy @@ -177,7 +177,7 @@ def _trend_alternate(dataframe=None, metadata=None): # Unit tests -def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) args = [ @@ -186,7 +186,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> '--strategy', 'DefaultStrategy', ] - config = setup_configuration(get_args(args), RunMode.BACKTEST) + config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -227,7 +227,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--fee', '0', ] - config = setup_configuration(get_args(args), RunMode.BACKTEST) + config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -260,7 +260,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --fee detected, setting fee to: {} ...'.format(config['fee']), caplog) -def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT patched_configuration_load_config_file(mocker, default_conf) @@ -272,7 +272,7 @@ def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog ] with pytest.raises(DependencyException, match=r'.*stake amount.*'): - setup_configuration(get_args(args), RunMode.BACKTEST) + setup_optimize_configuration(get_args(args), RunMode.BACKTEST) def test_start(mocker, fee, default_conf, caplog) -> None: diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index acc0d2d16..96dd0899d 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -3,14 +3,14 @@ from unittest.mock import MagicMock -from freqtrade.optimize import setup_configuration, start_edge +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge from freqtrade.optimize.edge_cli import EdgeCli from freqtrade.state import RunMode from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) -def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) args = [ @@ -19,7 +19,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> '--strategy', 'DefaultStrategy', ] - config = setup_configuration(get_args(args), RunMode.EDGE) + config = setup_optimize_configuration(get_args(args), RunMode.EDGE) assert config['runmode'] == RunMode.EDGE assert 'max_open_trades' in config @@ -53,7 +53,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N '--stoplosses=-0.01,-0.10,-0.001' ] - config = setup_configuration(get_args(args), RunMode.EDGE) + config = setup_optimize_configuration(get_args(args), RunMode.EDGE) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 473d760ac..69d110649 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -9,10 +9,10 @@ import pytest from arrow import Arrow from filelock import Timeout -from freqtrade.exceptions import OperationalException +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file -from freqtrade.optimize import setup_configuration, start_hyperopt +from freqtrade.exceptions import OperationalException from freqtrade.optimize.default_hyperopt import DefaultHyperOpt from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss from freqtrade.optimize.hyperopt import Hyperopt @@ -77,7 +77,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca '--hyperopt', 'DefaultHyperOpt', ] - config = setup_configuration(get_args(args), RunMode.HYPEROPT) + config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -117,7 +117,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo '--print-all' ] - config = setup_configuration(get_args(args), RunMode.HYPEROPT) + config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config diff --git a/tests/test_main.py b/tests/test_main.py index 76b1bf658..462ac6427 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -26,7 +26,7 @@ def test_parse_args_backtesting(mocker) -> None: Test that main() can start backtesting and also ensure we can pass some specific arguments further argument parsing is done in test_arguments.py """ - backtesting_mock = mocker.patch('freqtrade.optimize.start_backtesting', MagicMock()) + backtesting_mock = mocker.patch('freqtrade.commands.start_backtesting') backtesting_mock.__name__ = PropertyMock("start_backtesting") # it's sys.exit(0) at the end of backtesting with pytest.raises(SystemExit): @@ -42,7 +42,7 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: - hyperopt_mock = mocker.patch('freqtrade.optimize.start_hyperopt', MagicMock()) + hyperopt_mock = mocker.patch('freqtrade.commands.start_hyperopt', MagicMock()) hyperopt_mock.__name__ = PropertyMock("start_hyperopt") # it's sys.exit(0) at the end of hyperopt with pytest.raises(SystemExit): From a1c684f67ca3dcf0c6293d71511c72a0dcbd2cb3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:36:33 +0100 Subject: [PATCH 14/19] Simplify noqa setup for module imports --- freqtrade/configuration/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 63c38d8c5..a631609d7 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,5 +1,7 @@ -from freqtrade.configuration.arguments import Arguments # noqa: F401 -from freqtrade.configuration.check_exchange import check_exchange, remove_credentials # noqa: F401 -from freqtrade.configuration.timerange import TimeRange # noqa: F401 -from freqtrade.configuration.configuration import Configuration # noqa: F401 -from freqtrade.configuration.config_validation import validate_config_consistency # noqa: F401 +# flake8: noqa: F401 + +from freqtrade.configuration.arguments import Arguments +from freqtrade.configuration.check_exchange import check_exchange, remove_credentials +from freqtrade.configuration.timerange import TimeRange +from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration.config_validation import validate_config_consistency From 7f851ad8d950d5c26da08f76ad239610136af51c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:38:56 +0100 Subject: [PATCH 15/19] Move arguments and cli_options to commands module --- freqtrade/{configuration => commands}/arguments.py | 0 freqtrade/{configuration => commands}/cli_options.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename freqtrade/{configuration => commands}/arguments.py (100%) rename freqtrade/{configuration => commands}/cli_options.py (100%) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/commands/arguments.py similarity index 100% rename from freqtrade/configuration/arguments.py rename to freqtrade/commands/arguments.py diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/commands/cli_options.py similarity index 100% rename from freqtrade/configuration/cli_options.py rename to freqtrade/commands/cli_options.py From a3e9d04383bdc37aea465561306334042adc801a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:41:04 +0100 Subject: [PATCH 16/19] Adjust imports to new place for arguments --- freqtrade/commands/__init__.py | 1 + freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 6 +++--- freqtrade/configuration/__init__.py | 1 - freqtrade/main.py | 2 +- tests/conftest.py | 2 +- tests/test_arguments.py | 4 ++-- tests/test_configuration.py | 3 ++- tests/test_main.py | 2 +- 9 files changed, 12 insertions(+), 11 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 8339486fb..dcb814c50 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 +from freqtrade.commands.arguments import Arguments from freqtrade.commands.data_commands import start_download_data from freqtrade.commands.deploy_commands import (start_create_userdir, start_new_hyperopt, diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 21800e821..724814554 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional from freqtrade import constants -from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS +from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 1807cd591..490f26cfa 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -1,7 +1,7 @@ """ Definition of cli arguments used in arguments.py """ -import argparse +from argparse import ArgumentTypeError from freqtrade import __version__, constants @@ -12,7 +12,7 @@ def check_int_positive(value: str) -> int: if uint <= 0: raise ValueError except ValueError: - raise argparse.ArgumentTypeError( + raise ArgumentTypeError( f"{value} is invalid for this parameter, should be a positive integer value" ) return uint @@ -24,7 +24,7 @@ def check_int_nonzero(value: str) -> int: if uint == 0: raise ValueError except ValueError: - raise argparse.ArgumentTypeError( + raise ArgumentTypeError( f"{value} is invalid for this parameter, should be a non-zero integer value" ) return uint diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index a631609d7..54fc4e427 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,6 +1,5 @@ # flake8: noqa: F401 -from freqtrade.configuration.arguments import Arguments from freqtrade.configuration.check_exchange import check_exchange, remove_credentials from freqtrade.configuration.timerange import TimeRange from freqtrade.configuration.configuration import Configuration diff --git a/freqtrade/main.py b/freqtrade/main.py index 811e29864..a75eeebed 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -14,7 +14,7 @@ if sys.version_info < (3, 6): import logging from typing import Any, List -from freqtrade.configuration import Arguments +from freqtrade.commands import Arguments logger = logging.getLogger('freqtrade') diff --git a/tests/conftest.py b/tests/conftest.py index 295c91f56..395388f73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,7 @@ import pytest from telegram import Chat, Message, Update from freqtrade import constants, persistence -from freqtrade.configuration import Arguments +from freqtrade.commands import Arguments from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.exchange import Exchange diff --git a/tests/test_arguments.py b/tests/test_arguments.py index d8fbace0f..60da0082a 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock import pytest -from freqtrade.configuration import Arguments -from freqtrade.configuration.cli_options import check_int_positive +from freqtrade.commands import Arguments +from freqtrade.commands.cli_options import check_int_positive # Parse common command-line-arguments. Used for all tools diff --git a/tests/test_configuration.py b/tests/test_configuration.py index cbcd6416a..74de166c1 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -10,7 +10,8 @@ from unittest.mock import MagicMock import pytest from jsonschema import ValidationError -from freqtrade.configuration import (Arguments, Configuration, check_exchange, +from freqtrade.commands import Arguments +from freqtrade.configuration import (Configuration, check_exchange, remove_credentials, validate_config_consistency) from freqtrade.configuration.config_validation import validate_config_schema diff --git a/tests/test_main.py b/tests/test_main.py index 462ac6427..1229f748a 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade.configuration import Arguments +from freqtrade.commands import Arguments from freqtrade.exceptions import OperationalException, FreqtradeException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.main import main From 2d02c3f2a4097aa56182ff9d56bf4f631ac06696 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:43:30 +0100 Subject: [PATCH 17/19] Split out pairlist_commands --- freqtrade/commands/__init__.py | 4 +-- freqtrade/commands/pairlist_commands.py | 43 +++++++++++++++++++++++++ freqtrade/commands/utils.py | 35 -------------------- 3 files changed, 45 insertions(+), 37 deletions(-) create mode 100644 freqtrade/commands/pairlist_commands.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index dcb814c50..e16a6f203 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -13,8 +13,8 @@ from freqtrade.commands.list_commands import (start_list_exchanges, start_list_timeframes) from freqtrade.commands.optimize_commands import (start_backtesting, start_edge, start_hyperopt) +from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import (start_plot_dataframe, start_plot_profit) from freqtrade.commands.trade_commands import start_trading -from freqtrade.commands.utils import (setup_utils_configuration, - start_test_pairlist) +from freqtrade.commands.utils import setup_utils_configuration diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py new file mode 100644 index 000000000..06ba2e680 --- /dev/null +++ b/freqtrade/commands/pairlist_commands.py @@ -0,0 +1,43 @@ +import logging +from typing import Any, Dict + +import rapidjson + +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode + +from .utils import setup_utils_configuration + +logger = logging.getLogger(__name__) + + +def start_test_pairlist(args: Dict[str, Any]) -> None: + """ + Test Pairlist configuration + """ + from freqtrade.pairlist.pairlistmanager import PairListManager + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + quote_currencies = args.get('quote_currencies') + if not quote_currencies: + quote_currencies = [config.get('stake_currency')] + results = {} + for curr in quote_currencies: + config['stake_currency'] = curr + # Do not use ticker_interval set in the config + pairlists = PairListManager(exchange, config) + pairlists.refresh_pairlist() + results[curr] = pairlists.whitelist + + for curr, pairlist in results.items(): + if not args.get('print_one_column', False): + print(f"Pairs for {curr}: ") + + if args.get('print_one_column', False): + print('\n'.join(pairlist)) + elif args.get('list_pairs_print_json', False): + print(rapidjson.dumps(list(pairlist), default=str)) + else: + print(pairlist) diff --git a/freqtrade/commands/utils.py b/freqtrade/commands/utils.py index a597d5335..f77a35383 100644 --- a/freqtrade/commands/utils.py +++ b/freqtrade/commands/utils.py @@ -1,11 +1,8 @@ import logging from typing import Any, Dict -import rapidjson - from freqtrade.configuration import (Configuration, remove_credentials, validate_config_consistency) -from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -25,35 +22,3 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str validate_config_consistency(config) return config - - -def start_test_pairlist(args: Dict[str, Any]) -> None: - """ - Test Pairlist configuration - """ - from freqtrade.pairlist.pairlistmanager import PairListManager - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) - - quote_currencies = args.get('quote_currencies') - if not quote_currencies: - quote_currencies = [config.get('stake_currency')] - results = {} - for curr in quote_currencies: - config['stake_currency'] = curr - # Do not use ticker_interval set in the config - pairlists = PairListManager(exchange, config) - pairlists.refresh_pairlist() - results[curr] = pairlists.whitelist - - for curr, pairlist in results.items(): - if not args.get('print_one_column', False): - print(f"Pairs for {curr}: ") - - if args.get('print_one_column', False): - print('\n'.join(pairlist)) - elif args.get('list_pairs_print_json', False): - print(rapidjson.dumps(list(pairlist), default=str)) - else: - print(pairlist) From 8c9119b4710e467d4833638d582aba5914d61d2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:45:37 +0100 Subject: [PATCH 18/19] Add docustring to commands module --- freqtrade/commands/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index e16a6f203..1dadbec1e 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,5 +1,11 @@ # flake8: noqa: F401 +""" +Commands module. +Contains all start-commands, subcommands and CLI Interface creation. +Note: Be careful with file-scoped imports in these subfiles. + as they are parsed on startup, nothing containing optional modules should be loaded. +""" from freqtrade.commands.arguments import Arguments from freqtrade.commands.data_commands import start_download_data from freqtrade.commands.deploy_commands import (start_create_userdir, From 02563019fc84d559584ae007c6c9d5f4055da911 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:55:48 +0100 Subject: [PATCH 19/19] move setup_utils_config to configuration module --- freqtrade/commands/__init__.py | 1 - freqtrade/commands/data_commands.py | 3 +-- freqtrade/commands/deploy_commands.py | 3 +-- freqtrade/commands/hyperopt_commands.py | 2 +- freqtrade/commands/list_commands.py | 3 +-- freqtrade/commands/optimize_commands.py | 2 +- freqtrade/commands/pairlist_commands.py | 3 +-- freqtrade/commands/plot_commands.py | 2 +- freqtrade/configuration/__init__.py | 1 + .../{commands/utils.py => configuration/config_setup.py} | 5 +++-- tests/commands/__init__.py | 0 tests/{test_utils.py => commands/test_commands.py} | 8 ++++---- 12 files changed, 15 insertions(+), 18 deletions(-) rename freqtrade/{commands/utils.py => configuration/config_setup.py} (79%) create mode 100644 tests/commands/__init__.py rename tests/{test_utils.py => commands/test_commands.py} (99%) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 1dadbec1e..990c1107a 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -23,4 +23,3 @@ from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import (start_plot_dataframe, start_plot_profit) from freqtrade.commands.trade_commands import start_trading -from freqtrade.commands.utils import setup_utils_configuration diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index b1c847d1d..c01772023 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -4,14 +4,13 @@ from typing import Any, Dict, List import arrow -from freqtrade.configuration import TimeRange +from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) from freqtrade.exceptions import OperationalException from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode -from .utils import setup_utils_configuration logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 892e15db1..99ae63244 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -3,6 +3,7 @@ import sys from pathlib import Path from typing import Any, Dict +from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY @@ -10,8 +11,6 @@ from freqtrade.exceptions import OperationalException from freqtrade.misc import render_template from freqtrade.state import RunMode -from .utils import setup_utils_configuration - logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index db3f69050..5c6f25848 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List from colorama import init as colorama_init -from .utils import setup_utils_configuration +from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 90efaf66f..022822782 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -8,6 +8,7 @@ from typing import Any, Dict import rapidjson from tabulate import tabulate +from freqtrade.configuration import setup_utils_configuration from freqtrade.constants import USERPATH_STRATEGY from freqtrade.exceptions import OperationalException from freqtrade.exchange import (available_exchanges, ccxt_exchanges, @@ -16,8 +17,6 @@ from freqtrade.misc import plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode -from .utils import setup_utils_configuration - logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index e635e61fd..a2d1b4601 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -2,7 +2,7 @@ import logging from typing import Any, Dict from freqtrade import constants -from freqtrade.commands.utils import setup_utils_configuration +from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.state import RunMode diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index 06ba2e680..bf0b217a5 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -3,11 +3,10 @@ from typing import Any, Dict import rapidjson +from freqtrade.configuration import setup_utils_configuration from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode -from .utils import setup_utils_configuration - logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/plot_commands.py b/freqtrade/commands/plot_commands.py index 4c68cc3c5..028933ba7 100644 --- a/freqtrade/commands/plot_commands.py +++ b/freqtrade/commands/plot_commands.py @@ -1,8 +1,8 @@ from typing import Any, Dict +from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.commands.utils import setup_utils_configuration def validate_plot_args(args: Dict[str, Any]): diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 54fc4e427..d41ac97ec 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 +from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.check_exchange import check_exchange, remove_credentials from freqtrade.configuration.timerange import TimeRange from freqtrade.configuration.configuration import Configuration diff --git a/freqtrade/commands/utils.py b/freqtrade/configuration/config_setup.py similarity index 79% rename from freqtrade/commands/utils.py rename to freqtrade/configuration/config_setup.py index f77a35383..64f283e42 100644 --- a/freqtrade/commands/utils.py +++ b/freqtrade/configuration/config_setup.py @@ -1,8 +1,9 @@ import logging from typing import Any, Dict -from freqtrade.configuration import (Configuration, remove_credentials, - validate_config_consistency) +from .config_validation import validate_config_consistency +from .configuration import Configuration +from .check_exchange import remove_credentials from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_utils.py b/tests/commands/test_commands.py similarity index 99% rename from tests/test_utils.py rename to tests/commands/test_commands.py index 1328e3981..65d7f6eaf 100644 --- a/tests/test_utils.py +++ b/tests/commands/test_commands.py @@ -4,13 +4,13 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade.commands import (setup_utils_configuration, - start_create_userdir, start_download_data, +from freqtrade.commands import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, start_list_strategies, start_list_timeframes, start_new_hyperopt, start_new_strategy, start_test_pairlist, start_trading) +from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, @@ -639,7 +639,7 @@ def test_start_list_strategies(mocker, caplog, capsys): args = [ "list-strategies", "--strategy-path", - str(Path(__file__).parent / "strategy"), + str(Path(__file__).parent.parent / "strategy"), "-1" ] pargs = get_args(args) @@ -654,7 +654,7 @@ def test_start_list_strategies(mocker, caplog, capsys): args = [ "list-strategies", "--strategy-path", - str(Path(__file__).parent / "strategy"), + str(Path(__file__).parent.parent / "strategy"), ] pargs = get_args(args) # pargs['config'] = None