diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index ce76d52e5..10ea068ad 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1 +1 @@ -mkdocs-material==4.4.0 \ No newline at end of file +mkdocs-material==4.4.1 \ No newline at end of file diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 14f0bb819..5ccc2ff3c 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -11,7 +11,7 @@ class DependencyException(Exception): class OperationalException(Exception): """ - Requires manual intervention. + Requires manual intervention and will usually stop the bot. This happens when an exchange returns an unexpected error during runtime or given configuration is invalid. """ diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 007357d9a..aff9f5c74 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -280,6 +280,35 @@ def download_pair_history(datadir: Optional[Path], return False +def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], + dl_path: Path, timerange: TimeRange, + erase=False) -> List[str]: + """ + Refresh stored ohlcv data for backtesting and hyperopt operations. + Used by freqtrade download-data + :return: Pairs not available + """ + pairs_not_available = [] + for pair in pairs: + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for ticker_interval in timeframes: + + dl_file = pair_data_filename(dl_path, pair, ticker_interval) + if erase and dl_file.exists(): + logger.info( + f'Deleting existing data for pair {pair}, interval {ticker_interval}.') + dl_file.unlink() + + logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, ticker_interval=str(ticker_interval), + timerange=timerange) + return pairs_not_available + + def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ Get the maximum timeframe for the given backtest data diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 18e754e3f..5834f26cd 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -2,6 +2,9 @@ import logging from typing import Dict +import ccxt + +from freqtrade import DependencyException, OperationalException, TemporaryError from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) @@ -25,3 +28,53 @@ class Binance(Exchange): limit = min(list(filter(lambda x: limit <= x, limit_range))) return super().get_order_book(pair, limit) + + def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: + """ + creates a stoploss limit order. + this stoploss-limit is binance-specific. + It may work with a limited number of other exchanges, but this has not been tested yet. + + """ + ordertype = "stop_loss_limit" + + stop_price = self.symbol_price_prec(pair, stop_price) + + # Ensure rate is less than stop price + if stop_price <= rate: + raise OperationalException( + 'In stoploss limit order, stop price should be more than limit price') + + if self._config['dry_run']: + dry_order = self.dry_run_order( + pair, ordertype, "sell", amount, stop_price) + return dry_order + + try: + params = self._params.copy() + params.update({'stopPrice': stop_price}) + + amount = self.symbol_amount_prec(pair, amount) + + rate = self.symbol_price_prec(pair, rate) + + order = self._api.create_order(pair, ordertype, 'sell', + amount, rate, params) + logger.info('stoploss limit order added for %s. ' + 'stop price: %s. limit: %s', pair, stop_price, rate) + return order + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to create {ordertype} sell order on market {pair}.' + f'Tried to sell amount {amount} at rate {rate}.' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not create {ordertype} sell order on market {pair}. ' + f'Tried to sell amount {amount} at rate {rate}.' + f'Message: {e}') from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6abf1f270..e20546856 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -321,7 +321,7 @@ class Exchange(object): if (order_types.get("stoploss_on_exchange") and not self._ft_has.get("stoploss_on_exchange", False)): raise OperationalException( - 'On exchange stoploss is not supported for %s.' % self.name + f'On exchange stoploss is not supported for {self.name}.' ) def validate_order_time_in_force(self, order_time_in_force: Dict) -> None: @@ -451,30 +451,14 @@ class Exchange(object): def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: """ creates a stoploss limit order. - NOTICE: it is not supported by all exchanges. only binance is tested for now. - TODO: implementation maybe needs to be moved to the binance subclass + Since ccxt does not unify stoploss-limit orders yet, this needs to be implemented in each + exchange's subclass. + The exception below should never raise, since we disallow + starting the bot in validate_ordertypes() + Note: Changes to this interface need to be applied to all sub-classes too. """ - ordertype = "stop_loss_limit" - stop_price = self.symbol_price_prec(pair, stop_price) - - # Ensure rate is less than stop price - if stop_price <= rate: - raise OperationalException( - 'In stoploss limit order, stop price should be more than limit price') - - if self._config['dry_run']: - dry_order = self.dry_run_order( - pair, ordertype, "sell", amount, stop_price) - return dry_order - - params = self._params.copy() - params.update({'stopPrice': stop_price}) - - order = self.create_order(pair, ordertype, 'sell', amount, rate, params) - logger.info('stoploss limit order added for %s. ' - 'stop price: %s. limit: %s', pair, stop_price, rate) - return order + raise OperationalException(f"stoploss_limit is not implemented for {self.name}.") @retrier def get_balance(self, currency: str) -> float: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e88b9db6a..8857b95da 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -216,7 +216,7 @@ class FreqtradeBot(object): if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.get_open_trades()) if open_trades >= self.config['max_open_trades']: - logger.warning('Can\'t open a new trade: max number of trades is reached') + logger.warning("Can't open a new trade: max number of trades is reached") return None return available_amount / (self.config['max_open_trades'] - open_trades) @@ -351,8 +351,8 @@ class FreqtradeBot(object): min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( - f'Can\'t open a new trade for {pair_s}: stake amount ' - f'is too small ({stake_amount} < {min_stake_amount})' + f"Can't open a new trade for {pair_s}: stake amount " + f"is too small ({stake_amount} < {min_stake_amount})" ) return False diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 62a6ab27b..9c3f085b6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -124,14 +124,14 @@ class Hyperopt: Save hyperopt trials to file """ if self.trials: - logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file) + logger.info("Saving %d evaluations to '%s'", len(self.trials), self.trials_file) dump(self.trials, self.trials_file) def read_trials(self) -> List: """ Read hyperopt trials file """ - logger.info('Reading Trials from \'%s\'', self.trials_file) + logger.info("Reading Trials from '%s'", self.trials_file) trials = load(self.trials_file) self.trials_file.unlink() return trials @@ -379,7 +379,7 @@ class Hyperopt: self.load_previous_results() cpus = cpu_count() - logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') + logger.info(f"Found {cpus} CPU cores. Let's make them scream!") config_jobs = self.config.get('hyperopt_jobs', -1) logger.info(f'Number of parallel jobs set as: {config_jobs}') diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c844bbc4c..dff7e4ff6 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -48,8 +48,8 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: try: engine = create_engine(db_url, **kwargs) except NoSuchModuleError: - raise OperationalException(f'Given value for db_url: \'{db_url}\' ' - f'is no valid database URL! (See {_SQL_DOCS_URL})') + raise OperationalException(f"Given value for db_url: '{db_url}' " + f"is no valid database URL! (See {_SQL_DOCS_URL})") session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 7360f3c1c..ec4a05e63 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -5,7 +5,7 @@ import os import uuid from pathlib import Path from shutil import copyfile -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import arrow import pytest @@ -17,6 +17,7 @@ from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, load_tickerdata_file, make_testdata_path, + refresh_backtest_ohlcv_data, trim_tickerlist) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json @@ -558,3 +559,43 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', min_date, max_date, timeframe_to_minutes('5m')) assert len(caplog.record_tuples) == 0 + + +def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog): + dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "unlink", MagicMock()) + + ex = get_patched_exchange(mocker, default_conf) + timerange = TimeRange.parse_timerange("20190101-20190102") + refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], + timeframes=["1m", "5m"], dl_path=make_testdata_path(None), + timerange=timerange, erase=True + ) + + assert dl_mock.call_count == 4 + assert dl_mock.call_args[1]['timerange'].starttype == 'date' + + assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) + + +def test_download_data_no_markets(mocker, default_conf, caplog): + dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + ex = get_patched_exchange(mocker, default_conf) + timerange = TimeRange.parse_timerange("20190101-20190102") + unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], + timeframes=["1m", "5m"], + dl_path=make_testdata_path(None), + timerange=timerange, erase=False + ) + + assert dl_mock.call_count == 0 + assert "ETH/BTC" in unav_pairs + assert "XRP/BTC" in unav_pairs + assert log_has("Skipping pair ETH/BTC...", caplog) diff --git a/freqtrade/tests/exchange/test_binance.py b/freqtrade/tests/exchange/test_binance.py new file mode 100644 index 000000000..4afb7fcc4 --- /dev/null +++ b/freqtrade/tests/exchange/test_binance.py @@ -0,0 +1,90 @@ +from random import randint +from unittest.mock import MagicMock + +import ccxt +import pytest + +from freqtrade import DependencyException, OperationalException, TemporaryError +from freqtrade.tests.conftest import get_patched_exchange + + +def test_stoploss_limit_order(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'stop_loss_limit' + + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + + default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + + with pytest.raises(OperationalException): + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + + api_mock.create_order.reset_mock() + + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 + assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} + + # test exception handling + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(TemporaryError): + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(OperationalException, match=r".*DeadBeef.*"): + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + +def test_stoploss_limit_order_dry_run(default_conf, mocker): + api_mock = MagicMock() + order_type = 'stop_loss_limit' + default_conf['dry_run'] = True + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + + with pytest.raises(OperationalException): + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + + api_mock.create_order.reset_mock() + + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + assert 'id' in order + assert 'info' in order + assert 'type' in order + + assert order['type'] == order_type + assert order['price'] == 220 + assert order['amount'] == 1 diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index dad2d9c37..1fd045f54 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -101,18 +101,21 @@ def test_destroy(default_conf, mocker, caplog): def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' - with pytest.raises( - OperationalException, - match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + with pytest.raises(OperationalException, + match=f"Exchange {default_conf['exchange']['name']} is not supported"): Exchange(default_conf) default_conf['exchange']['name'] = 'binance' - with pytest.raises( - OperationalException, - match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + with pytest.raises(OperationalException, + match=f"Exchange {default_conf['exchange']['name']} is not supported"): mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError)) Exchange(default_conf) + with pytest.raises(OperationalException, + match=r"Initialization of ccxt failed. Reason: DeadBeef"): + mocker.patch("ccxt.binance", MagicMock(side_effect=ccxt.BaseError("DeadBeef"))) + Exchange(default_conf) + def test_exchange_resolver(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock())) @@ -1436,87 +1439,11 @@ def test_get_fee(default_conf, mocker, exchange_name): 'get_fee', 'calculate_fee') -def test_stoploss_limit_order(default_conf, mocker): - api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - order_type = 'stop_loss_limit' - - api_mock.create_order = MagicMock(return_value={ - 'id': order_id, - 'info': { - 'foo': 'bar' - } - }) - - default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - - with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) - - api_mock.create_order.reset_mock() - - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - assert 'id' in order - assert 'info' in order - assert order['id'] == order_id - assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' - assert api_mock.create_order.call_args[0][1] == order_type - assert api_mock.create_order.call_args[0][2] == 'sell' - assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][4] == 200 - assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} - - # test exception handling - with pytest.raises(DependencyException): - api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) +def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf, 'bittrex') + with pytest.raises(OperationalException, match=r"stoploss_limit is not implemented .*"): exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - with pytest.raises(DependencyException): - api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - with pytest.raises(TemporaryError): - api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - with pytest.raises(OperationalException): - api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - -def test_stoploss_limit_order_dry_run(default_conf, mocker): - api_mock = MagicMock() - order_type = 'stop_loss_limit' - default_conf['dry_run'] = True - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - - with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) - - api_mock.create_order.reset_mock() - - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) - - assert 'id' in order - assert 'info' in order - assert 'type' in order - - assert order['type'] == order_type - assert order['price'] == 220 - assert order['amount'] == 1 - def test_merge_ft_has_dict(default_conf, mocker): mocker.patch.multiple('freqtrade.exchange.Exchange', diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 7b525454c..f9ebf552d 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -381,7 +381,7 @@ def test_save_trials_saves_trials(mocker, hyperopt, caplog) -> None: hyperopt.save_trials() trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') - assert log_has('Saving 1 evaluations to \'{}\''.format(trials_file), caplog) + assert log_has("Saving 1 evaluations to '{}'".format(trials_file), caplog) mock_dump.assert_called_once() @@ -390,7 +390,7 @@ def test_read_trials_returns_trials_file(mocker, hyperopt, caplog) -> None: mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials) hyperopt_trial = hyperopt.read_trials() trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') - assert log_has('Reading Trials from \'{}\''.format(trials_file), caplog) + assert log_has("Reading Trials from '{}'".format(trials_file), caplog) assert hyperopt_trial == trials mock_load.assert_called_once() diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 10ce7e8cf..d98ff9ad0 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -41,14 +41,14 @@ def test_load_config_invalid_pair(default_conf) -> None: def test_load_config_missing_attributes(default_conf) -> None: default_conf.pop('exchange') - with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): + with pytest.raises(ValidationError, match=r".*'exchange' is a required property.*"): validate_config_schema(default_conf) def test_load_config_incorrect_stake_amount(default_conf) -> None: default_conf['stake_amount'] = 'fake' - with pytest.raises(ValidationError, match=r'.*\'fake\' does not match \'unlimited\'.*'): + with pytest.raises(ValidationError, match=r".*'fake' does not match 'unlimited'.*"): validate_config_schema(default_conf) @@ -472,7 +472,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: assert 'spaces' in config assert config['spaces'] == ['all'] - assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog) + assert log_has("Parameter -s/--spaces detected: ['all']", caplog) assert "runmode" in config assert config['runmode'] == RunMode.HYPEROPT @@ -722,7 +722,7 @@ def test_load_config_default_exchange(all_conf) -> None: assert 'exchange' not in all_conf with pytest.raises(ValidationError, - match=r'\'exchange\' is a required property'): + match=r"'exchange' is a required property"): validate_config_schema(all_conf) @@ -736,7 +736,7 @@ def test_load_config_default_exchange_name(all_conf) -> None: assert 'name' not in all_conf['exchange'] with pytest.raises(ValidationError, - match=r'\'name\' is a required property'): + match=r"'name' is a required property"): validate_config_schema(all_conf) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index dab7a9ff7..1119157c4 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2414,7 +2414,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit) freqtrade = FreqtradeBot(default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -2454,7 +2454,6 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, freqtrade.process_maybe_execute_sell(trade) assert trade.stoploss_order_id is None assert trade.is_open is False - print(trade.sell_reason) assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value assert rpc_mock.call_count == 2 diff --git a/freqtrade/tests/test_utils.py b/freqtrade/tests/test_utils.py index d04e62b28..9e09fd298 100644 --- a/freqtrade/tests/test_utils.py +++ b/freqtrade/tests/test_utils.py @@ -1,5 +1,4 @@ import re -from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pytest @@ -70,74 +69,8 @@ def test_create_datadir(caplog, mocker): assert len(caplog.record_tuples) == 0 -def test_download_data(mocker, markets, caplog): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) - patch_exchange(mocker) - mocker.patch( - 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) - ) - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - mocker.patch.object(Path, "unlink", MagicMock()) - - args = [ - "download-data", - "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC", - "--erase", - ] - start_download_data(get_args(args)) - - assert dl_mock.call_count == 4 - assert dl_mock.call_args[1]['timerange'].starttype is None - assert dl_mock.call_args[1]['timerange'].stoptype is None - assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog) - assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) - - -def test_download_data_days(mocker, markets, caplog): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) - patch_exchange(mocker) - mocker.patch( - 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) - ) - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - mocker.patch.object(Path, "unlink", MagicMock()) - - args = [ - "download-data", - "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC", - "--days", "20", - ] - - start_download_data(get_args(args)) - - assert dl_mock.call_count == 4 - assert dl_mock.call_args[1]['timerange'].starttype == 'date' - - assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) - - -def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) - patch_exchange(mocker) - mocker.patch( - 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) - ) - args = [ - "download-data", - "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC", - ] - start_download_data(get_args(args)) - - assert dl_mock.call_count == 0 - assert log_has("Skipping pair ETH/BTC...", caplog) - assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) - - def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', + dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -152,3 +85,21 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): start_download_data(get_args(args)) assert dl_mock.call_count == 1 + + +def test_download_data_no_markets(mocker, caplog): + dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20" + ] + start_download_data(get_args(args)) + assert dl_mock.call_args[1]['timerange'].starttype == "date" + assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 56e60ec82..162493a3f 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -2,13 +2,13 @@ import logging import sys from argparse import Namespace from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, List import arrow from freqtrade.configuration import Configuration, TimeRange from freqtrade.configuration.directory_operations import create_userdata_dir -from freqtrade.data.history import download_pair_history +from freqtrade.data.history import refresh_backtest_ohlcv_data from freqtrade.exchange import available_exchanges from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode @@ -75,39 +75,23 @@ def start_download_data(args: Namespace) -> None: logger.info(f'About to download pairs: {config["pairs"]}, ' f'intervals: {config["timeframes"]} to {dl_path}') - pairs_not_available = [] + pairs_not_available: List[str] = [] try: # Init exchange exchange = ExchangeResolver(config['exchange']['name'], config).exchange - for pair in config["pairs"]: - if pair not in exchange.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - for ticker_interval in config["timeframes"]: - pair_print = pair.replace('/', '_') - filename = f'{pair_print}-{ticker_interval}.json' - dl_file = dl_path.joinpath(filename) - if config.get("erase") and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}, interval {ticker_interval}.') - dl_file.unlink() - - logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') - download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, ticker_interval=str(ticker_interval), - timerange=timerange) + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=config["pairs"], timeframes=config["timeframes"], + dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") finally: if pairs_not_available: - logger.info( - f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {config['exchange']['name']}.") + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {config['exchange']['name']}.") # configuration.resolve_pairs_list() print(config) diff --git a/requirements-common.txt b/requirements-common.txt index bd3afd8de..c8a9c2f74 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.1084 +ccxt==1.18.1085 SQLAlchemy==1.3.7 python-telegram-bot==11.1.0 arrow==0.14.5 diff --git a/requirements-dev.txt b/requirements-dev.txt index 6436c60e4..a7b0d358d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ flake8==3.7.8 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 mypy==0.720 -pytest==5.1.0 +pytest==5.1.1 pytest-asyncio==0.10.0 pytest-cov==2.7.1 pytest-mock==1.10.4 diff --git a/requirements.txt b/requirements.txt index 9d558b5b8..e5015d620 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,5 @@ -r requirements-common.txt numpy==1.17.0 -pandas==0.25.0 +pandas==0.25.1 scipy==1.3.1