diff --git a/config.json.example b/config.json.example index f92849ec9..87c27090e 100644 --- a/config.json.example +++ b/config.json.example @@ -6,7 +6,10 @@ "ticker_interval" : "5m", "dry_run": false, "trailing_stop": false, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/config_full.json.example b/config_full.json.example index 244daabac..4003b1c5c 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -14,7 +14,10 @@ "0": 0.04 }, "stoploss": -0.10, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/docs/backtesting.md b/docs/backtesting.md index 1efb46b43..172969ae2 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -70,6 +70,34 @@ Where `-s TestStrategy` refers to the class name within the strategy file `test_ python3 ./freqtrade/main.py backtesting --export trades ``` +The exported trades can be read using the following code for manual analysis, or can be used by the plotting script `plot_dataframe.py` in the scripts folder. + +``` python +import json +from pathlib import Path +import pandas as pd + +filename=Path('user_data/backtest_data/backtest-result.json') + +with filename.open() as file: + data = json.load(file) + +columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] +df = pd.DataFrame(data, columns=columns) + +df['opents'] = pd.to_datetime(df['opents'], + unit='s', + utc=True, + infer_datetime_format=True + ) +df['closets'] = pd.to_datetime(df['closets'], + unit='s', + utc=True, + infer_datetime_format=True + ) +``` + #### Exporting trades to file specifying a custom filename ```bash diff --git a/docs/configuration.md b/docs/configuration.md index 984f2529b..dd16ef6b5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,8 @@ The table below will list all configuration parameters. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). | `trailing_stoploss_positve` | 0 | No | Changes stop-loss once profit has been reached. -| `unfilledtimeout` | 0 | No | How long (in minutes) the bot will wait for an unfilled order to complete, after which the order will be cancelled. +| `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. +| `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d80eea6f4..ec7765455 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -63,7 +63,13 @@ CONF_SCHEMA = { 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, 'trailing_stop': {'type': 'boolean'}, 'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1}, - 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, + 'unfilledtimeout': { + 'type': 'object', + 'properties': { + 'buy': {'type': 'number', 'minimum': 3}, + 'sell': {'type': 'number', 'minimum': 10} + } + }, 'bid_strategy': { 'type': 'object', 'properties': { diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e25ed66cf..530535710 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -160,7 +160,7 @@ class FreqtradeBot(object): if 'unfilledtimeout' in self.config: # Check and handle any timed out open orders - self.check_handle_timedout(self.config['unfilledtimeout']) + self.check_handle_timedout() Trade.session.flush() except TemporaryError as error: @@ -277,11 +277,14 @@ class FreqtradeBot(object): return None min_stake_amounts = [] - if 'cost' in market['limits'] and 'min' in market['limits']['cost']: - min_stake_amounts.append(market['limits']['cost']['min']) + limits = market['limits'] + if ('cost' in limits and 'min' in limits['cost'] + and limits['cost']['min'] is not None): + min_stake_amounts.append(limits['cost']['min']) - if 'amount' in market['limits'] and 'min' in market['limits']['amount']: - min_stake_amounts.append(market['limits']['amount']['min'] * price) + if ('amount' in limits and 'min' in limits['amount'] + and limits['amount']['min'] is not None): + min_stake_amounts.append(limits['amount']['min'] * price) if not min_stake_amounts: return None @@ -492,13 +495,16 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False - def check_handle_timedout(self, timeoutvalue: int) -> None: + def check_handle_timedout(self) -> None: """ Check if any orders are timed out and cancel if neccessary :param timeoutvalue: Number of minutes until order is considered timed out :return: None """ - timeoutthreashold = arrow.utcnow().shift(minutes=-timeoutvalue).datetime + buy_timeout = self.config['unfilledtimeout']['buy'] + sell_timeout = self.config['unfilledtimeout']['sell'] + buy_timeoutthreashold = arrow.utcnow().shift(minutes=-buy_timeout).datetime + sell_timeoutthreashold = arrow.utcnow().shift(minutes=-sell_timeout).datetime for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): try: @@ -521,10 +527,12 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ if int(order['remaining']) == 0: continue - if order['side'] == 'buy' and ordertime < timeoutthreashold: - self.handle_timedout_limit_buy(trade, order) - elif order['side'] == 'sell' and ordertime < timeoutthreashold: - self.handle_timedout_limit_sell(trade, order) + # Check if trade is still actually open + if order['status'] == 'open': + if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: + self.handle_timedout_limit_buy(trade, order) + elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: + self.handle_timedout_limit_sell(trade, order) # FIX: 20180110, why is cancel.order unconditionally here, whereas # it is conditionally called in the diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6982b36cb..316fba508 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -38,6 +38,8 @@ class BacktestResult(NamedTuple): close_index: int trade_duration: float open_at_end: bool + open_rate: float + close_rate: float class Backtesting(object): @@ -116,11 +118,10 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - records = [(trade_entry.pair, trade_entry.profit_percent, - trade_entry.open_time.timestamp(), - trade_entry.close_time.timestamp(), - trade_entry.open_index - 1, trade_entry.trade_duration) - for index, trade_entry in results.iterrows()] + records = [(t.pair, t.profit_percent, t.open_time.timestamp(), + t.close_time.timestamp(), t.open_index - 1, t.trade_duration, + t.open_rate, t.close_rate, t.open_at_end) + for index, t in results.iterrows()] if records: logger.info('Dumping backtest results to %s', recordfilename) @@ -159,7 +160,9 @@ class Backtesting(object): trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, - open_at_end=False + open_at_end=False, + open_rate=buy_row.close, + close_rate=sell_row.close ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period @@ -172,7 +175,9 @@ class Backtesting(object): trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, - open_at_end=True + open_at_end=True, + open_rate=buy_row.close, + close_rate=sell_row.close ) logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, btr.profit_percent, btr.profit_abs) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d9ccaa325..4877fe556 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -100,7 +100,10 @@ def default_conf(): "0": 0.04 }, "stoploss": -0.10, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 53e59b34b..6cf7f11b0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,13 +14,29 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange +def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): + """Function to test ccxt exception handling """ + + with pytest.raises(TemporaryError): + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1 + + with pytest.raises(OperationalException): + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 + + def test_init(default_conf, mocker, caplog): caplog.set_level(logging.INFO) get_patched_exchange(mocker, default_conf) assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) -def test_init_exception(default_conf): +def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' with pytest.raises( @@ -28,6 +44,13 @@ def test_init_exception(default_conf): match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): Exchange(default_conf) + default_conf['exchange']['name'] = 'binance' + with pytest.raises( + OperationalException, + match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError)) + Exchange(default_conf) + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() @@ -97,6 +120,20 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): Exchange(conf) +def test_exchangehas(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf) + assert not exchange.exchange_has('ASDFASDF') + api_mock = MagicMock() + + type(api_mock).has = PropertyMock(return_value={'deadbeef': True}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert exchange.exchange_has("deadbeef") + + type(api_mock).has = PropertyMock(return_value={'deadbeef': False}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert not exchange.exchange_has("deadbeef") + + def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) @@ -216,6 +253,11 @@ def test_get_balance_prod(default_conf, mocker): exchange.get_balance(currency='BTC') + with pytest.raises(TemporaryError, match=r'.*balance due to malformed exchange response:.*'): + exchange = get_patched_exchange(mocker, default_conf, api_mock) + mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) + exchange.get_balance(currency='BTC') + def test_get_balances_dry_run(default_conf, mocker): default_conf['dry_run'] = True @@ -243,17 +285,8 @@ def test_get_balances_prod(default_conf, mocker): assert exchange.get_balances()['1ST']['total'] == 10.0 assert exchange.get_balances()['1ST']['used'] == 0.0 - with pytest.raises(TemporaryError): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_balances() - assert api_mock.fetch_balance.call_count == API_RETRY_COUNT + 1 - - with pytest.raises(OperationalException): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_balances() - assert api_mock.fetch_balance.call_count == 1 + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_balances", "fetch_balance") def test_get_tickers(default_conf, mocker): @@ -282,15 +315,8 @@ def test_get_tickers(default_conf, mocker): assert tickers['BCH/BTC']['bid'] == 0.6 assert tickers['BCH/BTC']['ask'] == 0.5 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_tickers() - - with pytest.raises(OperationalException): - api_mock.fetch_tickers = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_tickers() + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_tickers", "fetch_tickers") with pytest.raises(OperationalException): api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NotSupported) @@ -345,15 +371,9 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_ticker = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker(pair='ETH/BTC', refresh=True) - - with pytest.raises(OperationalException): - api_mock.fetch_ticker = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker(pair='ETH/BTC', refresh=True) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker", "fetch_ticker", + pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock) @@ -416,17 +436,14 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][4] == 9 assert ticks[0][5] == 10 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - # new symbol to get around cache - exchange.get_ticker_history('ABCD/BTC', default_conf['ticker_interval']) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) - with pytest.raises(OperationalException): - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) + with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) exchange = get_patched_exchange(mocker, default_conf, api_mock) - # new symbol to get around cache - exchange.get_ticker_history('EFGH/BTC', default_conf['ticker_interval']) + exchange.get_ticker_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) def test_get_ticker_history_sort(default_conf, mocker): @@ -515,24 +532,15 @@ def test_cancel_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.cancel_order(order_id='_', pair='TKN/BTC') == 123 - with pytest.raises(TemporaryError): - api_mock.cancel_order = MagicMock(side_effect=ccxt.NetworkError) - - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(DependencyException): api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.cancel_order(order_id='_', pair='TKN/BTC') assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(OperationalException): - api_mock.cancel_order = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == 1 + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "cancel_order", "cancel_order", + order_id='_', pair='TKN/BTC') def test_get_order(default_conf, mocker): @@ -550,23 +558,15 @@ def test_get_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.get_order('X', 'TKN/BTC') == 456 - with pytest.raises(TemporaryError): - api_mock.fetch_order = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(DependencyException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(OperationalException): - api_mock.fetch_order = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == 1 + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_order', 'fetch_order', + order_id='_', pair='TKN/BTC') def test_name(default_conf, mocker): @@ -651,19 +651,12 @@ def test_get_trades_for_order(default_conf, mocker): assert len(orders) == 1 assert orders[0]['price'] == 165 - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_trades_for_order(order_id, 'LTC/BTC', since) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_trades_for_order', 'fetch_my_trades', + order_id=order_id, pair='LTC/BTC', since=since) - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_trades_for_order(order_id, 'LTC/BTC', since) - assert api_mock.fetch_my_trades.call_count == API_RETRY_COUNT + 1 + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + assert exchange.get_trades_for_order(order_id, 'LTC/BTC', since) == [] def test_get_markets(default_conf, mocker, markets): @@ -677,19 +670,8 @@ def test_get_markets(default_conf, mocker, markets): assert ret[0]["id"] == "ethbtc" assert ret[0]["symbol"] == "ETH/BTC" - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_markets() - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_markets() - assert api_mock.fetch_markets.call_count == API_RETRY_COUNT + 1 + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_markets', 'fetch_markets') def test_get_fee(default_conf, mocker): @@ -704,19 +686,8 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_fee() - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_fee() - assert api_mock.calculate_fee.call_count == API_RETRY_COUNT + 1 + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_fee', 'calculate_fee') def test_get_amount_lots(default_conf, mocker): diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index c3d2ad572..49a6370bb 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -627,9 +627,13 @@ def test_backtest_record(default_conf, fee, mocker): Arrow(2017, 11, 14, 22, 10, 00).datetime, Arrow(2017, 11, 14, 22, 43, 00).datetime, Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "open_index": [1, 119, 153, 185], "close_index": [118, 151, 184, 199], - "trade_duration": [123, 34, 31, 14]}) + "trade_duration": [123, 34, 31, 14], + "open_at_end": [False, False, False, True] + }) backtesting._store_backtest_result("backtest-result.json", results) assert len(results) == 4 # Assert file_dump_json was only called once @@ -640,12 +644,16 @@ def test_backtest_record(default_conf, fee, mocker): # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) # Below follows just a typecheck of the schema/type of trade-records oix = None - for (pair, profit, date_buy, date_sell, buy_index, dur) in records: + for (pair, profit, date_buy, date_sell, buy_index, dur, + openr, closer, open_at_end) in records: assert pair == 'UNITTEST/BTC' - isinstance(profit, float) + assert isinstance(profit, float) # FIX: buy/sell should be converted to ints - isinstance(date_buy, str) - isinstance(date_sell, str) + assert isinstance(date_buy, float) + assert isinstance(date_sell, float) + assert isinstance(openr, float) + assert isinstance(closer, float) + assert isinstance(open_at_end, bool) isinstance(buy_index, pd._libs.tslib.Timestamp) if oix: assert buy_index > oix diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 7b2786421..dde2dd6d9 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -348,6 +348,34 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) assert result is None + # no cost Min + mocker.patch( + 'freqtrade.exchange.Exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {"min": None}, + 'amount': {} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + + # no amount Min + mocker.patch( + 'freqtrade.exchange.Exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {}, + 'amount': {"min": None} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + # empty 'cost'/'amount' section mocker.patch( 'freqtrade.exchange.Exchange.get_markets', @@ -1124,7 +1152,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe Trade.session.add(trade_buy) # check it does cancel buy orders over the time limit - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() @@ -1165,7 +1193,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, Trade.session.add(trade_sell) # check it does cancel sell orders over the time limit - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 assert trade_sell.is_open is True @@ -1205,7 +1233,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old # check it does cancel buy orders over the time limit # note this is for a partially-complete buy order - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() @@ -1256,7 +1284,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - 'recent call last):\n.*' ) - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert filter(regexp.match, caplog.record_tuples) @@ -1599,6 +1627,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) @@ -1616,7 +1645,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke assert freqtrade.handle_trade(trade) is True -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1634,6 +1663,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) diff --git a/requirements.txt b/requirements.txt index d4f8b2f06..50441879a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -ccxt==1.14.272 -SQLAlchemy==1.2.8 +ccxt==1.14.301 +SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 7f9641222..1cc6b818a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -25,11 +25,13 @@ Example of usage: --indicators2 fastk,fastd """ import logging -import os import sys +import json +from pathlib import Path from argparse import Namespace from typing import Dict, List, Any +import pandas as pd import plotly.graph_objs as go from plotly import tools from plotly.offline import plot @@ -37,7 +39,7 @@ from plotly.offline import plot import freqtrade.optimize as optimize from freqtrade import persistence from freqtrade.analyze import Analyze -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange from freqtrade.exchange import Exchange from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade @@ -46,6 +48,45 @@ logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} +def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: + trades: pd.DataFrame = pd.DataFrame() + if args.db_url: + persistence.init(_CONF) + columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date, t.close_date, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp()) + for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + columns=columns) + + if args.exportfilename: + file = Path(args.exportfilename) + # must align with columns in backtest.py + columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] + with file.open() as f: + data = json.load(f) + trades = pd.DataFrame(data, columns=columns) + trades = trades.loc[trades["pair"] == pair] + if timerange: + if timerange.starttype == 'date': + trades = trades.loc[trades["opents"] >= timerange.startts] + if timerange.stoptype == 'date': + trades = trades.loc[trades["opents"] <= timerange.stopts] + + trades['opents'] = pd.to_datetime(trades['opents'], + unit='s', + utc=True, + infer_datetime_format=True) + trades['closets'] = pd.to_datetime(trades['closets'], + unit='s', + utc=True, + infer_datetime_format=True) + return trades + + def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe @@ -102,31 +143,32 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if tickers == {}: exit() + if args.db_url and args.exportfilename: + logger.critical("Can only specify --db-url or --export-filename") # Get trades already made from the DB - trades: List[Trade] = [] - if args.db_url: - persistence.init(_CONF) - trades = Trade.query.filter(Trade.pair.is_(pair)).all() + trades = load_trades(args, pair, timerange) dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) - if len(dataframe.index) > 750: - logger.warning('Ticker contained more than 750 candles, clipping.') - + if len(dataframe.index) > args.plot_limit: + logger.warning('Ticker contained more than %s candles as defined ' + 'with --plot-limit, clipping.', args.plot_limit) + dataframe = dataframe.tail(args.plot_limit) + trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] fig = generate_graph( pair=pair, trades=trades, - data=dataframe.tail(750), + data=dataframe, args=args ) - plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) + plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html'))) -def generate_graph(pair, trades, data, args) -> tools.make_subplots: +def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: """ Generate the graph from the data generated by Backtesting or from DB :param pair: Pair to Display on the graph @@ -187,8 +229,8 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: ) trade_buys = go.Scattergl( - x=[t.open_date.isoformat() for t in trades], - y=[t.open_rate for t in trades], + x=trades["opents"], + y=trades["open_rate"], mode='markers', name='trade_buy', marker=dict( @@ -199,8 +241,8 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: ) ) trade_sells = go.Scattergl( - x=[t.close_date.isoformat() for t in trades], - y=[t.close_rate for t in trades], + x=trades["closets"], + y=trades["close_rate"], mode='markers', name='trade_sell', marker=dict( @@ -299,11 +341,17 @@ def plot_parse_args(args: List[str]) -> Namespace: default='macd', dest='indicators2', ) - + arguments.parser.add_argument( + '--plot-limit', + help='Specify tick limit for plotting - too high values cause huge files - ' + 'Default: %(default)s', + dest='plot_limit', + default=750, + type=int, + ) arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) arguments.backtesting_options(arguments.parser) - return arguments.parse_args()