diff --git a/docs/hyperopt.md b/docs/hyperopt.md index e25f35c35..b4e42de16 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -62,7 +62,7 @@ If you have updated the buy strategy, ie. changed the contents of #### Sell optimization -Similar to the buy-signal above, sell-signals can also be optimized. +Similar to the buy-signal above, sell-signals can also be optimized. Place the corresponding settings into the following methods * Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. @@ -163,7 +163,7 @@ running at least several thousand evaluations. The `--spaces all` flag determines that all possible parameters should be optimized. Possibilities are listed below. !!! Warning -When switching parameters or changing configuration options, the file `user_data/hyperopt_results.pickle` should be removed. It's used to be able to continue interrupted calculations, but does not detect changes to settings or the hyperopt file. + When switching parameters or changing configuration options, the file `user_data/hyperopt_results.pickle` should be removed. It's used to be able to continue interrupted calculations, but does not detect changes to settings or the hyperopt file. ### Execute Hyperopt with Different Ticker-Data Source diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 54f9b8213..f41520bd9 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -1,5 +1,5 @@ # SQL Helper -This page constains some help if you want to edit your sqlite db. +This page contains some help if you want to edit your sqlite db. ## Install sqlite3 **Ubuntu/Debian installation** @@ -66,11 +66,11 @@ SELECT * FROM trades; ## Fix trade still open after a manual sell on the exchange !!! Warning - Manually selling on the exchange should not be done by default, since the bot does not detect this and will try to sell anyway. - /foresell should accomplish the same thing. + Manually selling a pair on the exchange will not be detected by the bot and it will try to sell anyway. Whenever possible, forcesell should be used to accomplish the same thing. + It is strongly advised to backup your database file before making any manual changes. !!! Note - This should not be necessary after /forcesell, as forcesell orders are closed automatically by the bot on the next iteration. + This should not be necessary after /forcesell, as forcesell orders are closed automatically by the bot on the next iteration. ```sql UPDATE trades diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 4cc8eaa5c..9d6877318 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -1,13 +1,13 @@ # Telegram usage -This page explains how to command your bot with Telegram. - ## Prerequisite + To control your bot with Telegram, you need first to [set up a Telegram bot](installation.md) and add your Telegram API keys into your config file. ## Telegram commands + Per default, the Telegram bot shows predefined commands. Some commands are only available by sending them to the bot. The table below list the official commands. You can ask at any moment for help with `/help`. diff --git a/docs/webhook-config.md b/docs/webhook-config.md index 2b5365e32..811b57f9b 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -1,7 +1,5 @@ # Webhook usage -This page explains how to configure your bot to talk to webhooks. - ## Configuration Enable webhooks by adding a webhook-section to your configuration file, and setting `webhook.enabled` to `true`. @@ -39,32 +37,30 @@ Different payloads can be configured for different events. Not all fields are ne The fields in `webhook.webhookbuy` are filled when the bot executes a buy. Parameters are filled using string.format. Possible parameters are: -* exchange -* pair -* limit -* stake_amount -* stake_amount_fiat -* stake_currency -* fiat_currency +* `exchange` +* `pair` +* `limit` +* `stake_amount` +* `stake_currency` +* `fiat_currency` ### Webhooksell The fields in `webhook.webhooksell` are filled when the bot sells a trade. Parameters are filled using string.format. Possible parameters are: -* exchange -* pair -* gain -* limit -* amount -* open_rate -* current_rate -* profit_amount -* profit_percent -* profit_fiat -* stake_currency -* fiat_currency -* sell_reason +* `exchange` +* `pair` +* `gain` +* `limit` +* `amount` +* `open_rate` +* `current_rate` +* `profit_amount` +* `profit_percent` +* `stake_currency` +* `fiat_currency` +* `sell_reason` ### Webhookstatus diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e0360cdef..3e4d642cb 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -210,6 +210,32 @@ class Backtesting(object): logger.info('Dumping backtest results to %s', recordfilename) file_dump_json(recordfilename, records) + def _get_ticker_list(self, processed) -> Dict[str, DataFrame]: + """ + Helper function to convert a processed tickerlist into a list for performance reasons. + + Used by backtest() - so keep this optimized for performance. + """ + headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high'] + ticker: Dict = {} + # Create ticker dict + for pair, pair_data in processed.items(): + pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run + + ticker_data = self.advise_sell( + self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() + + # to avoid using data from future, we buy/sell with signal from previous candle + ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + + ticker_data.drop(ticker_data.head(1).index, inplace=True) + + # Convert from Pandas to list for performance reasons + # (Looping Pandas is slow.) + ticker[pair] = [x for x in ticker_data.itertuples()] + return ticker + def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[BacktestResult]: @@ -304,7 +330,6 @@ class Backtesting(object): position_stacking: do we allow position stacking? (default: False) :return: DataFrame """ - headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) position_stacking = args.get('position_stacking', False) @@ -312,54 +337,50 @@ class Backtesting(object): end_date = args['end_date'] trades = [] trade_count_lock: Dict = {} - ticker: Dict = {} - pairs = [] - # Create ticker dict - for pair, pair_data in processed.items(): - pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - ticker_data = self.advise_sell( - self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - - # to avoid using data from future, we buy/sell with signal from previous candle - ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) - - ticker_data.drop(ticker_data.head(1).index, inplace=True) - - # Convert from Pandas to list for performance reasons - # (Looping Pandas is slow.) - ticker[pair] = [x for x in ticker_data.itertuples()] - pairs.append(pair) + # Dict of ticker-lists for performance (looping lists is a lot faster than dataframes) + ticker: Dict = self._get_ticker_list(processed) lock_pair_until: Dict = {} + # Indexes per pair, so some pairs are allowed to have a missing start. + indexes: Dict = {} tmp = start_date + timedelta(minutes=self.ticker_interval_mins) - index = 0 - # Loop timerange and test per pair + + # Loop timerange and get candle for each pair at that point in time while tmp < end_date: - # print(f"time: {tmp}") + for i, pair in enumerate(ticker): + if pair not in indexes: + indexes[pair] = 0 + try: - row = ticker[pair][index] + row = ticker[pair][indexes[pair]] except IndexError: - # missing Data for one pair ... + # missing Data for one pair at the end. # Warnings for this are shown by `validate_backtest_data` continue + # Waits until the time-counter reaches the start of the data for this pair. + if row.date > tmp.datetime: + continue + + indexes[pair] += 1 + if row.buy == 0 or row.sell == 1: continue # skip rows where no buy signal or that would immediately sell off - if not position_stacking: - if pair in lock_pair_until and row.date <= lock_pair_until[pair]: - continue + if (not position_stacking and pair in lock_pair_until + and row.date <= lock_pair_until[pair]): + # without positionstacking, we can only have one open trade per pair. + continue + if max_open_trades > 0: # Check if max_open_trades has already been reached for the given date if not trade_count_lock.get(row.date, 0) < max_open_trades: continue - trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - trade_entry = self._get_sell_trade_entry(pair, row, ticker[pair][index + 1:], + trade_entry = self._get_sell_trade_entry(pair, row, ticker[pair][indexes[pair]:], trade_count_lock, args) if trade_entry: @@ -367,11 +388,10 @@ class Backtesting(object): trades.append(trade_entry) else: # Set lock_pair_until to end of testing period if trade could not be closed - # This happens only if the buy-signal was with the last candle - lock_pair_until[pair] = end_date + lock_pair_until[pair] = end_date.datetime + # Move time one configured time_interval ahead. tmp += timedelta(minutes=self.ticker_interval_mins) - index += 1 return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index c1c1b49cd..af8674188 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -122,8 +122,8 @@ def test_edge_results(edge_conf, mocker, caplog, data) -> None: for c, trade in enumerate(data.trades): res = results.iloc[c] assert res.exit_type == trade.sell_reason - assert res.open_time == _get_frame_time_from_offset(trade.open_tick) - assert res.close_time == _get_frame_time_from_offset(trade.close_tick) + assert arrow.get(res.open_time) == _get_frame_time_from_offset(trade.open_tick) + assert arrow.get(res.close_time) == _get_frame_time_from_offset(trade.close_tick) def test_adjust(mocker, edge_conf): diff --git a/freqtrade/tests/optimize/__init__.py b/freqtrade/tests/optimize/__init__.py index 9203ec19c..227050770 100644 --- a/freqtrade/tests/optimize/__init__.py +++ b/freqtrade/tests/optimize/__init__.py @@ -33,7 +33,7 @@ class BTContainer(NamedTuple): def _get_frame_time_from_offset(offset): return ticker_start_time.shift(minutes=(offset * timeframe_to_minutes(tests_ticker_interval)) - ).datetime.replace(tzinfo=None) + ).datetime def _build_backtest_dataframe(ticker_with_signals): diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index d0b21b8f4..0596cffb5 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -685,25 +685,32 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): assert len(results.loc[results.open_at_end]) == 0 -def test_backtest_multi_pair(default_conf, fee, mocker): +@pytest.mark.parametrize("pair", ['ADA/BTC', 'LTC/BTC']) +@pytest.mark.parametrize("tres", [0, 20, 30]) +def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair): def _trend_alternate_hold(dataframe=None, metadata=None): """ - Buy every 8th candle - sell every other 8th -2 (hold on to pairs a bit) + Buy every xth candle - sell every other xth -2 (hold on to pairs a bit) """ - multi = 8 + if metadata['pair'] in('ETH/BTC', 'LTC/BTC'): + multi = 20 + else: + multi = 18 dataframe['buy'] = np.where(dataframe.index % multi == 0, 1, 0) dataframe['sell'] = np.where((dataframe.index + multi - 2) % multi == 0, 1, 0) - if metadata['pair'] in('ETH/BTC', 'LTC/BTC'): - dataframe['buy'] = dataframe['buy'].shift(-4) - dataframe['sell'] = dataframe['sell'].shift(-4) return dataframe mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) + pairs = ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC', 'NXT/BTC'] data = history.load_data(datadir=None, ticker_interval='5m', pairs=pairs) + # Only use 500 lines to increase performance data = trim_dictlist(data, -500) + + # Remove data for one pair from the beginning of the data + data[pair] = data[pair][tres:] # We need to enable sell-signal - otherwise it sells on ROI!! default_conf['experimental'] = {"use_sell_signal": True} default_conf['ticker_interval'] = '5m' diff --git a/mkdocs.yml b/mkdocs.yml index 9a6fec851..ecac265c1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,11 +3,12 @@ nav: - About: index.md - Installation: installation.md - Configuration: configuration.md - - Start the bot: bot-usage.md - - Stoploss: stoploss.md - Custom Strategy: bot-optimization.md - - Telegram: telegram-usage.md - - Web Hook: webhook-config.md + - Stoploss: stoploss.md + - Start the bot: bot-usage.md + - Control the bot: + - Telegram: telegram-usage.md + - Web Hook: webhook-config.md - Backtesting: backtesting.md - Hyperopt: hyperopt.md - Edge positioning: edge.md