From 1be4c59481bf706211416f8a544635b3f35e7fbe Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 3 May 2019 16:48:07 +0300 Subject: [PATCH 01/84] qtpylib/indicators.py updated --- freqtrade/vendor/qtpylib/indicators.py | 205 ++++++++++++++----------- 1 file changed, 119 insertions(+), 86 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index 3866d36c1..d5860ff5f 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -4,13 +4,13 @@ # QTPyLib: Quantitative Trading Python Library # https://github.com/ranaroussi/qtpylib # -# Copyright 2016 Ran Aroussi +# Copyright 2016-2018 Ran Aroussi # -# Licensed under the GNU Lesser General Public License, v3.0 (the "License"); +# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# https://www.gnu.org/licenses/lgpl-3.0.en.html +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -19,8 +19,8 @@ # limitations under the License. # -import sys import warnings +import sys from datetime import datetime, timedelta import numpy as np @@ -62,19 +62,20 @@ def numpy_rolling_series(func): @numpy_rolling_series def numpy_rolling_mean(data, window, as_source=False): - return np.mean(numpy_rolling_window(data, window), -1) + return np.mean(numpy_rolling_window(data, window), axis=-1) @numpy_rolling_series def numpy_rolling_std(data, window, as_source=False): - return np.std(numpy_rolling_window(data, window), -1) + return np.std(numpy_rolling_window(data, window), axis=-1, ddof=1) + # --------------------------------------------- def session(df, start='17:00', end='16:00'): """ remove previous globex day from df """ - if len(df) == 0: + if df.empty: return df # get start/end/now as decimals @@ -103,47 +104,50 @@ def session(df, start='17:00', end='16:00'): return df.copy() - # --------------------------------------------- + def heikinashi(bars): bars = bars.copy() bars['ha_close'] = (bars['open'] + bars['high'] + bars['low'] + bars['close']) / 4 - bars['ha_open'] = (bars['open'].shift(1) + bars['close'].shift(1)) / 2 - bars.loc[:1, 'ha_open'] = bars['open'].values[0] - for x in range(2): - bars.loc[1:, 'ha_open'] = ( - (bars['ha_open'].shift(1) + bars['ha_close'].shift(1)) / 2)[1:] + # ha open + bars.loc[:1, 'ha_open'] = (bars['open'] + bars['close']) / 2 + prev_open = bars[:1]['ha_open'].values[0] + for idx, _ in bars[1:][['ha_open', 'ha_close']].iterrows(): + loc = bars.index.get_loc(idx) + prev_open = (prev_open + bars['ha_close'].values[loc - 1]) / 2 + bars.loc[loc:loc + 1, 'ha_open'] = prev_open bars['ha_high'] = bars.loc[:, ['high', 'ha_open', 'ha_close']].max(axis=1) bars['ha_low'] = bars.loc[:, ['low', 'ha_open', 'ha_close']].min(axis=1) - return pd.DataFrame( - index=bars.index, - data={ - 'open': bars['ha_open'], - 'high': bars['ha_high'], - 'low': bars['ha_low'], - 'close': bars['ha_close']}) - + return pd.DataFrame(index=bars.index, + data={'open': bars['ha_open'], + 'high': bars['ha_high'], + 'low': bars['ha_low'], + 'close': bars['ha_close']}) # --------------------------------------------- -def tdi(series, rsi_len=13, bollinger_len=34, rsi_smoothing=2, - rsi_signal_len=7, bollinger_std=1.6185): - rsi_series = rsi(series, rsi_len) - bb_series = bollinger_bands(rsi_series, bollinger_len, bollinger_std) - signal = sma(rsi_series, rsi_signal_len) - rsi_series = sma(rsi_series, rsi_smoothing) + +def tdi(series, rsi_lookback=13, rsi_smooth_len=2, + rsi_signal_len=7, bb_lookback=34, bb_std=1.6185): + + rsi_data = rsi(series, rsi_lookback) + rsi_smooth = sma(rsi_data, rsi_smooth_len) + rsi_signal = sma(rsi_data, rsi_signal_len) + + bb_series = bollinger_bands(rsi_data, bb_lookback, bb_std) return pd.DataFrame(index=series.index, data={ - "rsi": rsi_series, - "signal": signal, - "bbupper": bb_series['upper'], - "bblower": bb_series['lower'], - "bbmid": bb_series['mid'] + "rsi": rsi_data, + "rsi_signal": rsi_signal, + "rsi_smooth": rsi_smooth, + "rsi_bb_upper": bb_series['upper'], + "rsi_bb_lower": bb_series['lower'], + "rsi_bb_mid": bb_series['mid'] }) # --------------------------------------------- @@ -163,8 +167,8 @@ def awesome_oscillator(df, weighted=False, fast=5, slow=34): # --------------------------------------------- -def nans(len=1): - mtx = np.empty(len) +def nans(length=1): + mtx = np.empty(length) mtx[:] = np.nan return mtx @@ -222,7 +226,7 @@ def crossed(series1, series2, direction=None): if isinstance(series1, np.ndarray): series1 = pd.Series(series1) - if isinstance(series2, int) or isinstance(series2, float) or isinstance(series2, np.ndarray): + if isinstance(series2, (float, int, np.ndarray)): series2 = pd.Series(index=series1.index, data=series2) if direction is None or direction == "above": @@ -256,7 +260,7 @@ def rolling_std(series, window=200, min_periods=None): else: try: return series.rolling(window=window, min_periods=min_periods).std() - except BaseException: + except Exception as e: return pd.Series(series).rolling(window=window, min_periods=min_periods).std() # --------------------------------------------- @@ -269,7 +273,7 @@ def rolling_mean(series, window=200, min_periods=None): else: try: return series.rolling(window=window, min_periods=min_periods).mean() - except BaseException: + except Exception as e: return pd.Series(series).rolling(window=window, min_periods=min_periods).mean() # --------------------------------------------- @@ -279,7 +283,7 @@ def rolling_min(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: return series.rolling(window=window, min_periods=min_periods).min() - except BaseException: + except Exception as e: return pd.Series(series).rolling(window=window, min_periods=min_periods).min() @@ -289,7 +293,7 @@ def rolling_max(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: return series.rolling(window=window, min_periods=min_periods).min() - except BaseException: + except Exception as e: return pd.Series(series).rolling(window=window, min_periods=min_periods).min() @@ -299,16 +303,17 @@ def rolling_weighted_mean(series, window=200, min_periods=None): min_periods = window if min_periods is None else min_periods try: return series.ewm(span=window, min_periods=min_periods).mean() - except BaseException: + except Exception as e: return pd.ewma(series, span=window, min_periods=min_periods) # --------------------------------------------- -def hull_moving_average(series, window=200): - wma = (2 * rolling_weighted_mean(series, window=window / 2)) - \ - rolling_weighted_mean(series, window=window) - return rolling_weighted_mean(wma, window=np.sqrt(window)) +def hull_moving_average(series, window=200, min_periods=None): + min_periods = window if min_periods is None else min_periods + ma = (2 * rolling_weighted_mean(series, window / 2, min_periods)) - \ + rolling_weighted_mean(series, window, min_periods) + return rolling_weighted_mean(ma, np.sqrt(window), min_periods) # --------------------------------------------- @@ -325,8 +330,8 @@ def wma(series, window=200, min_periods=None): # --------------------------------------------- -def hma(series, window=200): - return hull_moving_average(series, window=window) +def hma(series, window=200, min_periods=None): + return hull_moving_average(series, window=window, min_periods=min_periods) # --------------------------------------------- @@ -361,7 +366,7 @@ def rolling_vwap(bars, window=200, min_periods=None): min_periods=min_periods).sum() right = volume.rolling(window=window, min_periods=min_periods).sum() - return pd.Series(index=bars.index, data=(left / right)) + return pd.Series(index=bars.index, data=(left / right)).replace([np.inf, -np.inf], float('NaN')).ffill() # --------------------------------------------- @@ -370,6 +375,7 @@ def rsi(series, window=14): """ compute the n period relative strength indicator """ + # 100-(100/relative_strength) deltas = np.diff(series) seed = deltas[:window + 1] @@ -406,13 +412,13 @@ def macd(series, fast=3, slow=10, smooth=16): using a fast and slow exponential moving avg' return value is emaslow, emafast, macd which are len(x) arrays """ - macd = rolling_weighted_mean(series, window=fast) - \ + macd_line = rolling_weighted_mean(series, window=fast) - \ rolling_weighted_mean(series, window=slow) - signal = rolling_weighted_mean(macd, window=smooth) - histogram = macd - signal - # return macd, signal, histogram + signal = rolling_weighted_mean(macd_line, window=smooth) + histogram = macd_line - signal + # return macd_line, signal, histogram return pd.DataFrame(index=series.index, data={ - 'macd': macd.values, + 'macd': macd_line.values, 'signal': signal.values, 'histogram': histogram.values }) @@ -421,14 +427,14 @@ def macd(series, fast=3, slow=10, smooth=16): # --------------------------------------------- def bollinger_bands(series, window=20, stds=2): - sma = rolling_mean(series, window=window) - std = rolling_std(series, window=window) - upper = sma + std * stds - lower = sma - std * stds + ma = rolling_mean(series, window=window, min_periods=1) + std = rolling_std(series, window=window, min_periods=1) + upper = ma + std * stds + lower = ma - std * stds return pd.DataFrame(index=series.index, data={ 'upper': upper, - 'mid': sma, + 'mid': ma, 'lower': lower }) @@ -454,7 +460,7 @@ def returns(series): try: res = (series / series.shift(1) - 1).replace([np.inf, -np.inf], float('NaN')) - except BaseException: + except Exception as e: res = nans(len(series)) return pd.Series(index=series.index, data=res) @@ -465,8 +471,8 @@ def returns(series): def log_returns(series): try: res = np.log(series / series.shift(1) - ).replace([np.inf, -np.inf], float('NaN')) - except BaseException: + ).replace([np.inf, -np.inf], float('NaN')) + except Exception as e: res = nans(len(series)) return pd.Series(index=series.index, data=res) @@ -477,9 +483,9 @@ def log_returns(series): def implied_volatility(series, window=252): try: logret = np.log(series / series.shift(1) - ).replace([np.inf, -np.inf], float('NaN')) + ).replace([np.inf, -np.inf], float('NaN')) res = numpy_rolling_std(logret, window) * np.sqrt(window) - except BaseException: + except Exception as e: res = nans(len(series)) return pd.Series(index=series.index, data=res) @@ -530,32 +536,52 @@ def stoch(df, window=14, d=3, k=3, fast=False): compute the n period relative strength indicator http://excelta.blogspot.co.il/2013/09/stochastic-oscillator-technical.html """ - highs_ma = pd.concat([df['high'].shift(i) - for i in np.arange(window)], 1).apply(list, 1) - highs_ma = highs_ma.T.max().T - lows_ma = pd.concat([df['low'].shift(i) - for i in np.arange(window)], 1).apply(list, 1) - lows_ma = lows_ma.T.min().T + my_df = pd.DataFrame(index=df.index) - fast_k = ((df['close'] - lows_ma) / (highs_ma - lows_ma)) * 100 - fast_d = numpy_rolling_mean(fast_k, d) + my_df['rolling_max'] = df['high'].rolling(window).max() + my_df['rolling_min'] = df['low'].rolling(window).min() + + my_df['fast_k'] = 100 * (df['close'] - my_df['rolling_min'])/(my_df['rolling_max'] - my_df['rolling_min']) + my_df['fast_d'] = my_df['fast_k'].rolling(d).mean() if fast: - data = { - 'k': fast_k, - 'd': fast_d - } + return my_df.loc[:, ['fast_k', 'fast_d']] - else: - slow_k = numpy_rolling_mean(fast_k, k) - slow_d = numpy_rolling_mean(slow_k, d) - data = { - 'k': slow_k, - 'd': slow_d - } + my_df['slow_k'] = my_df['fast_k'].rolling(k).mean() + my_df['slow_d'] = my_df['slow_k'].rolling(d).mean() - return pd.DataFrame(index=df.index, data=data) + return my_df.loc[:, ['slow_k', 'slow_d']] + +# --------------------------------------------- + + +def zlma(series, window=20, min_periods=None, kind="ema"): + """ + John Ehlers' Zero lag (exponential) moving average + https://en.wikipedia.org/wiki/Zero_lag_exponential_moving_average + """ + min_periods = window if min_periods is None else min_periods + + lag = (window - 1) // 2 + series = 2 * series - series.shift(lag) + if kind in ['ewm', 'ema']: + return wma(series, lag, min_periods) + elif kind == "hma": + return hma(series, lag, min_periods) + return sma(series, lag, min_periods) + + +def zlema(series, window, min_periods=None): + return zlma(series, window, min_periods, kind="ema") + + +def zlsma(series, window, min_periods=None): + return zlma(series, window, min_periods, kind="sma") + + +def zlhma(series, window, min_periods=None): + return zlma(series, window, min_periods, kind="hma") # --------------------------------------------- @@ -571,13 +597,13 @@ def zscore(bars, window=20, stds=1, col='close'): def pvt(bars): """ Price Volume Trend """ - pvt = ((bars['close'] - bars['close'].shift(1)) / - bars['close'].shift(1)) * bars['volume'] - return pvt.cumsum() - + trend = ((bars['close'] - bars['close'].shift(1)) / + bars['close'].shift(1)) * bars['volume'] + return trend.cumsum() # ============================================= + PandasObject.session = session PandasObject.atr = atr PandasObject.bollinger_bands = bollinger_bands @@ -613,4 +639,11 @@ PandasObject.rolling_weighted_mean = rolling_weighted_mean PandasObject.sma = sma PandasObject.wma = wma +PandasObject.ema = wma PandasObject.hma = hma + +PandasObject.zlsma = zlsma +PandasObject.zlwma = zlema +PandasObject.zlema = zlema +PandasObject.zlhma = zlhma +PandasObject.zlma = zlma From 66c2bdd65a06dc06d535c903869269840c8f1c3f Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 3 May 2019 16:58:51 +0300 Subject: [PATCH 02/84] flake happy --- freqtrade/vendor/qtpylib/indicators.py | 28 +++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index d5860ff5f..8586968e8 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -260,7 +260,7 @@ def rolling_std(series, window=200, min_periods=None): else: try: return series.rolling(window=window, min_periods=min_periods).std() - except Exception as e: + except Exception as e: # noqa: F841 return pd.Series(series).rolling(window=window, min_periods=min_periods).std() # --------------------------------------------- @@ -273,7 +273,7 @@ def rolling_mean(series, window=200, min_periods=None): else: try: return series.rolling(window=window, min_periods=min_periods).mean() - except Exception as e: + except Exception as e: # noqa: F841 return pd.Series(series).rolling(window=window, min_periods=min_periods).mean() # --------------------------------------------- @@ -283,7 +283,7 @@ def rolling_min(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: return series.rolling(window=window, min_periods=min_periods).min() - except Exception as e: + except Exception as e: # noqa: F841 return pd.Series(series).rolling(window=window, min_periods=min_periods).min() @@ -293,7 +293,7 @@ def rolling_max(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: return series.rolling(window=window, min_periods=min_periods).min() - except Exception as e: + except Exception as e: # noqa: F841 return pd.Series(series).rolling(window=window, min_periods=min_periods).min() @@ -303,7 +303,7 @@ def rolling_weighted_mean(series, window=200, min_periods=None): min_periods = window if min_periods is None else min_periods try: return series.ewm(span=window, min_periods=min_periods).mean() - except Exception as e: + except Exception as e: # noqa: F841 return pd.ewma(series, span=window, min_periods=min_periods) @@ -366,7 +366,8 @@ def rolling_vwap(bars, window=200, min_periods=None): min_periods=min_periods).sum() right = volume.rolling(window=window, min_periods=min_periods).sum() - return pd.Series(index=bars.index, data=(left / right)).replace([np.inf, -np.inf], float('NaN')).ffill() + return pd.Series(index=bars.index, data=(left / right) + ).replace([np.inf, -np.inf], float('NaN')).ffill() # --------------------------------------------- @@ -460,7 +461,7 @@ def returns(series): try: res = (series / series.shift(1) - 1).replace([np.inf, -np.inf], float('NaN')) - except Exception as e: + except Exception as e: # noqa: F841 res = nans(len(series)) return pd.Series(index=series.index, data=res) @@ -471,8 +472,8 @@ def returns(series): def log_returns(series): try: res = np.log(series / series.shift(1) - ).replace([np.inf, -np.inf], float('NaN')) - except Exception as e: + ).replace([np.inf, -np.inf], float('NaN')) + except Exception as e: # noqa: F841 res = nans(len(series)) return pd.Series(index=series.index, data=res) @@ -483,9 +484,9 @@ def log_returns(series): def implied_volatility(series, window=252): try: logret = np.log(series / series.shift(1) - ).replace([np.inf, -np.inf], float('NaN')) + ).replace([np.inf, -np.inf], float('NaN')) res = numpy_rolling_std(logret, window) * np.sqrt(window) - except Exception as e: + except Exception as e: # noqa: F841 res = nans(len(series)) return pd.Series(index=series.index, data=res) @@ -542,7 +543,10 @@ def stoch(df, window=14, d=3, k=3, fast=False): my_df['rolling_max'] = df['high'].rolling(window).max() my_df['rolling_min'] = df['low'].rolling(window).min() - my_df['fast_k'] = 100 * (df['close'] - my_df['rolling_min'])/(my_df['rolling_max'] - my_df['rolling_min']) + my_df['fast_k'] = ( + 100 * (df['close'] - my_df['rolling_min']) / + (my_df['rolling_max'] - my_df['rolling_min']) + ) my_df['fast_d'] = my_df['fast_k'].rolling(d).mean() if fast: From f506644a8cabb542c176561a63b94ef879aa0164 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 4 May 2019 09:10:25 +0200 Subject: [PATCH 03/84] Improve docker documentation --- docs/installation.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 7060d7b39..11ddc010d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -237,14 +237,18 @@ docker run -d \ --name freqtrade \ -v /etc/localtime:/etc/localtime:ro \ -v ~/.freqtrade/config.json:/freqtrade/config.json \ + -v ~/.freqtrade/user_data/:/freqtrade/user_data \ -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ - freqtrade --db-url sqlite:///tradesv3.sqlite + freqtrade --db-url sqlite:///tradesv3.sqlite --strategy MyAwesomeStrategy ``` !!! Note db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` +!!! Note + All command line arguments can be added to the end of the `docker run` command. + ### 6. Monitor your Docker instance You can then use the following commands to monitor and manage your container: From 1e056ee415593b502dbb6277ecfc26e280956383 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 May 2019 14:07:08 +0200 Subject: [PATCH 04/84] Move trade jsonification to trade class --- freqtrade/persistence.py | 17 +++++++++++++++++ freqtrade/rpc/rpc.py | 19 ++++--------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 5a18a922a..d25ea9fed 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -213,6 +213,23 @@ class Trade(_DECL_BASE): return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' f'open_rate={self.open_rate:.8f}, open_since={open_since})') + def to_json(self) -> Dict[str, Any]: + + return { + 'trade_id': self.id, + 'pair': self.pair, + 'date': arrow.get(self.open_date), + 'open_rate': self.open_rate, + 'close_rate': self.close_rate, + 'amount': round(self.amount, 8), + 'stake_amount': round(self.stake_amount, 8), + 'stop_loss': self.stop_loss, + 'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None, + 'initial_stop_loss': self.initial_stop_loss, + 'initial_stop_loss_pct': (self.initial_stop_loss_pct * 100) + if self.initial_stop_loss_pct else None, + } + def adjust_min_max_rates(self, current_price: float): """ Adjust the max_rate and min_rate. diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index af384da45..37312318a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -100,28 +100,17 @@ class RPC(object): current_profit = trade.calc_profit_percent(current_rate) fmt_close_profit = (f'{round(trade.close_profit * 100, 2):.2f}%' if trade.close_profit else None) - results.append(dict( - trade_id=trade.id, - pair=trade.pair, + trade_dict = trade.to_json() + trade_dict.update(dict( base_currency=self._freqtrade.config['stake_currency'], - date=arrow.get(trade.open_date), - open_rate=trade.open_rate, - close_rate=trade.close_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - stake_amount=round(trade.stake_amount, 8), close_profit=fmt_close_profit, + current_rate=current_rate, current_profit=round(current_profit * 100, 2), - stop_loss=trade.stop_loss, - stop_loss_pct=(trade.stop_loss_pct * 100) - if trade.stop_loss_pct else None, - initial_stop_loss=trade.initial_stop_loss, - initial_stop_loss_pct=(trade.initial_stop_loss_pct * 100) - if trade.initial_stop_loss_pct else None, open_order='({} {} rem={:.8f})'.format( order['type'], order['side'], order['remaining'] ) if order else None, )) + results.append(trade_dict) return results def _rpc_status_table(self) -> DataFrame: From 2200a0223b679fe1f0b7b1bba0c7808cdf42e7b8 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 6 May 2019 00:30:21 +0300 Subject: [PATCH 05/84] fixed heikinashi --- freqtrade/vendor/qtpylib/indicators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index 8586968e8..cb1b794fd 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -113,7 +113,7 @@ def heikinashi(bars): bars['low'] + bars['close']) / 4 # ha open - bars.loc[:1, 'ha_open'] = (bars['open'] + bars['close']) / 2 + bars.loc[0:1, 'ha_open'] = (bars['open'].values[0] + bars['close'].values[0]) / 2 prev_open = bars[:1]['ha_open'].values[0] for idx, _ in bars[1:][['ha_open', 'ha_close']].iterrows(): loc = bars.index.get_loc(idx) From 31d271084fe78f29a2358d719a95fb8e4e119128 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 May 2019 06:55:12 +0200 Subject: [PATCH 06/84] Move json to persistence --- freqtrade/persistence.py | 12 ++++++++---- freqtrade/rpc/telegram.py | 5 +---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index d25ea9fed..6088dba72 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -214,11 +214,15 @@ class Trade(_DECL_BASE): f'open_rate={self.open_rate:.8f}, open_since={open_since})') def to_json(self) -> Dict[str, Any]: - return { 'trade_id': self.id, 'pair': self.pair, - 'date': arrow.get(self.open_date), + 'open_date_hum': arrow.get(self.open_date).humanize(), + 'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"), + 'close_date_hum': (arrow.get(self.close_date).humanize() + if self.close_date else None), + 'close_date': (self.close_date.strftime("%Y-%m-%d %H:%M:%S") + if self.close_date else None), 'open_rate': self.open_rate, 'close_rate': self.close_rate, 'amount': round(self.amount, 8), @@ -226,8 +230,8 @@ class Trade(_DECL_BASE): 'stop_loss': self.stop_loss, 'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None, 'initial_stop_loss': self.initial_stop_loss, - 'initial_stop_loss_pct': (self.initial_stop_loss_pct * 100) - if self.initial_stop_loss_pct else None, + 'initial_stop_loss_pct': (self.initial_stop_loss_pct * 100 + if self.initial_stop_loss_pct else None), } def adjust_min_max_rates(self, current_price: float): diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index dc0bad2b2..497c117ac 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -193,14 +193,11 @@ class Telegram(RPC): try: results = self._rpc_trade_status() - # pre format data - for result in results: - result['date'] = result['date'].humanize() messages = [] for r in results: lines = [ - "*Trade ID:* `{trade_id}` `(since {date})`", + "*Trade ID:* `{trade_id}` `(since {open_date_hum})`", "*Current Pair:* {pair}", "*Amount:* `{amount} ({stake_amount} {base_currency})`", "*Open Rate:* `{open_rate:.8f}`", From 2b78f73fe5e0ecb689c9d7d57b8f67bb6bc90b96 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 May 2019 06:56:07 +0200 Subject: [PATCH 07/84] Adapt tests to to_json method --- freqtrade/tests/rpc/test_rpc.py | 12 +++++++++--- freqtrade/tests/rpc/test_rpc_telegram.py | 5 ++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 93de6ef89..1f3f26dc9 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -47,12 +47,15 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: freqtradebot.create_trade() results = rpc._rpc_trade_status() - + print(results[0]) assert { 'trade_id': 1, 'pair': 'ETH/BTC', 'base_currency': 'BTC', - 'date': ANY, + 'open_date': ANY, + 'open_date_hum': ANY, + 'close_date': None, + 'close_date_hum': None, 'open_rate': 1.099e-05, 'close_rate': None, 'current_rate': 1.098e-05, @@ -78,7 +81,10 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: 'trade_id': 1, 'pair': 'ETH/BTC', 'base_currency': 'BTC', - 'date': ANY, + 'open_date': ANY, + 'open_date_hum': ANY, + 'close_date': None, + 'close_date_hum': None, 'open_rate': 1.099e-05, 'close_rate': None, 'current_rate': ANY, diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index f2f3f3945..86d728290 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -192,7 +192,10 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: 'trade_id': 1, 'pair': 'ETH/BTC', 'base_currency': 'BTC', - 'date': arrow.utcnow(), + 'open_date': arrow.utcnow(), + 'open_date_hum': arrow.utcnow().humanize, + 'close_date': None, + 'close_date_hum': None, 'open_rate': 1.099e-05, 'close_rate': None, 'current_rate': 1.098e-05, From 1a677c7441e0773826a12f8000384dd485062a32 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 May 2019 06:56:22 +0200 Subject: [PATCH 08/84] Add explicit test for to_json --- freqtrade/tests/rpc/test_rpc.py | 1 - freqtrade/tests/test_persistence.py | 69 ++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 1f3f26dc9..6ce543f3d 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -47,7 +47,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: freqtradebot.create_trade() results = rpc._rpc_trade_status() - print(results[0]) assert { 'trade_id': 1, 'pair': 'ETH/BTC', diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index f57a466e3..8c15fa8e8 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,7 +1,8 @@ # pragma pylint: disable=missing-docstring, C0103 -from unittest.mock import MagicMock import logging +from unittest.mock import MagicMock +import arrow import pytest from sqlalchemy import create_engine @@ -710,3 +711,69 @@ def test_get_open(default_conf, fee): Trade.session.add(trade) assert len(Trade.get_open_trades()) == 2 + + +def test_to_json(default_conf, fee): + init(default_conf) + + # Simulate dry_run entries + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_date=arrow.utcnow().shift(hours=-2).datetime, + open_rate=0.123, + exchange='bittrex', + open_order_id='dry_run_buy_12345' + ) + result = trade.to_json() + assert isinstance(result, dict) + print(result) + + assert result == {'trade_id': None, + 'pair': 'ETH/BTC', + 'open_date_hum': '2 hours ago', + 'open_date': trade.open_date.strftime("%Y-%m-%d %H:%M:%S"), + 'close_date_hum': None, + 'close_date': None, + 'open_rate': 0.123, + 'close_rate': None, + 'amount': 123.0, + 'stake_amount': 0.001, + 'stop_loss': None, + 'stop_loss_pct': None, + 'initial_stop_loss': None, + 'initial_stop_loss_pct': None} + + # Simulate dry_run entries + trade = Trade( + pair='XRP/BTC', + stake_amount=0.001, + amount=100.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_date=arrow.utcnow().shift(hours=-2).datetime, + close_date=arrow.utcnow().shift(hours=-1).datetime, + open_rate=0.123, + close_rate=0.125, + exchange='bittrex', + ) + result = trade.to_json() + assert isinstance(result, dict) + + assert result == {'trade_id': None, + 'pair': 'XRP/BTC', + 'open_date_hum': '2 hours ago', + 'open_date': trade.open_date.strftime("%Y-%m-%d %H:%M:%S"), + 'close_date_hum': 'an hour ago', + 'close_date': trade.close_date.strftime("%Y-%m-%d %H:%M:%S"), + 'open_rate': 0.123, + 'close_rate': 0.125, + 'amount': 100.0, + 'stake_amount': 0.001, + 'stop_loss': None, + 'stop_loss_pct': None, + 'initial_stop_loss': None, + 'initial_stop_loss_pct': None} From a8c4bed4e8b2ce99b255548f2d8f3804de15ec74 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 7 May 2019 12:42:07 +0000 Subject: [PATCH 09/84] Update ccxt from 1.18.508 to 1.18.509 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 703b77b0e..90949243c 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.508 +ccxt==1.18.509 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.1 From db0644eddfb58e2404eef3145170e68029b0e67e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 7 May 2019 12:42:10 +0000 Subject: [PATCH 10/84] Update plotly from 3.8.1 to 3.9.0 --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 4cdd6ec8f..23daee258 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==3.8.1 +plotly==3.9.0 From d642e03cd0d134bc0f9a542e9c01303b46852a53 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 7 May 2019 23:39:42 +0300 Subject: [PATCH 11/84] heikinashi performance problem resolved --- freqtrade/vendor/qtpylib/indicators.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index cb1b794fd..d16d7aa44 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -113,12 +113,11 @@ def heikinashi(bars): bars['low'] + bars['close']) / 4 # ha open - bars.loc[0:1, 'ha_open'] = (bars['open'].values[0] + bars['close'].values[0]) / 2 - prev_open = bars[:1]['ha_open'].values[0] - for idx, _ in bars[1:][['ha_open', 'ha_close']].iterrows(): - loc = bars.index.get_loc(idx) - prev_open = (prev_open + bars['ha_close'].values[loc - 1]) / 2 - bars.loc[loc:loc + 1, 'ha_open'] = prev_open + for i in range(0, len(bars)): + bars.at[i, 'ha_open'] = ( + (bars.at[0, 'open'] if i == 0 else bars.at[i - 1, 'ha_open']) + + (bars.at[0, 'close'] if i == 0 else bars.at[i - 1, 'ha_close']) + ) / 2 bars['ha_high'] = bars.loc[:, ['high', 'ha_open', 'ha_close']].max(axis=1) bars['ha_low'] = bars.loc[:, ['low', 'ha_open', 'ha_close']].min(axis=1) From 2554ebf2739a7d400c2b766469a92cc6d4d2acd4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 8 May 2019 00:00:44 +0300 Subject: [PATCH 12/84] fixed: heikinashi worked in backtesting, but failed in tests with testing arrays --- freqtrade/vendor/qtpylib/indicators.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index d16d7aa44..d62d062d7 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -113,11 +113,15 @@ def heikinashi(bars): bars['low'] + bars['close']) / 4 # ha open + idx = bars.index.name + bars.reset_index(inplace=True) for i in range(0, len(bars)): bars.at[i, 'ha_open'] = ( (bars.at[0, 'open'] if i == 0 else bars.at[i - 1, 'ha_open']) + (bars.at[0, 'close'] if i == 0 else bars.at[i - 1, 'ha_close']) ) / 2 + if idx: + bars.set_index(idx, inplace=True) bars['ha_high'] = bars.loc[:, ['high', 'ha_open', 'ha_close']].max(axis=1) bars['ha_low'] = bars.loc[:, ['low', 'ha_open', 'ha_close']].min(axis=1) From cf1ad3fd8c4be05d2f62617e1bd02057e8a3908c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 8 May 2019 12:42:06 +0000 Subject: [PATCH 13/84] Update ccxt from 1.18.509 to 1.18.512 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 90949243c..b79729731 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.509 +ccxt==1.18.512 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.1 From 1ccc25b4860e9b48f82c30430020a0a58872b6a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 May 2019 20:33:22 +0200 Subject: [PATCH 14/84] Fix test-data indexing --- freqtrade/tests/optimize/test_backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 281ce2bfe..02f8840e2 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -33,7 +33,7 @@ def get_args(args) -> List[str]: def trim_dictlist(dict_list, num): new = {} for pair, pair_data in dict_list.items(): - new[pair] = pair_data[num:] + new[pair] = pair_data[num:].reset_index() return new @@ -708,7 +708,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair): data = trim_dictlist(data, -500) # Remove data for one pair from the beginning of the data - data[pair] = data[pair][tres:] + data[pair] = data[pair][tres:].reset_index() # We need to enable sell-signal - otherwise it sells on ROI!! default_conf['experimental'] = {"use_sell_signal": True} default_conf['ticker_interval'] = '5m' From 45e586773639da57f93d5fa3b7767ace37568160 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 8 May 2019 23:41:45 +0300 Subject: [PATCH 15/84] heikinashi loop optimized; reset_index moved to tests --- freqtrade/vendor/qtpylib/indicators.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index d62d062d7..6edf626f0 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -113,15 +113,9 @@ def heikinashi(bars): bars['low'] + bars['close']) / 4 # ha open - idx = bars.index.name - bars.reset_index(inplace=True) - for i in range(0, len(bars)): - bars.at[i, 'ha_open'] = ( - (bars.at[0, 'open'] if i == 0 else bars.at[i - 1, 'ha_open']) + - (bars.at[0, 'close'] if i == 0 else bars.at[i - 1, 'ha_close']) - ) / 2 - if idx: - bars.set_index(idx, inplace=True) + bars.at[0, 'ha_open'] = (bars.at[0, 'open'] + bars.at[0, 'close']) / 2 + for i in range(1, len(bars)): + bars.at[i, 'ha_open'] = (bars.at[i - 1, 'ha_open'] + bars.at[i - 1, 'ha_close']) / 2 bars['ha_high'] = bars.loc[:, ['high', 'ha_open', 'ha_close']].max(axis=1) bars['ha_low'] = bars.loc[:, ['low', 'ha_open', 'ha_close']].min(axis=1) From 0410654c2cac12b4b2e8c13b432dfb1bf29cdde0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 May 2019 06:51:30 +0200 Subject: [PATCH 16/84] Add printing dataframe to documentation --- docs/bot-optimization.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 9e754c213..c83e4d0b1 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -345,6 +345,30 @@ if self.wallets: - `get_used(asset)` - currently tied up balance (open orders) - `get_total(asset)` - total available balance - sum of the 2 above +### Print created dataframe + +To inspect the created dataframe, you can issue a print-statement in either `populate_buy_trend()` or `populate_sell_trend()`. +You may also want to print the pair so it's clear what data is currently shown. + +``` python +def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe.loc[ + ( + #>> whatever condition<<< + ), + 'buy'] = 1 + + # Print the Analyzed pair + print(f"result for {metadata['pair']}") + + # Inspect the last 5 rows + print(dataframe.tail()) + + return dataframe +``` + +Printing more than a few rows is possible (don't use `.tail()`), however not recommended, as that will be very verbose (~500 lines per pair every 5 seconds). + ### Where is the default strategy? The default buy strategy is located in the file From 909df0d7bbe2094a53c0799c4f17d42b11a2debb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 May 2019 08:56:27 +0200 Subject: [PATCH 17/84] Improve doc wording --- docs/bot-optimization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index c83e4d0b1..5e080eab1 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -367,7 +367,7 @@ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: return dataframe ``` -Printing more than a few rows is possible (don't use `.tail()`), however not recommended, as that will be very verbose (~500 lines per pair every 5 seconds). +Printing more than a few rows is also possible (simply use `print(dataframe)` instead of `print(dataframe.tail())`), however not recommended, as that will be very verbose (~500 lines per pair every 5 seconds). ### Where is the default strategy? From f36ccdd9fa71c63743dcf513843d4bdfde960c51 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 9 May 2019 12:42:08 +0000 Subject: [PATCH 18/84] Update ccxt from 1.18.512 to 1.18.514 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index b79729731..07c65ffc7 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.512 +ccxt==1.18.514 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.1 From 00383b9438a7c597f39fc74de8c3f1e3cd55e132 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 9 May 2019 12:42:09 +0000 Subject: [PATCH 19/84] Update pytest from 4.4.1 to 4.4.2 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0c3dc9727..3c99a87f6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ flake8==3.7.7 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 -pytest==4.4.1 +pytest==4.4.2 pytest-mock==1.10.4 pytest-asyncio==0.10.0 pytest-cov==2.7.1 From 0f43e0bb7d786f13f00f6b246851ac44c674231c Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 10 May 2019 10:54:44 +0300 Subject: [PATCH 20/84] minor hyperopt output improvements --- freqtrade/optimize/hyperopt.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 235a20156..06c7ae495 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -33,6 +33,7 @@ from freqtrade.resolvers import HyperOptResolver logger = logging.getLogger(__name__) +INITIAL_POINTS = 30 MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization TICKERDATA_PICKLE = os.path.join('user_data', 'hyperopt_tickerdata.pkl') TRIALSDATA_PICKLE = os.path.join('user_data', 'hyperopt_results.pickle') @@ -120,14 +121,20 @@ class Hyperopt(Backtesting): """ Log results if it is better than any previous evaluation """ - if self.config.get('print_all', False) or results['loss'] < self.current_best_loss: - current = results['current_tries'] + print_all = self.config.get('print_all', False) + if print_all or results['loss'] < self.current_best_loss: + # Output human-friendly index here (starting from 1) + current = results['current_tries'] + 1 total = results['total_tries'] res = results['result'] loss = results['loss'] self.current_best_loss = results['loss'] - log_msg = f'\n{current:5d}/{total}: {res}. Loss {loss:.5f}' - print(log_msg) + log_msg = f'{current:5d}/{total}: {res} Objective: {loss:.5f}' + log_msg = f'*{log_msg}' if results['initial_point'] else f' {log_msg}' + if print_all: + print(log_msg) + else: + print('\n' + log_msg) else: print('.', end='') sys.stdout.flush() @@ -228,13 +235,13 @@ class Hyperopt(Backtesting): Return the format result in a string """ trades = len(results.index) - avg_profit = results.profit_percent.mean() * 100.0 + avg_profit = results.profit_percent.mean() total_profit = results.profit_abs.sum() stake_cur = self.config['stake_currency'] profit = results.profit_percent.sum() duration = results.trade_duration.mean() - return (f'{trades:6d} trades. Avg profit {avg_profit: 5.2f}%. ' + return (f'{trades:6d} trades. Avg profit {avg_profit: 9.6f}%. ' f'Total profit {total_profit: 11.8f} {stake_cur} ' f'({profit:.4f}Σ%). Avg duration {duration:5.1f} mins.') @@ -243,7 +250,7 @@ class Hyperopt(Backtesting): self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", - n_initial_points=30, + n_initial_points=INITIAL_POINTS, acq_optimizer_kwargs={'n_jobs': cpu_count}, random_state=self.config.get('hyperopt_random_state', None) ) @@ -301,9 +308,11 @@ class Hyperopt(Backtesting): self.trials += f_val for j in range(jobs): + current = i * jobs + j self.log_results({ 'loss': f_val[j]['loss'], - 'current_tries': i * jobs + j, + 'current_tries': current, + 'initial_point': current < INITIAL_POINTS, 'total_tries': self.total_tries, 'result': f_val[j]['result'], }) From 349d5563394d0fada998c936f133c88a4e643d94 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 10 May 2019 12:42:13 +0000 Subject: [PATCH 21/84] Update ccxt from 1.18.514 to 1.18.516 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 07c65ffc7..da8b76550 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.514 +ccxt==1.18.516 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.1 From ab23db2fa1bf594a5ceb7047fa98ac8b78663c3a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 10 May 2019 12:42:14 +0000 Subject: [PATCH 22/84] Update scikit-learn from 0.20.3 to 0.21.0 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index da8b76550..61c753784 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -8,7 +8,7 @@ cachetools==3.1.0 requests==2.21.0 urllib3==1.24.2 # pyup: ignore wrapt==1.11.1 -scikit-learn==0.20.3 +scikit-learn==0.21.0 joblib==0.13.2 jsonschema==3.0.1 TA-Lib==0.4.17 From 75306b7a6ea559efbfcf0b79f2d4fe671f4c5c3f Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 11 May 2019 10:17:46 +0300 Subject: [PATCH 23/84] tests adjusted --- freqtrade/tests/optimize/test_hyperopt.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 8bd5ffd0b..325713ba7 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -249,11 +249,12 @@ def test_log_results_if_loss_improves(hyperopt, capsys) -> None: 'loss': 1, 'current_tries': 1, 'total_tries': 2, - 'result': 'foo' + 'result': 'foo.', + 'initial_point': False } ) out, err = capsys.readouterr() - assert ' 1/2: foo. Loss 1.00000' in out + assert ' 2/2: foo. Objective: 1.00000' in out def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: @@ -458,7 +459,7 @@ def test_generate_optimizer(mocker, default_conf) -> None: } response_expected = { 'loss': 1.9840569076926293, - 'result': ' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' + 'result': ' 1 trades. Avg profit 0.023117%. Total profit 0.00023300 BTC ' '(0.0231Σ%). Avg duration 100.0 mins.', 'params': optimizer_param } From 52da64b6dcc9b07f9cdf0e7b9828c1631c6a110f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 May 2019 14:33:26 +0200 Subject: [PATCH 24/84] Align configuration files --- config_binance.json.example | 2 +- config_full.json.example | 2 +- config_kraken.json.example | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/config_binance.json.example b/config_binance.json.example index ab57db88f..1d492fc3c 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -11,8 +11,8 @@ "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0, "use_order_book": false, + "ask_last_balance": 0.0, "order_book_top": 1, "check_depth_of_market": { "enabled": false, diff --git a/config_full.json.example b/config_full.json.example index 806b5f718..4c4ad3c58 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -22,8 +22,8 @@ "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0, "use_order_book": false, + "ask_last_balance": 0.0, "order_book_top": 1, "check_depth_of_market": { "enabled": false, diff --git a/config_kraken.json.example b/config_kraken.json.example index 7a47b701f..ea3677b2d 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -5,15 +5,14 @@ "fiat_display_currency": "EUR", "ticker_interval" : "5m", "dry_run": true, - "db_url": "sqlite:///tradesv3.dryrun.sqlite", "trailing_stop": false, "unfilledtimeout": { "buy": 10, "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0, "use_order_book": false, + "ask_last_balance": 0.0, "order_book_top": 1, "check_depth_of_market": { "enabled": false, @@ -60,8 +59,8 @@ }, "telegram": { "enabled": false, - "token": "", - "chat_id": "" + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id" }, "initial_state": "running", "forcebuy_enable": false, From 131b232155c4c049adafc450e2b14ca7c6d57396 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 May 2019 14:33:35 +0200 Subject: [PATCH 25/84] Add sample for order_types in config (slightly different syntax) --- docs/configuration.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index dc623a728..df116b3c2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -191,14 +191,28 @@ If this is configured, all 4 values (`buy`, `sell`, `stoploss` and `stoploss_on_exchange`) need to be present, otherwise the bot will warn about it and fail to start. The below is the default which is used if this is not configured in either strategy or configuration file. +Syntax for Strategy: + ```python -"order_types": { +order_types = { "buy": "limit", "sell": "limit", "stoploss": "market", "stoploss_on_exchange": False, "stoploss_on_exchange_interval": 60 -}, +} +``` + +Configuration: + +```json +"order_types": { + "buy": "limit", + "sell": "limit", + "stoploss": "market", + "stoploss_on_exchange": false, + "stoploss_on_exchange_interval": 60 +} ``` !!! Note From 22f902f0f77d4ee9e92d2e368b55c3057809708f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 11 May 2019 12:41:06 +0000 Subject: [PATCH 26/84] Update ccxt from 1.18.516 to 1.18.519 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 61c753784..41e79eeda 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.516 +ccxt==1.18.519 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.1 From 652914a67b2237c7fe2c04857d17fd5b16220d61 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 11 May 2019 12:41:07 +0000 Subject: [PATCH 27/84] Update python-rapidjson from 0.7.0 to 0.7.1 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 41e79eeda..6e5c1ea8a 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -23,7 +23,7 @@ filelock==3.0.10 py_find_1st==1.1.3 #Load ticker files 30% faster -python-rapidjson==0.7.0 +python-rapidjson==0.7.1 # Notify systemd sdnotify==0.3.2 From 46b1ecc77d53054aed6835d69f9989316422aa78 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 May 2019 15:27:09 +0200 Subject: [PATCH 28/84] Fix #1840 - Support balances other than USDT --- freqtrade/rpc/rpc.py | 5 +++-- freqtrade/tests/rpc/test_rpc_telegram.py | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 37312318a..2189a0d17 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -276,11 +276,12 @@ class RPC(object): rate = 1.0 else: try: - if coin == 'USDT': - rate = 1.0 / self._freqtrade.get_sell_rate('BTC/USDT', False) + if coin in('USDT', 'USD', 'EUR'): + rate = 1.0 / self._freqtrade.get_sell_rate('BTC/' + coin, False) else: rate = self._freqtrade.get_sell_rate(coin + '/BTC', False) except (TemporaryError, DependencyException): + logger.warning(f" Could not get rate for pair {coin}.") continue est_btc: float = rate * balance['total'] total = total + est_btc diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 86d728290..69e3006cd 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -522,6 +522,11 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: 'total': 1.0, 'free': 1.0, 'used': 0.0 + }, + 'EUR': { + 'total': 10.0, + 'free': 10.0, + 'used': 0.0 } } @@ -565,6 +570,7 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: assert '*BTC:*' in result assert '*ETH:*' not in result assert '*USDT:*' in result + assert '*EUR:*' in result assert 'Balance:' in result assert 'Est. BTC:' in result assert 'BTC: 12.00000000' in result From dccd6b4a91b2cc0ee34f9cf3386872b1e8121013 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 12 May 2019 12:41:05 +0000 Subject: [PATCH 29/84] Update ccxt from 1.18.519 to 1.18.522 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 6e5c1ea8a..41f1b6218 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.519 +ccxt==1.18.522 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.1 From 11dca0bd29c04aac339698e53a9a68d53bd220ea Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 12 May 2019 12:41:06 +0000 Subject: [PATCH 30/84] Update pytest from 4.4.2 to 4.5.0 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3c99a87f6..fa52a4869 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ flake8==3.7.7 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 -pytest==4.4.2 +pytest==4.5.0 pytest-mock==1.10.4 pytest-asyncio==0.10.0 pytest-cov==2.7.1 From 00b4501c598e23592ddc3f86e73e1ef487a6e469 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 12 May 2019 21:14:00 +0300 Subject: [PATCH 31/84] avg profit and total profit corrected (to be %, not ratio); comments cleaned up a bit; typo in the log msg fixed --- freqtrade/optimize/hyperopt.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 06c7ae495..252b76252 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -63,9 +63,11 @@ class Hyperopt(Backtesting): # if eval ends with higher value, we consider it a failed eval self.max_accepted_trade_duration = 300 - # this is expexted avg profit * expected trade count - # for example 3.5%, 1100 trades, self.expected_max_profit = 3.85 - # check that the reported Σ% values do not exceed this! + # This is assumed to be expected avg profit * expected trade count. + # For example, for 0.35% avg per trade (or 0.0035 as ratio) and 1100 trades, + # self.expected_max_profit = 3.85 + # Check that the reported Σ% values do not exceed this! + # Note, this is ratio. 3.85 stated above means 385Σ%. self.expected_max_profit = 3.0 # Previous evaluations @@ -211,8 +213,8 @@ class Hyperopt(Backtesting): trade_count = len(results.index) trade_duration = results.trade_duration.mean() - # If this evaluation contains too short small amount of trades - # to be interesting -- consider it as 'bad' (assign max. loss value) + # If this evaluation contains too short amount of trades to be + # interesting -- consider it as 'bad' (assigned max. loss value) # in order to cast this hyperspace point away from optimization # path. We do not want to optimize 'hodl' strategies. if trade_count < self.config['hyperopt_min_trades']: @@ -235,15 +237,15 @@ class Hyperopt(Backtesting): Return the format result in a string """ trades = len(results.index) - avg_profit = results.profit_percent.mean() + avg_profit = results.profit_percent.mean() * 100.0 total_profit = results.profit_abs.sum() stake_cur = self.config['stake_currency'] - profit = results.profit_percent.sum() + profit = results.profit_percent.sum() * 100.0 duration = results.trade_duration.mean() - return (f'{trades:6d} trades. Avg profit {avg_profit: 9.6f}%. ' + return (f'{trades:6d} trades. Avg profit {avg_profit: 5.2f}%. ' f'Total profit {total_profit: 11.8f} {stake_cur} ' - f'({profit:.4f}Σ%). Avg duration {duration:5.1f} mins.') + f'({profit: 7.2f}Σ%). Avg duration {duration:5.1f} mins.') def get_optimizer(self, cpu_count) -> Optimizer: return Optimizer( @@ -318,7 +320,7 @@ class Hyperopt(Backtesting): }) logger.debug(f"Optimizer params: {f_val[j]['params']}") for j in range(jobs): - logger.debug(f"Opimizer state: Xi: {opt.Xi[-j-1]}, yi: {opt.yi[-j-1]}") + logger.debug(f"Optimizer state: Xi: {opt.Xi[-j-1]}, yi: {opt.yi[-j-1]}") except KeyboardInterrupt: print('User interrupted..') From 003461ec96648b061a4d92127cb6634833921413 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 12 May 2019 21:19:20 +0300 Subject: [PATCH 32/84] tests adjusted --- freqtrade/tests/optimize/test_hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 325713ba7..86c5f2a34 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -459,8 +459,8 @@ def test_generate_optimizer(mocker, default_conf) -> None: } response_expected = { 'loss': 1.9840569076926293, - 'result': ' 1 trades. Avg profit 0.023117%. Total profit 0.00023300 BTC ' - '(0.0231Σ%). Avg duration 100.0 mins.', + 'result': ' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' + '( 2.31Σ%). Avg duration 100.0 mins.', 'params': optimizer_param } From 600f660f5e0b4a05615c511f5dfe687853fa050a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 13 May 2019 12:41:06 +0000 Subject: [PATCH 33/84] Update ccxt from 1.18.522 to 1.18.523 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 41f1b6218..4f7309a6a 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.522 +ccxt==1.18.523 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.1 From 1cd98665def8e19cece3f8f9c89d82265ee528a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 May 2019 19:50:56 +0200 Subject: [PATCH 34/84] Update pyup only weekly --- .pyup.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pyup.yml b/.pyup.yml index 3494a3fd3..b1b721113 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -11,7 +11,10 @@ update: all # allowed: True, False pin: True -schedule: "every day" +# update schedule +# default: empty +# allowed: "every day", "every week", .. +schedule: "every week" search: False From 5677c4882edd9faa39efa5df5d832bed5a6b76c2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 13 May 2019 23:56:59 +0300 Subject: [PATCH 35/84] minor: add ticker data validation; log backtesting interval --- freqtrade/optimize/hyperopt.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 252b76252..5b8797d99 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -24,7 +24,8 @@ from freqtrade import DependencyException from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.data.history import load_data -from freqtrade.optimize import get_timeframe +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.optimize import get_timeframe, validate_backtest_data from freqtrade.optimize.backtesting import Backtesting from freqtrade.state import RunMode from freqtrade.resolvers import HyperOptResolver @@ -282,9 +283,25 @@ class Hyperopt(Backtesting): timerange=timerange ) + if not data: + logger.critical("No data found. Terminating.") + return + + min_date, max_date = get_timeframe(data) + # Validate dataframe for missing values (mainly at start and end, as fillup is called) + validate_backtest_data(data, min_date, max_date, + timeframe_to_minutes(self.ticker_interval)) + logger.info( + 'Backtesting data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + if self.has_space('buy') or self.has_space('sell'): self.strategy.advise_indicators = \ self.custom_hyperopt.populate_indicators # type: ignore + dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) # We don't need exchange instance anymore while running hyperopt From 90a52e46021fb100e5e298d8b495ee407fed77f4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 14 May 2019 09:23:09 +0300 Subject: [PATCH 36/84] tests adjusted; new test_start_no_data() added for hyperopt --- freqtrade/tests/optimize/test_hyperopt.py | 32 +++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 86c5f2a34..f50f58e5b 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -195,6 +195,33 @@ def test_start(mocker, default_conf, caplog) -> None: assert start_mock.call_count == 1 +def test_start_no_data(mocker, default_conf, caplog) -> None: + mocker.patch( + 'freqtrade.configuration.Configuration._load_config_file', + lambda *args, **kwargs: default_conf + ) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock(return_value={})) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + patch_exchange(mocker) + + args = [ + '--config', 'config.json', + 'hyperopt', + '--epochs', '5' + ] + args = get_args(args) + start(args) + + import pprint + pprint.pprint(caplog.record_tuples) + + assert log_has('No data found. Terminating.', caplog.record_tuples) + + def test_start_failure(mocker, default_conf, caplog) -> None: start_mock = MagicMock() mocker.patch( @@ -310,6 +337,11 @@ def test_roi_table_generation(hyperopt) -> None: def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', MagicMock(return_value=[{'loss': 1, 'result': 'foo result', 'params': {}}]) From 8b95e12468474b9c25258d921491e08eab307b27 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 15 May 2019 12:05:35 +0300 Subject: [PATCH 37/84] log message adjusted in backtesting and hyperopt --- freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index eefbd4e04..51122cfb2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -445,7 +445,7 @@ class Backtesting(object): optimize.validate_backtest_data(data, min_date, max_date, timeframe_to_minutes(self.ticker_interval)) logger.info( - 'Measuring data from %s up to %s (%s days)..', + 'Backtesting with data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5b8797d99..92589aed2 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -292,7 +292,7 @@ class Hyperopt(Backtesting): validate_backtest_data(data, min_date, max_date, timeframe_to_minutes(self.ticker_interval)) logger.info( - 'Backtesting data from %s up to %s (%s days)..', + 'Hyperopting with data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 02f8840e2..6a39deed4 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -495,7 +495,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: 'Using local backtesting data (using whitelist in given config) ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Measuring data from 2017-11-14T21:17:00+00:00 ' + 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' 'up to 2017-11-14T22:59:00+00:00 (0 days)..' ] for line in exists: @@ -858,7 +858,8 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', - 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Backtesting with data from 2017-11-14T19:31:00+00:00 ' + 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...' ] @@ -916,7 +917,8 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', - 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Backtesting with data from 2017-11-14T19:31:00+00:00 ' + 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy TestStrategy', From 2741c5c3307ffb1a89e0815ba5047d9e13403d82 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 16 May 2019 22:38:59 +0300 Subject: [PATCH 38/84] inherit freqtrade exceptions from Exception i.o. BaseException --- freqtrade/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index f84f5240e..340d5b515 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -2,14 +2,14 @@ __version__ = '0.18.5-dev' -class DependencyException(BaseException): +class DependencyException(Exception): """ - Indicates that a assumed dependency is not met. + Indicates that an assumed dependency is not met. This could happen when there is currently not enough money on the account. """ -class OperationalException(BaseException): +class OperationalException(Exception): """ Requires manual intervention. This happens when an exchange returns an unexpected error during runtime @@ -17,7 +17,7 @@ class OperationalException(BaseException): """ -class InvalidOrderException(BaseException): +class InvalidOrderException(Exception): """ This is returned when the order is not valid. Example: If stoploss on exchange order is hit, then trying to cancel the order @@ -25,7 +25,7 @@ class InvalidOrderException(BaseException): """ -class TemporaryError(BaseException): +class TemporaryError(Exception): """ Temporary network or exchange related error. This could happen when an exchange is congested, unavailable, or the user From e2b83624a397b85807d23f7b5353ade9727e54a3 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 17 May 2019 19:05:36 +0300 Subject: [PATCH 39/84] data/history cleanup --- freqtrade/data/history.py | 38 ++++++++++++++++------------ freqtrade/tests/data/test_history.py | 29 ++++++++++++++++----- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 4dba1b760..86d3c3071 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -90,13 +90,8 @@ def load_pair_history(pair: str, :return: DataFrame with ohlcv data """ - # If the user force the refresh of pairs + # The user forced the refresh of pairs if refresh_pairs: - if not exchange: - raise OperationalException("Exchange needs to be initialized when " - "calling load_data with refresh_pairs=True") - - logger.info('Download data for pair and store them in %s', datadir) download_pair_history(datadir=datadir, exchange=exchange, pair=pair, @@ -115,10 +110,11 @@ def load_pair_history(pair: str, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing) else: - logger.warning('No data for pair: "%s", Interval: %s. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' - 'script to download the data', - pair, ticker_interval) + logger.warning( + f'No history data for pair: "{pair}", interval: {ticker_interval}. ' + 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'script to download the data' + ) return None @@ -190,7 +186,7 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str, def download_pair_history(datadir: Optional[Path], - exchange: Exchange, + exchange: Optional[Exchange], pair: str, ticker_interval: str = '5m', timerange: Optional[TimeRange] = None) -> bool: @@ -201,18 +197,26 @@ def download_pair_history(datadir: Optional[Path], the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data + :param pair: pair to download :param ticker_interval: ticker interval :param timerange: range of time to download :return: bool with success state - """ + if not exchange: + raise OperationalException( + "Exchange needs to be initialized when downloading pair history data" + ) + try: path = make_testdata_path(datadir) filepair = pair.replace("/", "_") filename = path.joinpath(f'{filepair}-{ticker_interval}.json') - logger.info('Download the pair: "%s", Interval: %s', pair, ticker_interval) + logger.info( + f'Download history data for pair: "{pair}", interval: {ticker_interval} ' + f'and store in {datadir}.' + ) data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange) @@ -231,7 +235,9 @@ def download_pair_history(datadir: Optional[Path], misc.file_dump_json(filename, data) return True - except BaseException: - logger.info('Failed to download the pair: "%s", Interval: %s', - pair, ticker_interval) + + except Exception: + logger.error( + f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}.' + ) return False diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 14ec99042..15442f577 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -59,7 +59,11 @@ def _clean_test_file(file: str) -> None: def test_load_data_30min_ticker(mocker, caplog, default_conf) -> None: ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='30m', datadir=None) assert isinstance(ld, DataFrame) - assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 30m', caplog.record_tuples) + assert not log_has( + 'Download history data for pair: "UNITTEST/BTC", interval: 30m ' + 'and store in None.', + caplog.record_tuples + ) def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: @@ -67,7 +71,7 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: assert not isinstance(ld, DataFrame) assert ld is None assert log_has( - 'No data for pair: "UNITTEST/BTC", Interval: 7m. ' + 'No history data for pair: "UNITTEST/BTC", interval: 7m. ' 'Use --refresh-pairs-cached option or download_backtest_data.py ' 'script to download the data', caplog.record_tuples @@ -80,7 +84,11 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: _backup_file(file, copy_file=True) history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC']) assert os.path.isfile(file) is True - assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 1m', caplog.record_tuples) + assert not log_has( + 'Download history data for pair: "UNITTEST/BTC", interval: 1m ' + 'and store in None.', + caplog.record_tuples + ) _clean_test_file(file) @@ -100,7 +108,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau pair='MEME/BTC') assert os.path.isfile(file) is False assert log_has( - 'No data for pair: "MEME/BTC", Interval: 1m. ' + 'No history data for pair: "MEME/BTC", interval: 1m. ' 'Use --refresh-pairs-cached option or download_backtest_data.py ' 'script to download the data', caplog.record_tuples @@ -113,7 +121,11 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau exchange=exchange, pair='MEME/BTC') assert os.path.isfile(file) is True - assert log_has('Download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) + assert log_has( + 'Download history data for pair: "MEME/BTC", interval: 1m ' + 'and store in None.', + caplog.record_tuples + ) with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): history.load_pair_history(datadir=None, ticker_interval='1m', @@ -293,7 +305,7 @@ def test_download_pair_history2(mocker, default_conf) -> None: def test_download_backtesting_data_exception(ticker_history, mocker, caplog, default_conf) -> None: mocker.patch('freqtrade.exchange.Exchange.get_history', - side_effect=BaseException('File Error')) + side_effect=Exception('File Error')) exchange = get_patched_exchange(mocker, default_conf) @@ -308,7 +320,10 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) - assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) + assert log_has( + 'Failed to download history data for pair: "MEME/BTC", interval: 1m.', + caplog.record_tuples + ) def test_load_tickerdata_file() -> None: From 8d8b4a69b759c0774e913abc26c82105c18d6f3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 May 2019 09:03:56 +0200 Subject: [PATCH 40/84] Clearly warn about using future data during strategy development --- docs/bot-optimization.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 5e080eab1..97166a736 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -53,6 +53,12 @@ file as reference.** It is therefore best to use vectorized operations (across the whole dataframe, not loops) and avoid index referencing (`df.iloc[-1]`), but instead use `df.shift()` to get to the previous candle. +!!! Warning Using future data + Since backtesting passes the full time interval to the `populate_*()` methods, the strategy author + needs to take care to avoid having the strategy utilize data from the future. + Samples for usage of future data are `dataframe.shift(-1)`, `dataframe.resample("1h")` (this uses the left border of the interval, so moves data from an hour to the start of the hour). + They all use data which is not available during regular operations, so these strategies will perform well during backtesting, but will fail / perform badly dry-run tests. + ### Customize Indicators Buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `populate_indicators()` from your strategy file. From f93e6ad0f6461fee649cfaeeed2f2400ada8d0e1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 May 2019 09:07:43 +0200 Subject: [PATCH 41/84] Rename strategy customization file --- docs/bot-usage.md | 4 ++-- docs/{bot-optimization.md => strategy-customization.md} | 0 mkdocs.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename docs/{bot-optimization.md => strategy-customization.md} (100%) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 2b2fef640..cb98e1ea5 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -103,7 +103,7 @@ If the bot does not find your strategy file, it will display in an error message the reason (File not found, or errors in your code). Learn more about strategy file in -[optimize your bot](bot-optimization.md). +[Strategy Customization](strategy-customization.md). ### How to use **--strategy-path**? @@ -296,4 +296,4 @@ in [misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc. ## Next step The optimal strategy of the bot will change with time depending of the market trends. The next step is to -[optimize your bot](bot-optimization.md). +[Strategy Customization](strategy-customization.md). diff --git a/docs/bot-optimization.md b/docs/strategy-customization.md similarity index 100% rename from docs/bot-optimization.md rename to docs/strategy-customization.md diff --git a/mkdocs.yml b/mkdocs.yml index ecac265c1..9932ff316 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,7 +3,7 @@ nav: - About: index.md - Installation: installation.md - Configuration: configuration.md - - Custom Strategy: bot-optimization.md + - Strategy Customization: strategy-customization.md - Stoploss: stoploss.md - Start the bot: bot-usage.md - Control the bot: From fc96da869a4d0b5051eefd8e437b088dc35cf941 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 May 2019 16:07:16 +0200 Subject: [PATCH 42/84] Fix grammar messup --- docs/strategy-customization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 97166a736..51540f690 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -57,7 +57,7 @@ file as reference.** Since backtesting passes the full time interval to the `populate_*()` methods, the strategy author needs to take care to avoid having the strategy utilize data from the future. Samples for usage of future data are `dataframe.shift(-1)`, `dataframe.resample("1h")` (this uses the left border of the interval, so moves data from an hour to the start of the hour). - They all use data which is not available during regular operations, so these strategies will perform well during backtesting, but will fail / perform badly dry-run tests. + They all use data which is not available during regular operations, so these strategies will perform well during backtesting, but will fail / perform badly in dry-runs. ### Customize Indicators From e7b9bc6808be9dc54a94e43831a60a55dcc04013 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 20 May 2019 12:27:30 +0300 Subject: [PATCH 43/84] minor: remove noisy useless debug message --- freqtrade/persistence.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 6088dba72..e64e0b89c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -238,7 +238,6 @@ class Trade(_DECL_BASE): """ Adjust the max_rate and min_rate. """ - logger.debug("Adjusting min/max rates") self.max_rate = max(current_price, self.max_rate or self.open_rate) self.min_rate = min(current_price, self.min_rate or self.open_rate) From 703fdb2bc637efa73e1515afcf8628969cb7a3aa Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:07 +0000 Subject: [PATCH 44/84] Update scipy from 1.2.1 to 1.3.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 78585f8f5..da87f56d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ numpy==1.16.3 pandas==0.24.2 -scipy==1.2.1 +scipy==1.3.0 From de95e50804addaafb785f1392a0a10075f523a0b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:08 +0000 Subject: [PATCH 45/84] Update ccxt from 1.18.523 to 1.18.551 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 4f7309a6a..8139d2cbb 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.523 +ccxt==1.18.551 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.1 From 3404bb1865b31823bd978ec3084f66c6955f63e2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:09 +0000 Subject: [PATCH 46/84] Update arrow from 0.13.1 to 0.13.2 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 8139d2cbb..4ec3a0092 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -3,7 +3,7 @@ ccxt==1.18.551 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 -arrow==0.13.1 +arrow==0.13.2 cachetools==3.1.0 requests==2.21.0 urllib3==1.24.2 # pyup: ignore From 34c7ac8926079673cbf92f7a204f08daff38231b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:10 +0000 Subject: [PATCH 47/84] Update requests from 2.21.0 to 2.22.0 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 4ec3a0092..deea30faf 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -5,7 +5,7 @@ SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.2 cachetools==3.1.0 -requests==2.21.0 +requests==2.22.0 urllib3==1.24.2 # pyup: ignore wrapt==1.11.1 scikit-learn==0.21.0 From 5b24ac78981946bf334d19e28c14a180add4431c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:11 +0000 Subject: [PATCH 48/84] Update scikit-learn from 0.21.0 to 0.21.1 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index deea30faf..4ba4426b7 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -8,7 +8,7 @@ cachetools==3.1.0 requests==2.22.0 urllib3==1.24.2 # pyup: ignore wrapt==1.11.1 -scikit-learn==0.21.0 +scikit-learn==0.21.1 joblib==0.13.2 jsonschema==3.0.1 TA-Lib==0.4.17 From 04e13eed7dbc006bd17ea50dab6da94317a39750 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:13 +0000 Subject: [PATCH 49/84] Update filelock from 3.0.10 to 3.0.12 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 4ba4426b7..3f755b8c0 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -17,7 +17,7 @@ coinmarketcap==5.0.3 # Required for hyperopt scikit-optimize==0.5.2 -filelock==3.0.10 +filelock==3.0.12 # find first, C search in arrays py_find_1st==1.1.3 From 96a34f753b0be9a86e09eb9fd00aeace08302bfd Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 May 2019 19:48:12 +0200 Subject: [PATCH 50/84] Adapt test to new output from arrow --- freqtrade/tests/rpc/test_rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 6ce543f3d..c3fcd62fb 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -119,7 +119,7 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: freqtradebot.create_trade() result = rpc._rpc_status_table() - assert 'just now' in result['Since'].all() + assert 'instantly' in result['Since'].all() assert 'ETH/BTC' in result['Pair'].all() assert '-0.59%' in result['Profit'].all() @@ -128,7 +128,7 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: # invalidate ticker cache rpc._freqtrade.exchange._cached_ticker = {} result = rpc._rpc_status_table() - assert 'just now' in result['Since'].all() + assert 'instantly' in result['Since'].all() assert 'ETH/BTC' in result['Pair'].all() assert 'nan%' in result['Profit'].all() From 11dce9128193184623d9a4896d503c351ef37faf Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 21 May 2019 20:49:02 +0300 Subject: [PATCH 51/84] data/history minor cleanup --- freqtrade/data/history.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 86d3c3071..27e68b533 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -63,12 +63,8 @@ def load_tickerdata_file( Load a pair from file, either .json.gz or .json :return tickerlist or None if unsuccesful """ - path = make_testdata_path(datadir) - pair_s = pair.replace('/', '_') - file = path.joinpath(f'{pair_s}-{ticker_interval}.json') - - pairdata = misc.file_load_json(file) - + filename = pair_data_filename(datadir, pair, ticker_interval) + pairdata = misc.file_load_json(filename) if not pairdata: return None @@ -142,11 +138,18 @@ def load_data(datadir: Optional[Path], return result -def make_testdata_path(datadir: Optional[Path]) -> Path: +def make_datadir_path(datadir: Optional[Path]) -> Path: """Return the path where testdata files are stored""" return datadir or (Path(__file__).parent.parent / "tests" / "testdata").resolve() +def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) -> Path: + path = make_datadir_path(datadir) + pair_s = pair.replace("/", "_") + filename = path.joinpath(f'{pair_s}-{ticker_interval}.json') + return filename + + def load_cached_data_for_updating(filename: Path, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: @@ -209,9 +212,7 @@ def download_pair_history(datadir: Optional[Path], ) try: - path = make_testdata_path(datadir) - filepair = pair.replace("/", "_") - filename = path.joinpath(f'{filepair}-{ticker_interval}.json') + filename = pair_data_filename(datadir, pair, ticker_interval) logger.info( f'Download history data for pair: "{pair}", interval: {ticker_interval} ' @@ -236,8 +237,9 @@ def download_pair_history(datadir: Optional[Path], misc.file_dump_json(filename, data) return True - except Exception: + except Exception as e: logger.error( - f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}.' + f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}. ' + f'Error: {e}' ) return False From 7cb753754b039898c3cfae2dd8049068cda6b4a2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 21 May 2019 20:49:19 +0300 Subject: [PATCH 52/84] tests adjusted --- freqtrade/tests/data/test_btanalysis.py | 4 ++-- freqtrade/tests/data/test_history.py | 7 ++++--- freqtrade/tests/test_misc.py | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index dd7cbe0d9..4c0426b93 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -2,12 +2,12 @@ import pytest from pandas import DataFrame from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data -from freqtrade.data.history import make_testdata_path +from freqtrade.data.history import make_datadir_path def test_load_backtest_data(): - filename = make_testdata_path(None) / "backtest-result_test.json" + filename = make_datadir_path(None) / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"] diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 15442f577..a37b42351 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -16,7 +16,7 @@ from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, load_tickerdata_file, - make_testdata_path, + make_datadir_path, trim_tickerlist) from freqtrade.misc import file_dump_json from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -136,7 +136,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau def test_testdata_path() -> None: - assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) + assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_datadir_path(None)) def test_load_cached_data_for_updating(mocker) -> None: @@ -321,7 +321,8 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def _clean_test_file(file1_1) _clean_test_file(file1_5) assert log_has( - 'Failed to download history data for pair: "MEME/BTC", interval: 1m.', + 'Failed to download history data for pair: "MEME/BTC", interval: 1m. ' + 'Error: File Error', caplog.record_tuples ) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 2da6b8718..c7bcf7edf 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, file_load_json, format_ms_time, shorten_date) -from freqtrade.data.history import load_tickerdata_file, make_testdata_path +from freqtrade.data.history import load_tickerdata_file, pair_data_filename from freqtrade.strategy.default_strategy import DefaultStrategy @@ -60,13 +60,13 @@ def test_file_dump_json(mocker) -> None: def test_file_load_json(mocker) -> None: # 7m .json does not exist - ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-7m.json')) + ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '7m')) assert not ret # 1m json exists (but no .gz exists) - ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-1m.json')) + ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '1m')) assert ret # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json - ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-8m.json')) + ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '8m')) assert ret From 98eeec31451c844bbdabc0a8c8dacd2474122596 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 22 May 2019 14:04:58 +0300 Subject: [PATCH 53/84] renaming of make_testdata_path reverted --- freqtrade/data/history.py | 4 ++-- freqtrade/tests/data/test_btanalysis.py | 4 ++-- freqtrade/tests/data/test_history.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 27e68b533..3bec63926 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -138,13 +138,13 @@ def load_data(datadir: Optional[Path], return result -def make_datadir_path(datadir: Optional[Path]) -> Path: +def make_testdata_path(datadir: Optional[Path]) -> Path: """Return the path where testdata files are stored""" return datadir or (Path(__file__).parent.parent / "tests" / "testdata").resolve() def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) -> Path: - path = make_datadir_path(datadir) + path = make_testdata_path(datadir) pair_s = pair.replace("/", "_") filename = path.joinpath(f'{pair_s}-{ticker_interval}.json') return filename diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index 4c0426b93..dd7cbe0d9 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -2,12 +2,12 @@ import pytest from pandas import DataFrame from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data -from freqtrade.data.history import make_datadir_path +from freqtrade.data.history import make_testdata_path def test_load_backtest_data(): - filename = make_datadir_path(None) / "backtest-result_test.json" + filename = make_testdata_path(None) / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"] diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index a37b42351..0d4210d3a 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -16,7 +16,7 @@ from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, load_tickerdata_file, - make_datadir_path, + make_testdata_path, trim_tickerlist) from freqtrade.misc import file_dump_json from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -136,7 +136,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau def test_testdata_path() -> None: - assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_datadir_path(None)) + assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) def test_load_cached_data_for_updating(mocker) -> None: From 2c9a519c5e359ba2fea353bfd29155c06c8a2d3b Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 22 May 2019 14:21:36 +0300 Subject: [PATCH 54/84] edge: handle properly the 'No trades' case --- freqtrade/edge/__init__.py | 1 + freqtrade/optimize/edge_cli.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4801c6cb3..5c7252d88 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -139,6 +139,7 @@ class Edge(): # If no trade found then exit if len(trades) == 0: + logger.info("No trades created.") return False # Fill missing, calculable columns, profit, duration , abs etc. diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 9b628cf2e..818c1e050 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -73,9 +73,10 @@ class EdgeCli(object): floatfmt=floatfmt, tablefmt="pipe") def start(self) -> None: - self.edge.calculate() - print('') # blank like for readability - print(self._generate_edge_table(self.edge._cached_pairs)) + result = self.edge.calculate() + if result: + print('') # blank like for readability + print(self._generate_edge_table(self.edge._cached_pairs)) def setup_configuration(args: Namespace) -> Dict[str, Any]: From 406e266bb4ea12caaaaf86b6ddde5369e7450fea Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 22 May 2019 14:34:35 +0300 Subject: [PATCH 55/84] typo in comment fixed --- freqtrade/optimize/edge_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 818c1e050..d37b930b8 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -75,7 +75,7 @@ class EdgeCli(object): def start(self) -> None: result = self.edge.calculate() if result: - print('') # blank like for readability + print('') # blank line for readability print(self._generate_edge_table(self.edge._cached_pairs)) From 6e1da13920eb3b1ed7ce501cf0c13f5c11a667df Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 22 May 2019 17:19:11 +0300 Subject: [PATCH 56/84] Log message changed --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 5c7252d88..053be6bc3 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -139,7 +139,7 @@ class Edge(): # If no trade found then exit if len(trades) == 0: - logger.info("No trades created.") + logger.info("No trades found.") return False # Fill missing, calculable columns, profit, duration , abs etc. From 7b074765ab059193ec75cacd8739d5e7e5ea6204 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 May 2019 19:48:22 +0200 Subject: [PATCH 57/84] Improve edge tests - cleanup test file --- freqtrade/tests/edge/test_edge.py | 128 ++++++++++++++++++++---------- 1 file changed, 87 insertions(+), 41 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index af8674188..a14e3282e 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -10,10 +10,11 @@ import numpy as np import pytest from pandas import DataFrame, to_datetime +from freqtrade import OperationalException from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.strategy.interface import SellType -from freqtrade.tests.conftest import get_patched_freqtradebot +from freqtrade.tests.conftest import get_patched_freqtradebot, log_has from freqtrade.tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset) @@ -30,7 +31,50 @@ ticker_start_time = arrow.get(2018, 10, 3) ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} +# Helpers for this test file + +def _validate_ohlc(buy_ohlc_sell_matrice): + for index, ohlc in enumerate(buy_ohlc_sell_matrice): + # if not high < open < low or not high < close < low + if not ohlc[3] >= ohlc[2] >= ohlc[4] or not ohlc[3] >= ohlc[5] >= ohlc[4]: + raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!') + return True + + +def _build_dataframe(buy_ohlc_sell_matrice): + _validate_ohlc(buy_ohlc_sell_matrice) + tickers = [] + for ohlc in buy_ohlc_sell_matrice: + ticker = { + 'date': ticker_start_time.shift( + minutes=( + ohlc[0] * + ticker_interval_in_minute)).timestamp * + 1000, + 'buy': ohlc[1], + 'open': ohlc[2], + 'high': ohlc[3], + 'low': ohlc[4], + 'close': ohlc[5], + 'sell': ohlc[6]} + tickers.append(ticker) + + frame = DataFrame(tickers) + frame['date'] = to_datetime(frame['date'], + unit='ms', + utc=True, + infer_datetime_format=True) + + return frame + + +def _time_on_candle(number): + return np.datetime64(ticker_start_time.shift( + minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') + + +# End helper functions # Open trade should be removed from the end tc0 = BTContainer(data=[ # D O H L C V B S @@ -203,46 +247,6 @@ def test_nonexisting_stake_amount(mocker, edge_conf): assert edge.stake_amount('N/O', 1, 2, 1) == 0.15 -def _validate_ohlc(buy_ohlc_sell_matrice): - for index, ohlc in enumerate(buy_ohlc_sell_matrice): - # if not high < open < low or not high < close < low - if not ohlc[3] >= ohlc[2] >= ohlc[4] or not ohlc[3] >= ohlc[5] >= ohlc[4]: - raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!') - return True - - -def _build_dataframe(buy_ohlc_sell_matrice): - _validate_ohlc(buy_ohlc_sell_matrice) - tickers = [] - for ohlc in buy_ohlc_sell_matrice: - ticker = { - 'date': ticker_start_time.shift( - minutes=( - ohlc[0] * - ticker_interval_in_minute)).timestamp * - 1000, - 'buy': ohlc[1], - 'open': ohlc[2], - 'high': ohlc[3], - 'low': ohlc[4], - 'close': ohlc[5], - 'sell': ohlc[6]} - tickers.append(ticker) - - frame = DataFrame(tickers) - frame['date'] = to_datetime(frame['date'], - unit='ms', - utc=True, - infer_datetime_format=True) - - return frame - - -def _time_on_candle(number): - return np.datetime64(ticker_start_time.shift( - minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') - - def test_edge_heartbeat_calculate(mocker, edge_conf): freqtrade = get_patched_freqtradebot(mocker, edge_conf) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) @@ -298,6 +302,40 @@ def test_edge_process_downloaded_data(mocker, edge_conf): assert edge._last_updated <= arrow.utcnow().timestamp + 2 +def test_edge_process_no_data(mocker, edge_conf, caplog): + edge_conf['datadir'] = None + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + + assert not edge.calculate() + assert len(edge._cached_pairs) == 0 + assert log_has("No data found. Edge is stopped ...", caplog.record_tuples) + assert edge._last_updated == 0 + + +def test_edge_process_no_trades(mocker, edge_conf, caplog): + edge_conf['datadir'] = None + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.data.history.load_data', mocked_load_data) + # Return empty + mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', MagicMock(return_value=[])) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + + assert not edge.calculate() + assert len(edge._cached_pairs) == 0 + assert log_has("No trades found.", caplog.record_tuples) + + +def test_edge_init_error(mocker, edge_conf,): + edge_conf['stake_amount'] = 0.5 + mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + with pytest.raises(OperationalException, match='Edge works only with unlimited stake amount'): + get_patched_freqtradebot(mocker, edge_conf) + + def test_process_expectancy(mocker, edge_conf): edge_conf['edge']['min_trade_number'] = 2 freqtrade = get_patched_freqtradebot(mocker, edge_conf) @@ -360,3 +398,11 @@ def test_process_expectancy(mocker, edge_conf): assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384 assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0 assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 + + # Pop last item so no trade is profitable + trades.pop() + trades_df = DataFrame(trades) + trades_df = edge._fill_calculable_fields(trades_df) + final = edge._process_expectancy(trades_df) + assert len(final) == 0 + assert isinstance(final, dict) From 253025c0feb173ccb304ca3d38dcfc2ac7b28477 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 May 2019 19:53:42 +0200 Subject: [PATCH 58/84] Add tests for check_int_positive --- freqtrade/tests/test_arguments.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 0952d1c5d..d71502abb 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -185,3 +185,12 @@ def test_testdata_dl_options() -> None: assert args.export == 'export/folder' assert args.days == 30 assert args.exchange == 'binance' + + +def test_check_int_positive() -> None: + assert Arguments.check_int_positive(2) == 2 + assert Arguments.check_int_positive(6) == 6 + with pytest.raises(argparse.ArgumentTypeError): + Arguments.check_int_positive(-6) + + assert Arguments.check_int_positive(2.5) == 2 From 7bbe8b24832099ca7bac9508373c232c4497f683 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 May 2019 06:22:27 +0200 Subject: [PATCH 59/84] Add a few more testcases for check_int_positive --- freqtrade/tests/test_arguments.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index d71502abb..455f3dbc6 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring, C0103 - import argparse import pytest @@ -194,3 +193,9 @@ def test_check_int_positive() -> None: Arguments.check_int_positive(-6) assert Arguments.check_int_positive(2.5) == 2 + assert Arguments.check_int_positive("3") == 3 + with pytest.raises(argparse.ArgumentTypeError): + Arguments.check_int_positive("3.5") + + with pytest.raises(argparse.ArgumentTypeError): + Arguments.check_int_positive("DeadBeef") From c3e93e7593b7f092e80464ebad25918420246bd3 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 24 May 2019 23:08:56 +0300 Subject: [PATCH 60/84] fix reduce() TypeError in hyperopts --- docs/hyperopt.md | 7 ++++--- freqtrade/optimize/default_hyperopt.py | 14 ++++++++------ user_data/hyperopts/sample_hyperopt.py | 14 ++++++++------ 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index b4e42de16..79ea4771b 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -122,9 +122,10 @@ So let's write the buy strategy using these values: dataframe['macd'], dataframe['macdsignal'] )) - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 return dataframe diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 721848d2e..7f1cb2435 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -70,9 +70,10 @@ class DefaultHyperOpts(IHyperOpt): dataframe['close'], dataframe['sar'] )) - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 return dataframe @@ -129,9 +130,10 @@ class DefaultHyperOpts(IHyperOpt): dataframe['sar'], dataframe['close'] )) - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 return dataframe diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 54f65a7e6..7cb55378e 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -79,9 +79,10 @@ class SampleHyperOpts(IHyperOpt): dataframe['close'], dataframe['sar'] )) - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 return dataframe @@ -138,9 +139,10 @@ class SampleHyperOpts(IHyperOpt): dataframe['sar'], dataframe['close'] )) - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 return dataframe From 469c0b6a558d3194026c97ec637f47d1e4e06eae Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 13:16:00 +0200 Subject: [PATCH 61/84] Adjust check_int_positive tests --- freqtrade/arguments.py | 2 +- freqtrade/tests/test_arguments.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 327915b61..5afa8fa06 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -405,7 +405,7 @@ class Arguments(object): raise Exception('Incorrect syntax for timerange "%s"' % text) @staticmethod - def check_int_positive(value) -> int: + def check_int_positive(value: str) -> int: try: uint = int(value) if uint <= 0: diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 455f3dbc6..ecd108b5e 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -187,13 +187,17 @@ def test_testdata_dl_options() -> None: def test_check_int_positive() -> None: - assert Arguments.check_int_positive(2) == 2 - assert Arguments.check_int_positive(6) == 6 - with pytest.raises(argparse.ArgumentTypeError): - Arguments.check_int_positive(-6) - assert Arguments.check_int_positive(2.5) == 2 assert Arguments.check_int_positive("3") == 3 + assert Arguments.check_int_positive("1") == 1 + assert Arguments.check_int_positive("100") == 100 + + with pytest.raises(argparse.ArgumentTypeError): + Arguments.check_int_positive("-2") + + with pytest.raises(argparse.ArgumentTypeError): + Arguments.check_int_positive("0") + with pytest.raises(argparse.ArgumentTypeError): Arguments.check_int_positive("3.5") From b6484cb2b42e9f7df977f5717180e2442ce42e5e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 15:54:35 +0200 Subject: [PATCH 62/84] Replace technical link --- Dockerfile.technical | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.technical b/Dockerfile.technical index 5339eb232..9431e72d0 100644 --- a/Dockerfile.technical +++ b/Dockerfile.technical @@ -3,4 +3,4 @@ FROM freqtradeorg/freqtrade:develop RUN apt-get update \ && apt-get -y install git \ && apt-get clean \ - && pip install git+https://github.com/berlinguyinca/technical + && pip install git+https://github.com/freqtrade/technical From 4394701de37853183b84d9d982745d29f8fe8a77 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 16:13:18 +0200 Subject: [PATCH 63/84] Seperate docker-documentation --- README.md | 1 - docs/docker.md | 204 +++++++++++++++++++++++++++++++++++++++++ docs/index.md | 7 +- docs/installation.md | 210 ++++--------------------------------------- 4 files changed, 226 insertions(+), 196 deletions(-) create mode 100644 docs/docker.md diff --git a/README.md b/README.md index 8f7578561..98dad1d2e 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,6 @@ The project is currently setup in two main branches: - `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested. - `feat/*` - These are feature branches, which are being worked on heavily. Please don't use these unless you want to test a specific feature. - ## A note on Binance For Binance, please add `"BNB/"` to your blacklist to avoid issues. diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 000000000..767cabf01 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,204 @@ +# Using FreqTrade with Docker + +## Install Docker + +Start by downloading and installing Docker CE for your platform: + +* [Mac](https://docs.docker.com/docker-for-mac/install/) +* [Windows](https://docs.docker.com/docker-for-windows/install/) +* [Linux](https://docs.docker.com/install/) + +Once you have Docker installed, simply create the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. + +## Download the official FreqTrade docker image + +Pull the image from docker hub. + +Branches / tags available can be checked out on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). + +```bash +docker pull freqtradeorg/freqtrade:develop +# Optionally tag the repository so the run-commands remain shorter +docker tag freqtradeorg/freqtrade:develop freqtrade +``` + +To update the image, simply run the above commands again and restart your running container. + +Should you require additional libraries, please [build the image yourself](#build-your-own-docker-image). + +### Prepare the configuration files + +Even though you will use docker, you'll still need some files from the github repository. + +#### Clone the git repository + +Linux/Mac/Windows with WSL + +```bash +git clone https://github.com/freqtrade/freqtrade.git +``` + +Windows with docker + +```bash +git clone --config core.autocrlf=input https://github.com/freqtrade/freqtrade.git +``` + +#### Copy `config.json.example` to `config.json` + +```bash +cd freqtrade +cp -n config.json.example config.json +``` + +> To understand the configuration options, please refer to the [Bot Configuration](configuration.md) page. + +#### Create your database file + +Production + +```bash +touch tradesv3.sqlite +```` + +Dry-Run + +```bash +touch tradesv3.dryrun.sqlite +``` + +!!! Note + Make sure to use the path to this file when starting the bot in docker. + +### Build your own Docker image + +Best start by pulling the official docker image from dockerhub as explained [here](#download-the-official-docker-image) to speed up building. + +To add additional libaries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image. + +```bash +docker build -t freqtrade -f Dockerfile.technical . +``` + +If you are developing using Docker, use `Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies: + +```bash +docker build -f Dockerfile.develop -t freqtrade-dev . +``` + +!!! Note + For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see the "5. Run a restartable docker image" section) to keep it between updates. + +#### Verify the Docker image + +After the build process you can verify that the image was created with: + +```bash +docker images +``` + +The output should contain the freqtrade image. + +### Run the Docker image + +You can run a one-off container that is immediately deleted upon exiting with the following command (`config.json` must be in the current working directory): + +```bash +docker run --rm -v `pwd`/config.json:/freqtrade/config.json -it freqtrade +``` + +!!! Warning + In this example, the database will be created inside the docker instance and will be lost when you will refresh your image. + +#### Adjust timezone + +By default, the container will use UTC timezone. +Should you find this irritating please add the following to your docker commands: + +##### Linux + +``` bash +-v /etc/timezone:/etc/timezone:ro + +# Complete command: +docker run --rm -v /etc/timezone:/etc/timezone:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade +``` + +##### MacOS + +There is known issue in OSX Docker versions after 17.09.1, whereby `/etc/localtime` cannot be shared causing Docker to not start. A work-around for this is to start with the following cmd. + +```bash +docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade +``` + +More information on this docker issue and work-around can be read [here](https://github.com/docker/for-mac/issues/2396). + +### Run a restartable docker image + +To run a restartable instance in the background (feel free to place your configuration and database files wherever it feels comfortable on your filesystem). + +#### Move your config file and database + +The following will assume that you place your configuration / database files to `~/.freqtrade`, which is a hidden folder in your home directory. Feel free to use a different folder and replace the folder in the upcomming commands. + +```bash +mkdir ~/.freqtrade +mv config.json ~/.freqtrade +mv tradesv3.sqlite ~/.freqtrade +``` + +#### Run the docker image + +```bash +docker run -d \ + --name freqtrade \ + -v ~/.freqtrade/config.json:/freqtrade/config.json \ + -v ~/.freqtrade/user_data/:/freqtrade/user_data \ + -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ + freqtrade --db-url sqlite:///tradesv3.sqlite --strategy MyAwesomeStrategy +``` + +!!! Note + db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. + To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` + +!!! Note + All available command line arguments can be added to the end of the `docker run` command. + +### Monitor your Docker instance + +You can use the following commands to monitor and manage your container: + +```bash +docker logs freqtrade +docker logs -f freqtrade +docker restart freqtrade +docker stop freqtrade +docker start freqtrade +``` + +For more information on how to operate Docker, please refer to the [official Docker documentation](https://docs.docker.com/). + +!!! Note + You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. + +### Backtest with docker + +The following assumes that the download/setup of the docker image have been completed successfully. +Also, backtest-data should be available at `~/.freqtrade/user_data/`. + +```bash +docker run -d \ + --name freqtrade \ + -v /etc/localtime:/etc/localtime:ro \ + -v ~/.freqtrade/config.json:/freqtrade/config.json \ + -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ + -v ~/.freqtrade/user_data/:/freqtrade/user_data/ \ + freqtrade --strategy AwsomelyProfitableStrategy backtesting +``` + +Head over to the [Backtesting Documentation](backtesting.md) for more details. + +!!! Note + Additional parameters can be appended after the image name (`freqtrade` in the above example). diff --git a/docs/index.md b/docs/index.md index 9abc71747..a6ae6312d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -36,12 +36,14 @@ Freqtrade is a cryptocurrency trading bot written in Python. - Daily summary of profit/loss: Receive the daily summary of your profit/loss. - Performance status report: Receive the performance status of your current trades. - ## Requirements + ### Up to date clock + The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges. ### Hardware requirements + To run this bot we recommend you a cloud instance with a minimum of: - 2GB RAM @@ -49,6 +51,7 @@ To run this bot we recommend you a cloud instance with a minimum of: - 2vCPU ### Software requirements + - Python 3.6.x - pip (pip3) - git @@ -58,10 +61,12 @@ To run this bot we recommend you a cloud instance with a minimum of: ## Support + Help / Slack For any questions not covered by the documentation or for further information about the bot, we encourage you to join our Slack channel. Click [here](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) to join Slack channel. ## Ready to try? + Begin by reading our installation guide [here](installation). diff --git a/docs/installation.md b/docs/installation.md index 11ddc010d..ed38c1340 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,7 +1,9 @@ # Installation + This page explains how to prepare your environment for running the bot. ## Prerequisite + Before running your bot in production you will need to setup few external API. In production mode, the bot required valid Bittrex API credentials and a Telegram bot (optional but recommended). @@ -10,9 +12,11 @@ credentials and a Telegram bot (optional but recommended). - [Backtesting commands](#setup-your-telegram-bot) ### Setup your exchange account + *To be completed, please feel free to complete this section.* ### Setup your Telegram bot + The only things you need is a working Telegram bot and its API token. Below we explain how to create your Telegram Bot, and how to get your Telegram user id. @@ -35,7 +39,9 @@ Good. Now let's choose a username for your bot. It must end in `bot`. Like this, **1.5. Father bot will return you the token (API key)**
Copy it and keep it you will use it for the config parameter `token`. + *BotFather response:* + ```hl_lines="4" Done! Congratulations on your new bot. You will find it at t.me/My_own_freqtrade_bot. You can now add a description, about section and profile picture for your bot, see /help for a list of commands. By the way, when you've finished creating your cool bot, ping our Bot Support if you want a better username for it. Just make sure the bot is fully operational before you do this. @@ -44,15 +50,18 @@ Use this token to access the HTTP API: For a description of the Bot API, see this page: https://core.telegram.org/bots/api ``` + **1.6. Don't forget to start the conversation with your bot, by clicking /START button** ### 2. Get your user id + **2.1. Talk to https://telegram.me/userinfobot** **2.2. Get your "Id", you will use it for the config parameter `chat_id`.** -
+ ## Quick start + Freqtrade provides a Linux/MacOS script to install all dependencies and help you to configure the bot. ```bash @@ -61,9 +70,10 @@ cd freqtrade git checkout develop ./setup.sh --install ``` + !!! Note Windows installation is explained [here](#windows). -
+ ## Easy Installation - Linux Script If you are on Debian, Ubuntu or MacOS a freqtrade provides a script to Install, Update, Configure, and Reset your bot. @@ -101,193 +111,6 @@ Config parameter is a `config.json` configurator. This script will ask you quest ------ -## Automatic Installation - Docker - -Start by downloading Docker for your platform: - -* [Mac](https://www.docker.com/products/docker#/mac) -* [Windows](https://www.docker.com/products/docker#/windows) -* [Linux](https://www.docker.com/products/docker#/linux) - -Once you have Docker installed, simply create the config file (e.g. `config.json`) and then create a Docker image for `freqtrade` using the Dockerfile in this repo. - -### 1. Prepare the Bot - -**1.1. Clone the git repository** - -Linux/Mac/Windows with WSL -```bash -git clone https://github.com/freqtrade/freqtrade.git -``` - -Windows with docker -```bash -git clone --config core.autocrlf=input https://github.com/freqtrade/freqtrade.git -``` - -**1.2. (Optional) Checkout the develop branch** - -```bash -git checkout develop -``` - -**1.3. Go into the new directory** - -```bash -cd freqtrade -``` - -**1.4. Copy `config.json.example` to `config.json`** - -```bash -cp -n config.json.example config.json -``` - -> To edit the config please refer to the [Bot Configuration](configuration.md) page. - -**1.5. Create your database file *(optional - the bot will create it if it is missing)** - -Production - -```bash -touch tradesv3.sqlite -```` - -Dry-Run - -```bash -touch tradesv3.dryrun.sqlite -``` - -### 2. Download or build the docker image - -Either use the prebuilt image from docker hub - or build the image yourself if you would like more control on which version is used. - -Branches / tags available can be checked out on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). - -**2.1. Download the docker image** - -Pull the image from docker hub and (optionally) change the name of the image - -```bash -docker pull freqtradeorg/freqtrade:develop -# Optionally tag the repository so the run-commands remain shorter -docker tag freqtradeorg/freqtrade:develop freqtrade -``` - -To update the image, simply run the above commands again and restart your running container. - -**2.2. Build the Docker image** - -```bash -cd freqtrade -docker build -t freqtrade . -``` - -If you are developing using Docker, use `Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies: - -```bash -docker build -f ./Dockerfile.develop -t freqtrade-dev . -``` - -For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see the "5. Run a restartable docker image" section) to keep it between updates. - -### 3. Verify the Docker image - -After the build process you can verify that the image was created with: - -```bash -docker images -``` - -### 4. Run the Docker image - -You can run a one-off container that is immediately deleted upon exiting with the following command (`config.json` must be in the current working directory): - -```bash -docker run --rm -v /etc/localtime:/etc/localtime:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` - -There is known issue in OSX Docker versions after 17.09.1, whereby /etc/localtime cannot be shared causing Docker to not start. A work-around for this is to start with the following cmd. - -```bash -docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` - -More information on this docker issue and work-around can be read [here](https://github.com/docker/for-mac/issues/2396). - -In this example, the database will be created inside the docker instance and will be lost when you will refresh your image. - -### 5. Run a restartable docker image - -To run a restartable instance in the background (feel free to place your configuration and database files wherever it feels comfortable on your filesystem). - -**5.1. Move your config file and database** - -```bash -mkdir ~/.freqtrade -mv config.json ~/.freqtrade -mv tradesv3.sqlite ~/.freqtrade -``` - -**5.2. Run the docker image** - -```bash -docker run -d \ - --name freqtrade \ - -v /etc/localtime:/etc/localtime:ro \ - -v ~/.freqtrade/config.json:/freqtrade/config.json \ - -v ~/.freqtrade/user_data/:/freqtrade/user_data \ - -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ - freqtrade --db-url sqlite:///tradesv3.sqlite --strategy MyAwesomeStrategy -``` - -!!! Note - db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. - To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` - -!!! Note - All command line arguments can be added to the end of the `docker run` command. - -### 6. Monitor your Docker instance - -You can then use the following commands to monitor and manage your container: - -```bash -docker logs freqtrade -docker logs -f freqtrade -docker restart freqtrade -docker stop freqtrade -docker start freqtrade -``` - -For more information on how to operate Docker, please refer to the [official Docker documentation](https://docs.docker.com/). - -!!! Note - You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. - -### 7. Backtest with docker - -The following assumes that the above steps (1-4) have been completed successfully. -Also, backtest-data should be available at `~/.freqtrade/user_data/`. - -```bash -docker run -d \ - --name freqtrade \ - -v /etc/localtime:/etc/localtime:ro \ - -v ~/.freqtrade/config.json:/freqtrade/config.json \ - -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ - -v ~/.freqtrade/user_data/:/freqtrade/user_data/ \ - freqtrade --strategy AwsomelyProfitableStrategy backtesting -``` - -Head over to the [Backtesting Documentation](backtesting.md) for more details. - -!!! Note - Additional parameters can be appended after the image name (`freqtrade` in the above example). - ------- - ## Custom Installation We've included/collected install instructions for Ubuntu 16.04, MacOS, and Windows. These are guidelines and your success may vary with other distros. @@ -413,7 +236,7 @@ If this is the first time you run the bot, ensure you are running it in Dry-run python3.6 freqtrade -c config.json ``` -*Note*: If you run the bot on a server, you should consider using [Docker](#automatic-installation---docker) a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout. +*Note*: If you run the bot on a server, you should consider using [Docker](docker.md) or a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout. #### 7. [Optional] Configure `freqtrade` as a `systemd` service @@ -441,14 +264,13 @@ The `freqtrade.service.watchdog` file contains an example of the service unit co as the watchdog. !!! Note - The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a - Docker container. + The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container. ------ ## Windows -We recommend that Windows users use [Docker](#docker) as this will work much easier and smoother (also more secure). +We recommend that Windows users use [Docker](docker.md) as this will work much easier and smoother (also more secure). If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work. If that is not available on your system, feel free to try the instructions below, which led to success for some. @@ -492,7 +314,7 @@ error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Unfortunately, many packages requiring compilation don't provide a pre-build wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use. -The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building c code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or docker first. +The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building c code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or [docker](docker.md) first. --- From 3e0a71f69f99ef82ebc6ad4730259ed4e780d106 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 16:27:18 +0200 Subject: [PATCH 64/84] Add docker install script to mkdocs index --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index 9932ff316..489107f2e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -2,6 +2,7 @@ site_name: Freqtrade nav: - About: index.md - Installation: installation.md + - Installation Docker: docker.md - Configuration: configuration.md - Strategy Customization: strategy-customization.md - Stoploss: stoploss.md From 26a8cdcc031d6fa7cba4dc2a3f7ada3807d5a297 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 16:27:36 +0200 Subject: [PATCH 65/84] Move telegram-setup to telegram page --- docs/installation.md | 52 +++--------------------------------------- docs/telegram-usage.md | 43 ++++++++++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 53 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index ed38c1340..d215dc8d6 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -5,60 +5,14 @@ This page explains how to prepare your environment for running the bot. ## Prerequisite Before running your bot in production you will need to setup few -external API. In production mode, the bot required valid Bittrex API -credentials and a Telegram bot (optional but recommended). +external API. In production mode, the bot will require valid Exchange API +credentials. We also reccomend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot) (optional but recommended). - [Setup your exchange account](#setup-your-exchange-account) -- [Backtesting commands](#setup-your-telegram-bot) ### Setup your exchange account -*To be completed, please feel free to complete this section.* - -### Setup your Telegram bot - -The only things you need is a working Telegram bot and its API token. -Below we explain how to create your Telegram Bot, and how to get your -Telegram user id. - -### 1. Create your Telegram bot - -**1.1. Start a chat with https://telegram.me/BotFather** - -**1.2. Send the message `/newbot`. ** *BotFather response:* -``` -Alright, a new bot. How are we going to call it? Please choose a name for your bot. -``` - -**1.3. Choose the public name of your bot (e.x. `Freqtrade bot`)** -*BotFather response:* -``` -Good. Now let's choose a username for your bot. It must end in `bot`. Like this, for example: TetrisBot or tetris_bot. -``` -**1.4. Choose the name id of your bot (e.x "`My_own_freqtrade_bot`")** - -**1.5. Father bot will return you the token (API key)**
-Copy it and keep it you will use it for the config parameter `token`. - -*BotFather response:* - -```hl_lines="4" -Done! Congratulations on your new bot. You will find it at t.me/My_own_freqtrade_bot. You can now add a description, about section and profile picture for your bot, see /help for a list of commands. By the way, when you've finished creating your cool bot, ping our Bot Support if you want a better username for it. Just make sure the bot is fully operational before you do this. - -Use this token to access the HTTP API: -521095879:AAEcEZEL7ADJ56FtG_qD0bQJSKETbXCBCi0 - -For a description of the Bot API, see this page: https://core.telegram.org/bots/api -``` - -**1.6. Don't forget to start the conversation with your bot, by clicking /START button** - -### 2. Get your user id - -**2.1. Talk to https://telegram.me/userinfobot** - -**2.2. Get your "Id", you will use it for the config parameter -`chat_id`.** +You will need to create API Keys (Usually you get `key` and `secret`) from the Exchange website and insert this into the appropriate fields in the configuration or when asked by the installation script. ## Quick start diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 3947168c5..e06d4fdfc 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -1,10 +1,45 @@ # Telegram usage -## Prerequisite +## Setup your Telegram bot -To control your bot with Telegram, you need first to -[set up a Telegram bot](installation.md) -and add your Telegram API keys into your config file. +Below we explain how to create your Telegram Bot, and how to get your +Telegram user id. + +### 1. Create your Telegram bot + +Start a chat with the [Telegram BotFather](https://telegram.me/BotFather) + +Send the message `/newbot`. + +*BotFather response:* + +> Alright, a new bot. How are we going to call it? Please choose a name for your bot. + +Choose the public name of your bot (e.x. `Freqtrade bot`) + +*BotFather response:* + +> Good. Now let's choose a username for your bot. It must end in `bot`. Like this, for example: TetrisBot or tetris_bot. + +Choose the name id of your bot and send it to the BotFather (e.g. "`My_own_freqtrade_bot`") + +*BotFather response:* + +> Done! Congratulations on your new bot. You will find it at `t.me/yourbots_name_bot`. You can now add a description, about section and profile picture for your bot, see /help for a list of commands. By the way, when you've finished creating your cool bot, ping our Bot Support if you want a better username for it. Just make sure the bot is fully operational before you do this. + +> Use this token to access the HTTP API: `22222222:APITOKEN` + +> For a description of the Bot API, see this page: https://core.telegram.org/bots/api Father bot will return you the token (API key) + +Copy the API Token (`22222222:APITOKEN` in the above example) and keep use it for the config parameter `token`. + +Don't forget to start the conversation with your bot, by clicking `/START` button + +### 2. Get your user id + +Talk to the [userinfobot](https://telegram.me/userinfobot) + +Get your "Id", you will use it for the config parameter `chat_id`. ## Telegram commands From 9225cdea8a770453c59b203fb8c2b8f8fbb40fe2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 16:51:52 +0200 Subject: [PATCH 66/84] Move validate_backtest_data and get_timeframe to histoyr --- freqtrade/data/history.py | 44 ++++++++++++++++++++++++++++--- freqtrade/edge/__init__.py | 4 +-- freqtrade/optimize/__init__.py | 48 ---------------------------------- 3 files changed, 42 insertions(+), 54 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 3bec63926..e0f9f67db 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -5,19 +5,21 @@ Includes: * load data for a pair (or a list of pairs) from disk * download data from exchange and store to disk """ + import logging +import operator +from datetime import datetime from pathlib import Path -from typing import Optional, List, Dict, Tuple, Any +from typing import Any, Dict, List, Optional, Tuple import arrow from pandas import DataFrame -from freqtrade import misc, OperationalException +from freqtrade import OperationalException, misc from freqtrade.arguments import TimeRange from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.exchange import Exchange, timeframe_to_minutes - logger = logging.getLogger(__name__) @@ -243,3 +245,39 @@ def download_pair_history(datadir: Optional[Path], f'Error: {e}' ) return False + + +def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum timeframe for the given backtest data + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + timeframe = [ + (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) + for frame in data.values() + ] + return min(timeframe, key=operator.itemgetter(0))[0], \ + max(timeframe, key=operator.itemgetter(1))[1] + + +def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, + max_date: datetime, ticker_interval_mins: int) -> bool: + """ + Validates preprocessed backtesting data for missing values and shows warnings about it that. + + :param data: dictionary with preprocessed backtesting data + :param min_date: start-date of the data + :param max_date: end-date of the data + :param ticker_interval_mins: ticker interval in minutes + """ + # total difference in minutes / interval-minutes + expected_frames = int((max_date - min_date).total_seconds() // 60 // ticker_interval_mins) + found_missing = False + for pair, df in data.items(): + dflen = len(df) + if dflen < expected_frames: + found_missing = True + logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values", + pair, expected_frames, dflen, expected_frames - dflen) + return found_missing diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 053be6bc3..3ddff4772 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -13,7 +13,6 @@ from freqtrade import constants, OperationalException from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.data import history -from freqtrade.optimize import get_timeframe from freqtrade.strategy.interface import SellType @@ -49,7 +48,6 @@ class Edge(): self.strategy = strategy self.ticker_interval = self.strategy.ticker_interval self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.get_timeframe = get_timeframe self.advise_sell = self.strategy.advise_sell self.advise_buy = self.strategy.advise_buy @@ -117,7 +115,7 @@ class Edge(): preprocessed = self.tickerdata_to_dataframe(data) # Print timeframe - min_date, max_date = self.get_timeframe(preprocessed) + min_date, max_date = history.get_timeframe(preprocessed) logger.info( 'Measuring data from %s up to %s (%s days) ...', min_date.isoformat(), diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 19b8dd90a..f4f31720b 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,49 +1 @@ -# pragma pylint: disable=missing-docstring - -import logging -from datetime import datetime -from typing import Dict, Tuple -import operator - -import arrow -from pandas import DataFrame - from freqtrade.optimize.default_hyperopt import DefaultHyperOpts # noqa: F401 - -logger = logging.getLogger(__name__) - - -def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: - """ - Get the maximum timeframe for the given backtest data - :param data: dictionary with preprocessed backtesting data - :return: tuple containing min_date, max_date - """ - timeframe = [ - (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) - for frame in data.values() - ] - return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] - - -def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, - max_date: datetime, ticker_interval_mins: int) -> bool: - """ - Validates preprocessed backtesting data for missing values and shows warnings about it that. - - :param data: dictionary with preprocessed backtesting data - :param min_date: start-date of the data - :param max_date: end-date of the data - :param ticker_interval_mins: ticker interval in minutes - """ - # total difference in minutes / interval-minutes - expected_frames = int((max_date - min_date).total_seconds() // 60 // ticker_interval_mins) - found_missing = False - for pair, df in data.items(): - dflen = len(df) - if dflen < expected_frames: - found_missing = True - logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values", - pair, expected_frames, dflen, expected_frames - dflen) - return found_missing From b38c43141c44d26702cb69f88ce58b85c111e577 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 16:53:35 +0200 Subject: [PATCH 67/84] Adjust imports to new location --- freqtrade/optimize/backtesting.py | 7 +++---- freqtrade/optimize/hyperopt.py | 3 +-- freqtrade/tests/data/test_converter.py | 3 +-- freqtrade/tests/optimize/test_backtest_detail.py | 12 ++++++------ freqtrade/tests/optimize/test_backtesting.py | 6 +++--- freqtrade/tests/optimize/test_hyperopt.py | 2 +- freqtrade/tests/optimize/test_optimize.py | 15 +++++++-------- 7 files changed, 22 insertions(+), 26 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 51122cfb2..c7ce29120 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -13,7 +13,6 @@ from typing import Any, Dict, List, NamedTuple, Optional from pandas import DataFrame from tabulate import tabulate -from freqtrade import optimize from freqtrade import DependencyException, constants from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration @@ -440,10 +439,10 @@ class Backtesting(object): logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) self._set_strategy(strat) - min_date, max_date = optimize.get_timeframe(data) + min_date, max_date = history.get_timeframe(data) # Validate dataframe for missing values (mainly at start and end, as fillup is called) - optimize.validate_backtest_data(data, min_date, max_date, - timeframe_to_minutes(self.ticker_interval)) + history.validate_backtest_data(data, min_date, max_date, + timeframe_to_minutes(self.ticker_interval)) logger.info( 'Backtesting with data from %s up to %s (%s days)..', min_date.isoformat(), diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 92589aed2..68c7b2508 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -23,9 +23,8 @@ from skopt.space import Dimension from freqtrade import DependencyException from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration -from freqtrade.data.history import load_data +from freqtrade.data.history import load_data, get_timeframe, validate_backtest_data from freqtrade.exchange import timeframe_to_minutes -from freqtrade.optimize import get_timeframe, validate_backtest_data from freqtrade.optimize.backtesting import Backtesting from freqtrade.state import RunMode from freqtrade.resolvers import HyperOptResolver diff --git a/freqtrade/tests/data/test_converter.py b/freqtrade/tests/data/test_converter.py index 46d564003..4c8de575d 100644 --- a/freqtrade/tests/data/test_converter.py +++ b/freqtrade/tests/data/test_converter.py @@ -2,8 +2,7 @@ import logging from freqtrade.data.converter import parse_ticker_dataframe, ohlcv_fill_up_missing_data -from freqtrade.data.history import load_pair_history -from freqtrade.optimize import validate_backtest_data, get_timeframe +from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timeframe from freqtrade.tests.conftest import log_has diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index b98369533..32c6bd09b 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -2,17 +2,17 @@ import logging from unittest.mock import MagicMock -from pandas import DataFrame import pytest +from pandas import DataFrame - -from freqtrade.optimize import get_timeframe +from freqtrade.data.history import get_timeframe from freqtrade.optimize.backtesting import Backtesting from freqtrade.strategy.interface import SellType -from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, - _get_frame_time_from_offset, tests_ticker_interval) from freqtrade.tests.conftest import patch_exchange - +from freqtrade.tests.optimize import (BTContainer, BTrade, + _build_backtest_dataframe, + _get_frame_time_from_offset, + tests_ticker_interval) # Test 1 Minus 8% Close # Test with Stop-loss at 1% diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 6a39deed4..07ad7eaff 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -17,7 +17,7 @@ from freqtrade.data import history from freqtrade.data.btanalysis import evaluate_result_multi from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.dataprovider import DataProvider -from freqtrade.optimize import get_timeframe +from freqtrade.data.history import get_timeframe from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) from freqtrade.state import RunMode @@ -472,7 +472,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.data.history.load_data', mocked_load_data) - mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) + mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( @@ -507,7 +507,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) - mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) + mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index f50f58e5b..9d1789171 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -12,7 +12,7 @@ from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.optimize.default_hyperopt import DefaultHyperOpts from freqtrade.optimize.hyperopt import Hyperopt, setup_configuration, start -from freqtrade.resolvers import HyperOptResolver +from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.state import RunMode from freqtrade.tests.conftest import log_has, log_has_re, patch_exchange from freqtrade.tests.optimize.test_backtesting import get_args diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index d746aa44f..401592b53 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 -from freqtrade import optimize from freqtrade.arguments import TimeRange from freqtrade.data import history from freqtrade.exchange import timeframe_to_minutes @@ -18,7 +17,7 @@ def test_get_timeframe(default_conf, mocker) -> None: pairs=['UNITTEST/BTC'] ) ) - min_date, max_date = optimize.get_timeframe(data) + min_date, max_date = history.get_timeframe(data) assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' @@ -35,10 +34,10 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: fill_up_missing=False ) ) - min_date, max_date = optimize.get_timeframe(data) + min_date, max_date = history.get_timeframe(data) caplog.clear() - assert optimize.validate_backtest_data(data, min_date, max_date, - timeframe_to_minutes('1m')) + assert history.validate_backtest_data(data, min_date, max_date, + timeframe_to_minutes('1m')) assert len(caplog.record_tuples) == 1 assert log_has( "UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values", @@ -59,8 +58,8 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: ) ) - min_date, max_date = optimize.get_timeframe(data) + min_date, max_date = history.get_timeframe(data) caplog.clear() - assert not optimize.validate_backtest_data(data, min_date, max_date, - timeframe_to_minutes('5m')) + assert not history.validate_backtest_data(data, min_date, max_date, + timeframe_to_minutes('5m')) assert len(caplog.record_tuples) == 0 From 236c392d282fc1516f05531651546c732b520b84 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 20:00:31 +0200 Subject: [PATCH 68/84] Don't load hyperopts / optimize dependency tree if that module is not used --- freqtrade/arguments.py | 6 +- freqtrade/optimize/__init__.py | 100 +++++++++++++++++++++++++++++- freqtrade/optimize/backtesting.py | 42 +------------ freqtrade/optimize/hyperopt.py | 66 +------------------- freqtrade/resolvers/__init__.py | 3 +- 5 files changed, 106 insertions(+), 111 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 5afa8fa06..e79d0c6d4 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -340,13 +340,13 @@ class Arguments(object): Builds and attaches all subcommands :return: None """ - from freqtrade.optimize import backtesting, hyperopt, edge_cli + from freqtrade.optimize import start_backtesting, start_hyperopt, edge_cli subparsers = self.parser.add_subparsers(dest='subparser') # Add backtesting subcommand backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.') - backtesting_cmd.set_defaults(func=backtesting.start) + backtesting_cmd.set_defaults(func=start_backtesting) self.optimizer_shared_options(backtesting_cmd) self.backtesting_options(backtesting_cmd) @@ -358,7 +358,7 @@ class Arguments(object): # Add hyperopt subcommand hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.') - hyperopt_cmd.set_defaults(func=hyperopt.start) + hyperopt_cmd.set_defaults(func=start_hyperopt) self.optimizer_shared_options(hyperopt_cmd) self.hyperopt_options(hyperopt_cmd) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index f4f31720b..34076ee43 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1 +1,99 @@ -from freqtrade.optimize.default_hyperopt import DefaultHyperOpts # noqa: F401 +import logging +from argparse import Namespace +from typing import Any, Dict + +from filelock import FileLock, Timeout + +from freqtrade import DependencyException, constants +from freqtrade.configuration import Configuration +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def setup_configuration(args: Namespace, method: RunMode) -> Dict[str, Any]: + """ + Prepare the configuration for the Hyperopt module + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args, method) + config = configuration.load_config() + + # Ensure we do not use Exchange credentials + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + + if method == RunMode.BACKTEST: + if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: + raise DependencyException('stake amount could not be "%s" for backtesting' % + constants.UNLIMITED_STAKE_AMOUNT) + + if method == RunMode.HYPEROPT: + # Special cases for Hyperopt + if config.get('strategy') and config.get('strategy') != 'DefaultStrategy': + logger.error("Please don't use --strategy for hyperopt.") + logger.error( + "Read the documentation at " + "https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md " + "to understand how to configure hyperopt.") + raise DependencyException("--strategy configured but not supported for hyperopt") + + return config + + +def start_backtesting(args: Namespace) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + # Import here to avoid loading backtesting module when it's not used + from freqtrade.optimize.backtesting import Backtesting + + # Initialize configuration + config = setup_configuration(args, RunMode.BACKTEST) + + logger.info('Starting freqtrade in Backtesting mode') + + # Initialize backtesting object + backtesting = Backtesting(config) + backtesting.start() + + +def start_hyperopt(args: Namespace) -> None: + """ + Start hyperopt script + :param args: Cli args from Arguments() + :return: None + """ + # Import here to avoid loading hyperopt module when it's not used + from freqtrade.optimize.hyperopt import Hyperopt, HYPEROPT_LOCKFILE + + # Initialize configuration + config = setup_configuration(args, RunMode.HYPEROPT) + + logger.info('Starting freqtrade in Hyperopt mode') + + lock = FileLock(HYPEROPT_LOCKFILE) + + try: + with lock.acquire(timeout=1): + + # Remove noisy log messages + logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) + logging.getLogger('filelock').setLevel(logging.WARNING) + + # Initialize backtesting object + hyperopt = Hyperopt(config) + hyperopt.start() + + except Timeout: + logger.info("Another running instance of freqtrade Hyperopt detected.") + logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. " + "Hyperopt module is resource hungry. Please run your Hyperopts sequentially " + "or on separate machines.") + logger.info("Quitting now.") + # TODO: return False here in order to help freqtrade to exit + # with non-zero exit code... + # Same in Edge and Backtesting start() functions. diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c7ce29120..bdd42943b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -4,7 +4,6 @@ This module contains the backtesting logic """ import logging -from argparse import Namespace from copy import deepcopy from datetime import datetime, timedelta from pathlib import Path @@ -13,9 +12,7 @@ from typing import Any, Dict, List, NamedTuple, Optional from pandas import DataFrame from tabulate import tabulate -from freqtrade import DependencyException, constants from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration from freqtrade.data import history from freqtrade.data.dataprovider import DataProvider from freqtrade.exchange import timeframe_to_minutes @@ -23,8 +20,7 @@ from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode -from freqtrade.strategy.interface import SellType, IStrategy - +from freqtrade.strategy.interface import IStrategy, SellType logger = logging.getLogger(__name__) @@ -485,39 +481,3 @@ class Backtesting(object): print(' Strategy Summary '.center(133, '=')) print(self._generate_text_table_strategy(all_results)) print('\nFor more details, please look at the detail tables above') - - -def setup_configuration(args: Namespace) -> Dict[str, Any]: - """ - Prepare the configuration for the backtesting - :param args: Cli args from Arguments() - :return: Configuration - """ - configuration = Configuration(args, RunMode.BACKTEST) - config = configuration.get_config() - - # Ensure we do not use Exchange credentials - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - - if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: - raise DependencyException('stake amount could not be "%s" for backtesting' % - constants.UNLIMITED_STAKE_AMOUNT) - - return config - - -def start(args: Namespace) -> None: - """ - Start Backtesting script - :param args: Cli args from Arguments() - :return: None - """ - # Initialize configuration - config = setup_configuration(args) - - logger.info('Starting freqtrade in Backtesting mode') - - # Initialize backtesting object - backtesting = Backtesting(config) - backtesting.start() diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 68c7b2508..d19d54031 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -7,27 +7,22 @@ This module contains the hyperopt logic import logging import os import sys -from argparse import Namespace from math import exp from operator import itemgetter from pathlib import Path from pprint import pprint from typing import Any, Dict, List -from filelock import Timeout, FileLock from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects, cpu_count from pandas import DataFrame from skopt import Optimizer from skopt.space import Dimension -from freqtrade import DependencyException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration from freqtrade.data.history import load_data, get_timeframe, validate_backtest_data from freqtrade.exchange import timeframe_to_minutes from freqtrade.optimize.backtesting import Backtesting -from freqtrade.state import RunMode -from freqtrade.resolvers import HyperOptResolver +from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver logger = logging.getLogger(__name__) @@ -342,62 +337,3 @@ class Hyperopt(Backtesting): self.save_trials() self.log_trials_result() - - -def setup_configuration(args: Namespace) -> Dict[str, Any]: - """ - Prepare the configuration for the Hyperopt module - :param args: Cli args from Arguments() - :return: Configuration - """ - configuration = Configuration(args, RunMode.HYPEROPT) - config = configuration.load_config() - - # Ensure we do not use Exchange credentials - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - - if config.get('strategy') and config.get('strategy') != 'DefaultStrategy': - logger.error("Please don't use --strategy for hyperopt.") - logger.error( - "Read the documentation at " - "https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md " - "to understand how to configure hyperopt.") - raise DependencyException("--strategy configured but not supported for hyperopt") - - return config - - -def start(args: Namespace) -> None: - """ - Start Backtesting script - :param args: Cli args from Arguments() - :return: None - """ - # Initialize configuration - config = setup_configuration(args) - - logger.info('Starting freqtrade in Hyperopt mode') - - lock = FileLock(HYPEROPT_LOCKFILE) - - try: - with lock.acquire(timeout=1): - - # Remove noisy log messages - logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) - logging.getLogger('filelock').setLevel(logging.WARNING) - - # Initialize backtesting object - hyperopt = Hyperopt(config) - hyperopt.start() - - except Timeout: - logger.info("Another running instance of freqtrade Hyperopt detected.") - logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. " - "Hyperopt module is resource hungry. Please run your Hyperopts sequentially " - "or on separate machines.") - logger.info("Quitting now.") - # TODO: return False here in order to help freqtrade to exit - # with non-zero exit code... - # Same in Edge and Backtesting start() functions. diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py index 5cf6c616a..8f79349fe 100644 --- a/freqtrade/resolvers/__init__.py +++ b/freqtrade/resolvers/__init__.py @@ -1,5 +1,6 @@ from freqtrade.resolvers.iresolver import IResolver # noqa: F401 from freqtrade.resolvers.exchange_resolver import ExchangeResolver # noqa: F401 -from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver # noqa: F401 +# Don't import HyperoptResolver to avoid loading the whole Optimize tree +# from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver # noqa: F401 from freqtrade.resolvers.pairlist_resolver import PairListResolver # noqa: F401 from freqtrade.resolvers.strategy_resolver import StrategyResolver # noqa: F401 From 65a4862d1f420b9a1df673ba6bfe32a6457db639 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 20:01:43 +0200 Subject: [PATCH 69/84] Adapt tests to load start_* methods from optimize --- freqtrade/tests/optimize/test_backtesting.py | 16 ++++++++-------- freqtrade/tests/optimize/test_hyperopt.py | 13 +++++++------ freqtrade/tests/test_main.py | 4 ++-- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 07ad7eaff..5b42cae34 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -18,8 +18,8 @@ from freqtrade.data.btanalysis import evaluate_result_multi from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timeframe -from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, - start) +from freqtrade.optimize import setup_configuration, start_backtesting +from freqtrade.optimize.backtesting import Backtesting from freqtrade.state import RunMode from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import SellType @@ -178,7 +178,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> 'backtesting' ] - config = setup_configuration(get_args(args)) + config = setup_configuration(get_args(args), RunMode.BACKTEST) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -228,7 +228,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--export-filename', 'foo_bar.json' ] - config = setup_configuration(get_args(args)) + config = setup_configuration(get_args(args), RunMode.BACKTEST) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -290,7 +290,7 @@ def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog ] with pytest.raises(DependencyException, match=r'.*stake amount.*'): - setup_configuration(get_args(args)) + setup_configuration(get_args(args), RunMode.BACKTEST) def test_start(mocker, fee, default_conf, caplog) -> None: @@ -307,7 +307,7 @@ def test_start(mocker, fee, default_conf, caplog) -> None: 'backtesting' ] args = get_args(args) - start(args) + start_backtesting(args) assert log_has( 'Starting freqtrade in Backtesting mode', caplog.record_tuples @@ -847,7 +847,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): '--disable-max-market-positions' ] args = get_args(args) - start(args) + start_backtesting(args) # check the logs, that will contain the backtest result exists = [ 'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', @@ -901,7 +901,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): 'TestStrategy', ] args = get_args(args) - start(args) + start_backtesting(args) # 2 backtests, 4 tables assert backtestmock.call_count == 2 assert gen_table_mock.call_count == 4 diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 9d1789171..9128efa0c 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -11,7 +11,8 @@ from freqtrade import DependencyException from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.optimize.default_hyperopt import DefaultHyperOpts -from freqtrade.optimize.hyperopt import Hyperopt, setup_configuration, start +from freqtrade.optimize.hyperopt import Hyperopt +from freqtrade.optimize import setup_configuration, start_hyperopt from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.state import RunMode from freqtrade.tests.conftest import log_has, log_has_re, patch_exchange @@ -52,7 +53,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca 'hyperopt' ] - config = setup_configuration(get_args(args)) + config = setup_configuration(get_args(args), RunMode.HYPEROPT) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -100,7 +101,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo '--print-all' ] - config = setup_configuration(get_args(args)) + config = setup_configuration(get_args(args), RunMode.HYPEROPT) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -183,7 +184,7 @@ def test_start(mocker, default_conf, caplog) -> None: '--epochs', '5' ] args = get_args(args) - start(args) + start_hyperopt(args) import pprint pprint.pprint(caplog.record_tuples) @@ -214,7 +215,7 @@ def test_start_no_data(mocker, default_conf, caplog) -> None: '--epochs', '5' ] args = get_args(args) - start(args) + start_hyperopt(args) import pprint pprint.pprint(caplog.record_tuples) @@ -239,7 +240,7 @@ def test_start_failure(mocker, default_conf, caplog) -> None: ] args = get_args(args) with pytest.raises(DependencyException): - start(args) + start_hyperopt(args) assert log_has( "Please don't use --strategy for hyperopt.", caplog.record_tuples diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index e4ffc5fae..1292fd24d 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -19,7 +19,7 @@ def test_parse_args_backtesting(mocker) -> None: Test that main() can start backtesting and also ensure we can pass some specific arguments further argument parsing is done in test_arguments.py """ - backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock()) + backtesting_mock = mocker.patch('freqtrade.optimize.start_backtesting', MagicMock()) main(['backtesting']) assert backtesting_mock.call_count == 1 call_args = backtesting_mock.call_args[0][0] @@ -32,7 +32,7 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: - hyperopt_mock = mocker.patch('freqtrade.optimize.hyperopt.start', MagicMock()) + hyperopt_mock = mocker.patch('freqtrade.optimize.start_hyperopt', MagicMock()) main(['hyperopt']) assert hyperopt_mock.call_count == 1 call_args = hyperopt_mock.call_args[0][0] From 104f1212e6134973ff89b00f042552c389499bb4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 20:05:19 +0200 Subject: [PATCH 70/84] Move edge_cli_start to optimize --- freqtrade/arguments.py | 4 ++-- freqtrade/optimize/__init__.py | 16 ++++++++++++++++ freqtrade/optimize/edge_cli.py | 34 ---------------------------------- 3 files changed, 18 insertions(+), 36 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index e79d0c6d4..d6f0063d0 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -340,7 +340,7 @@ class Arguments(object): Builds and attaches all subcommands :return: None """ - from freqtrade.optimize import start_backtesting, start_hyperopt, edge_cli + from freqtrade.optimize import start_backtesting, start_hyperopt, start_edgecli subparsers = self.parser.add_subparsers(dest='subparser') @@ -352,7 +352,7 @@ class Arguments(object): # Add edge subcommand edge_cmd = subparsers.add_parser('edge', help='Edge module.') - edge_cmd.set_defaults(func=edge_cli.start) + edge_cmd.set_defaults(func=start_edgecli) self.optimizer_shared_options(edge_cmd) self.edge_options(edge_cmd) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 34076ee43..cb01950b4 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -97,3 +97,19 @@ def start_hyperopt(args: Namespace) -> None: # TODO: return False here in order to help freqtrade to exit # with non-zero exit code... # Same in Edge and Backtesting start() functions. + + +def start_edgecli(args: Namespace) -> None: + """ + Start Edge script + :param args: Cli args from Arguments() + :return: None + """ + from freqtrade.optimize.edge_cli import EdgeCli + # Initialize configuration + config = setup_configuration(args, RunMode.EDGECLI) + logger.info('Starting freqtrade in Edge mode') + + # Initialize Edge object + edge_cli = EdgeCli(config) + edge_cli.start() diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index d37b930b8..8232c79c9 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -4,16 +4,13 @@ This module contains the edge backtesting interface """ import logging -from argparse import Namespace from typing import Dict, Any from tabulate import tabulate from freqtrade.edge import Edge from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.resolvers import StrategyResolver -from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -77,34 +74,3 @@ class EdgeCli(object): if result: print('') # blank line for readability print(self._generate_edge_table(self.edge._cached_pairs)) - - -def setup_configuration(args: Namespace) -> Dict[str, Any]: - """ - Prepare the configuration for edge backtesting - :param args: Cli args from Arguments() - :return: Configuration - """ - configuration = Configuration(args, RunMode.EDGECLI) - config = configuration.get_config() - - # Ensure we do not use Exchange credentials - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - - return config - - -def start(args: Namespace) -> None: - """ - Start Edge script - :param args: Cli args from Arguments() - :return: None - """ - # Initialize configuration - config = setup_configuration(args) - logger.info('Starting freqtrade in Edge mode') - - # Initialize Edge object - edge_cli = EdgeCli(config) - edge_cli.start() From 8ad30e262578f076c7b9bf8f2399fdbcadc1aa06 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 20:06:18 +0200 Subject: [PATCH 71/84] Adapt tests --- freqtrade/tests/optimize/test_edge_cli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/optimize/test_edge_cli.py b/freqtrade/tests/optimize/test_edge_cli.py index 488d552c8..dc40cc85c 100644 --- a/freqtrade/tests/optimize/test_edge_cli.py +++ b/freqtrade/tests/optimize/test_edge_cli.py @@ -7,7 +7,8 @@ from unittest.mock import MagicMock from freqtrade.arguments import Arguments from freqtrade.edge import PairInfo -from freqtrade.optimize.edge_cli import EdgeCli, setup_configuration, start +from freqtrade.optimize import start_edgecli, setup_configuration +from freqtrade.optimize.edge_cli import EdgeCli from freqtrade.state import RunMode from freqtrade.tests.conftest import log_has, log_has_re, patch_exchange @@ -27,7 +28,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> 'edge' ] - config = setup_configuration(get_args(args)) + config = setup_configuration(get_args(args), RunMode.EDGECLI) assert config['runmode'] == RunMode.EDGECLI assert 'max_open_trades' in config @@ -67,7 +68,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N '--stoplosses=-0.01,-0.10,-0.001' ] - config = setup_configuration(get_args(args)) + config = setup_configuration(get_args(args), RunMode.EDGECLI) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -106,7 +107,7 @@ def test_start(mocker, fee, edge_conf, caplog) -> None: 'edge' ] args = get_args(args) - start(args) + start_edgecli(args) assert log_has( 'Starting freqtrade in Edge mode', caplog.record_tuples From 71447e55aac787dce881adf32da776c747838791 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 20:14:31 +0200 Subject: [PATCH 72/84] Update missing import --- scripts/plot_dataframe.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 7fdc607e0..8a87d971c 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -41,9 +41,10 @@ from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data from freqtrade.exchange import Exchange -from freqtrade.optimize.backtesting import setup_configuration +from freqtrade.optimize import setup_configuration from freqtrade.persistence import Trade from freqtrade.resolvers import StrategyResolver +from freqtrade.state import RunMode logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} @@ -107,7 +108,7 @@ def get_trading_env(args: Namespace): global _CONF # Load the configuration - _CONF.update(setup_configuration(args)) + _CONF.update(setup_configuration(args, RunMode.BACKTEST)) print(_CONF) pairs = args.pairs.split(',') From 201e02e73fbb39da0ee2b9a0687339edc489b49a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 20:25:59 +0200 Subject: [PATCH 73/84] Add test for Timeout - move tests to test_history --- freqtrade/tests/data/test_history.py | 70 +++++++++++++++++++++-- freqtrade/tests/optimize/test_hyperopt.py | 25 +++++++- freqtrade/tests/optimize/test_optimize.py | 65 --------------------- 3 files changed, 89 insertions(+), 71 deletions(-) delete mode 100644 freqtrade/tests/optimize/test_optimize.py diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 0d4210d3a..4d70d4cdd 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -2,24 +2,25 @@ import json import os -from pathlib import Path import uuid +from pathlib import Path from shutil import copyfile import arrow -from pandas import DataFrame import pytest +from pandas import DataFrame from freqtrade import OperationalException from freqtrade.arguments import TimeRange from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, - load_tickerdata_file, - make_testdata_path, + load_tickerdata_file, make_testdata_path, trim_tickerlist) +from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json -from freqtrade.tests.conftest import get_patched_exchange, log_has +from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.tests.conftest import get_patched_exchange, log_has, patch_exchange # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -495,3 +496,62 @@ def test_file_dump_json_tofile() -> None: # Remove the file _clean_test_file(file) + + +def test_get_timeframe(default_conf, mocker) -> None: + patch_exchange(mocker) + strategy = DefaultStrategy(default_conf) + + data = strategy.tickerdata_to_dataframe( + history.load_data( + datadir=None, + ticker_interval='1m', + pairs=['UNITTEST/BTC'] + ) + ) + min_date, max_date = history.get_timeframe(data) + assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' + assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' + + +def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: + patch_exchange(mocker) + strategy = DefaultStrategy(default_conf) + + data = strategy.tickerdata_to_dataframe( + history.load_data( + datadir=None, + ticker_interval='1m', + pairs=['UNITTEST/BTC'], + fill_up_missing=False + ) + ) + min_date, max_date = history.get_timeframe(data) + caplog.clear() + assert history.validate_backtest_data(data, min_date, max_date, + timeframe_to_minutes('1m')) + assert len(caplog.record_tuples) == 1 + assert log_has( + "UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values", + caplog.record_tuples) + + +def test_validate_backtest_data(default_conf, mocker, caplog) -> None: + patch_exchange(mocker) + strategy = DefaultStrategy(default_conf) + + timerange = TimeRange('index', 'index', 200, 250) + data = strategy.tickerdata_to_dataframe( + history.load_data( + datadir=None, + ticker_interval='5m', + pairs=['UNITTEST/BTC'], + timerange=timerange + ) + ) + + min_date, max_date = history.get_timeframe(data) + caplog.clear() + assert not history.validate_backtest_data(data, min_date, max_date, + timeframe_to_minutes('5m')) + assert len(caplog.record_tuples) == 0 diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 9128efa0c..b41f8ac36 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -3,6 +3,7 @@ import json import os from datetime import datetime from unittest.mock import MagicMock +from filelock import Timeout import pandas as pd import pytest @@ -11,7 +12,7 @@ from freqtrade import DependencyException from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.optimize.default_hyperopt import DefaultHyperOpts -from freqtrade.optimize.hyperopt import Hyperopt +from freqtrade.optimize.hyperopt import Hyperopt, HYPEROPT_LOCKFILE from freqtrade.optimize import setup_configuration, start_hyperopt from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.state import RunMode @@ -247,6 +248,28 @@ def test_start_failure(mocker, default_conf, caplog) -> None: ) +def test_start_filelock(mocker, default_conf, caplog) -> None: + start_mock = MagicMock(side_effect=Timeout(HYPEROPT_LOCKFILE)) + mocker.patch( + 'freqtrade.configuration.Configuration._load_config_file', + lambda *args, **kwargs: default_conf + ) + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) + patch_exchange(mocker) + + args = [ + '--config', 'config.json', + 'hyperopt', + '--epochs', '5' + ] + args = get_args(args) + start_hyperopt(args) + assert log_has( + "Another running instance of freqtrade Hyperopt detected.", + caplog.record_tuples + ) + + def test_loss_calculation_prefer_correct_trade_count(hyperopt) -> None: correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py deleted file mode 100644 index 401592b53..000000000 --- a/freqtrade/tests/optimize/test_optimize.py +++ /dev/null @@ -1,65 +0,0 @@ -# pragma pylint: disable=missing-docstring, protected-access, C0103 -from freqtrade.arguments import TimeRange -from freqtrade.data import history -from freqtrade.exchange import timeframe_to_minutes -from freqtrade.strategy.default_strategy import DefaultStrategy -from freqtrade.tests.conftest import log_has, patch_exchange - - -def test_get_timeframe(default_conf, mocker) -> None: - patch_exchange(mocker) - strategy = DefaultStrategy(default_conf) - - data = strategy.tickerdata_to_dataframe( - history.load_data( - datadir=None, - ticker_interval='1m', - pairs=['UNITTEST/BTC'] - ) - ) - min_date, max_date = history.get_timeframe(data) - assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' - assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' - - -def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: - patch_exchange(mocker) - strategy = DefaultStrategy(default_conf) - - data = strategy.tickerdata_to_dataframe( - history.load_data( - datadir=None, - ticker_interval='1m', - pairs=['UNITTEST/BTC'], - fill_up_missing=False - ) - ) - min_date, max_date = history.get_timeframe(data) - caplog.clear() - assert history.validate_backtest_data(data, min_date, max_date, - timeframe_to_minutes('1m')) - assert len(caplog.record_tuples) == 1 - assert log_has( - "UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values", - caplog.record_tuples) - - -def test_validate_backtest_data(default_conf, mocker, caplog) -> None: - patch_exchange(mocker) - strategy = DefaultStrategy(default_conf) - - timerange = TimeRange('index', 'index', 200, 250) - data = strategy.tickerdata_to_dataframe( - history.load_data( - datadir=None, - ticker_interval='5m', - pairs=['UNITTEST/BTC'], - timerange=timerange - ) - ) - - min_date, max_date = history.get_timeframe(data) - caplog.clear() - assert not history.validate_backtest_data(data, min_date, max_date, - timeframe_to_minutes('5m')) - assert len(caplog.record_tuples) == 0 From 0e228acbfb5f2605c099696e37915e8d8a8fe005 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 25 May 2019 22:42:17 +0300 Subject: [PATCH 74/84] minor: exchange debug logging humanized --- freqtrade/exchange/exchange.py | 19 +++++++++++++++---- freqtrade/tests/exchange/test_exchange.py | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 66857a7a5..72a0efb1f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -510,7 +510,11 @@ class Exchange(object): _LIMIT = 500 one_call = timeframe_to_msecs(ticker_interval) * _LIMIT - logger.debug("one_call: %s msecs", one_call) + logger.debug( + "one_call: %s msecs (%s)", + one_call, + arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) + ) input_coroutines = [self._async_get_candle_history( pair, ticker_interval, since) for since in range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] @@ -541,7 +545,10 @@ class Exchange(object): or self._now_is_time_to_refresh(pair, ticker_interval)): input_coroutines.append(self._async_get_candle_history(pair, ticker_interval)) else: - logger.debug("Using cached ohlcv data for %s, %s ...", pair, ticker_interval) + logger.debug( + "Using cached ohlcv data for pair %s, interval %s ...", + pair, ticker_interval + ) tickers = asyncio.get_event_loop().run_until_complete( asyncio.gather(*input_coroutines, return_exceptions=True)) @@ -578,7 +585,11 @@ class Exchange(object): """ try: # fetch ohlcv asynchronously - logger.debug("fetching %s, %s since %s ...", pair, ticker_interval, since_ms) + s = '(' + arrow.get(since_ms // 1000).isoformat() + ') ' if since_ms is not None else '' + logger.debug( + "Fetching pair %s, interval %s, since %s %s...", + pair, ticker_interval, since_ms, s + ) data = await self._api_async.fetch_ohlcv(pair, timeframe=ticker_interval, since=since_ms) @@ -593,7 +604,7 @@ class Exchange(object): except IndexError: logger.exception("Error loading %s. Result was %s.", pair, data) return pair, ticker_interval, [] - logger.debug("done fetching %s, %s ...", pair, ticker_interval) + logger.debug("Done fetching pair %s, interval %s ...", pair, ticker_interval) return pair, ticker_interval, data except ccxt.NotSupported as e: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 924ed538f..fda9c8241 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1016,7 +1016,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]) assert exchange._api_async.fetch_ohlcv.call_count == 2 - assert log_has(f"Using cached ohlcv data for {pairs[0][0]}, {pairs[0][1]} ...", + assert log_has(f"Using cached ohlcv data for pair {pairs[0][0]}, interval {pairs[0][1]} ...", caplog.record_tuples) From e335e6c480350e8b7c91939cfae99e5ec9c91170 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 May 2019 13:40:07 +0200 Subject: [PATCH 75/84] Fix some wordings --- docs/docker.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/docker.md b/docs/docker.md index 767cabf01..939ab3f7d 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -8,7 +8,7 @@ Start by downloading and installing Docker CE for your platform: * [Windows](https://docs.docker.com/docker-for-windows/install/) * [Linux](https://docs.docker.com/install/) -Once you have Docker installed, simply create the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. +Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. ## Download the official FreqTrade docker image @@ -74,7 +74,7 @@ touch tradesv3.dryrun.sqlite Best start by pulling the official docker image from dockerhub as explained [here](#download-the-official-docker-image) to speed up building. -To add additional libaries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image. +To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image. ```bash docker build -t freqtrade -f Dockerfile.technical . @@ -164,7 +164,7 @@ docker run -d \ To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` !!! Note - All available command line arguments can be added to the end of the `docker run` command. + All available bot command line parameters can be added to the end of the `docker run` command. ### Monitor your Docker instance @@ -201,4 +201,4 @@ docker run -d \ Head over to the [Backtesting Documentation](backtesting.md) for more details. !!! Note - Additional parameters can be appended after the image name (`freqtrade` in the above example). + Additional bot command line parameters can be appended after the image name (`freqtrade` in the above example). From 1988662607842601b7d1813c074a07bcf9b48a13 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 May 2019 20:19:06 +0200 Subject: [PATCH 76/84] Update plot-script to work with exported trades --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 7fdc607e0..cc54bfff2 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -74,7 +74,7 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram file = Path(args.exportfilename) if file.exists(): - load_backtest_data(file) + trades = load_backtest_data(file) else: trades = pd.DataFrame([], columns=BT_DATA_COLUMNS) From 196a1bcc267696379efb539136a37f29399c92d9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 May 2019 15:29:06 +0000 Subject: [PATCH 77/84] Update ccxt from 1.18.551 to 1.18.578 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 3f755b8c0..1defdf19e 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.551 +ccxt==1.18.578 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.2 From bfb6dc4a8ea4eb031ef554c0b299fc4b9d380327 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 May 2019 15:29:07 +0000 Subject: [PATCH 78/84] Update cachetools from 3.1.0 to 3.1.1 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 1defdf19e..e885e3e0f 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -4,7 +4,7 @@ ccxt==1.18.578 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.2 -cachetools==3.1.0 +cachetools==3.1.1 requests==2.22.0 urllib3==1.24.2 # pyup: ignore wrapt==1.11.1 From 09e037c96ec2a0b1edaefef1f3641f0b7e8b6bc5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 May 2019 15:29:09 +0000 Subject: [PATCH 79/84] Update scikit-learn from 0.21.1 to 0.21.2 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index e885e3e0f..b149abacd 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -8,7 +8,7 @@ cachetools==3.1.1 requests==2.22.0 urllib3==1.24.2 # pyup: ignore wrapt==1.11.1 -scikit-learn==0.21.1 +scikit-learn==0.21.2 joblib==0.13.2 jsonschema==3.0.1 TA-Lib==0.4.17 From f7766d305b73d01ee90d7f45bfca5bbee30dfefd Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 27 May 2019 19:42:12 +0200 Subject: [PATCH 80/84] Improve plotting documentation --- docs/plotting.md | 55 +++++++++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index 60c642ab3..20183ab9c 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -1,63 +1,79 @@ # Plotting + This page explains how to plot prices, indicator, profits. ## Installation Plotting scripts use Plotly library. Install/upgrade it with: -``` -pip install --upgrade plotly +``` bash +pip install -U -r requirements-plot.txt ``` At least version 2.3.0 is required. ## Plot price and indicators + Usage for the price plotter: -``` -script/plot_dataframe.py [-h] [-p pairs] [--live] +``` bash +python3 script/plot_dataframe.py [-h] [-p pairs] [--live] ``` Example -``` -python scripts/plot_dataframe.py -p BTC/ETH + +``` bash +python3 scripts/plot_dataframe.py -p BTC/ETH ``` The `-p` pairs argument, can be used to specify pairs you would like to plot. -**Advanced use** +### Advanced use To plot multiple pairs, separate them with a comma: -``` -python scripts/plot_dataframe.py -p BTC/ETH,XRP/ETH + +``` bash +python3 scripts/plot_dataframe.py -p BTC/ETH,XRP/ETH ``` To plot the current live price use the `--live` flag: -``` -python scripts/plot_dataframe.py -p BTC/ETH --live + +``` bash +python3 scripts/plot_dataframe.py -p BTC/ETH --live ``` To plot a timerange (to zoom in): + +``` bash +python3 scripts/plot_dataframe.py -p BTC/ETH --timerange=100-200 ``` -python scripts/plot_dataframe.py -p BTC/ETH --timerange=100-200 -``` + Timerange doesn't work with live data. To plot trades stored in a database use `--db-url` argument: + +``` bash +python3 scripts/plot_dataframe.py --db-url sqlite:///tradesv3.dry_run.sqlite -p BTC/ETH ``` -python scripts/plot_dataframe.py --db-url sqlite:///tradesv3.dry_run.sqlite -p BTC/ETH + +To polt trades from a backtesting result, use `--export-filename ` + +``` bash +python3 scripts/plot_dataframe.py --export-filename user_data/backtest_data/backtest-result.json -p BTC/ETH ``` To plot a test strategy the strategy should have first be backtested. The results may then be plotted with the -s argument: -``` -python scripts/plot_dataframe.py -s Strategy_Name -p BTC/ETH --datadir user_data/data// + +``` bash +python3 scripts/plot_dataframe.py -s Strategy_Name -p BTC/ETH --datadir user_data/data// ``` ## Plot profit The profit plotter show a picture with three plots: + 1) Average closing price for all pairs 2) The summarized profit made by backtesting. Note that this is not the real-world profit, but @@ -76,13 +92,14 @@ that makes profit spikes. Usage for the profit plotter: -``` -script/plot_profit.py [-h] [-p pair] [--datadir directory] [--ticker_interval num] +``` bash +python3 script/plot_profit.py [-h] [-p pair] [--datadir directory] [--ticker_interval num] ``` The `-p` pair argument, can be used to plot a single pair Example -``` + +``` bash python3 scripts/plot_profit.py --datadir ../freqtrade/freqtrade/tests/testdata-20171221/ -p LTC/BTC ``` From 8b028068bbe22b3aa0988c830aeb85316ee844e0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 May 2019 07:06:26 +0200 Subject: [PATCH 81/84] Fix typos, add section for custom indicators --- docs/plotting.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index 20183ab9c..eb72b0502 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -1,6 +1,6 @@ # Plotting -This page explains how to plot prices, indicator, profits. +This page explains how to plot prices, indicators and profits. ## Installation @@ -10,8 +10,6 @@ Plotting scripts use Plotly library. Install/upgrade it with: pip install -U -r requirements-plot.txt ``` -At least version 2.3.0 is required. - ## Plot price and indicators Usage for the price plotter: @@ -26,8 +24,14 @@ Example python3 scripts/plot_dataframe.py -p BTC/ETH ``` -The `-p` pairs argument, can be used to specify -pairs you would like to plot. +The `-p` pairs argument can be used to specify pairs you would like to plot. + +Specify custom indicators. +Use `--indicators1` for the main plot and `--indicators2` for the subplot below (if values are in a different range than prices). + +``` bash +python3 scripts/plot_dataframe.py -p BTC/ETH --indicators1 sma,ema --indicators2 macd +``` ### Advanced use @@ -57,13 +61,13 @@ To plot trades stored in a database use `--db-url` argument: python3 scripts/plot_dataframe.py --db-url sqlite:///tradesv3.dry_run.sqlite -p BTC/ETH ``` -To polt trades from a backtesting result, use `--export-filename ` +To plot trades from a backtesting result, use `--export-filename ` ``` bash python3 scripts/plot_dataframe.py --export-filename user_data/backtest_data/backtest-result.json -p BTC/ETH ``` -To plot a test strategy the strategy should have first be backtested. +To plot a custom strategy the strategy should have first be backtested. The results may then be plotted with the -s argument: ``` bash @@ -72,7 +76,7 @@ python3 scripts/plot_dataframe.py -s Strategy_Name -p BTC/ETH --datadir user_dat ## Plot profit -The profit plotter show a picture with three plots: +The profit plotter shows a picture with three plots: 1) Average closing price for all pairs 2) The summarized profit made by backtesting. From 89f44c10a1d17e673a105b062d98857e0ee948ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 May 2019 19:20:41 +0200 Subject: [PATCH 82/84] Fix grammar error --- docs/plotting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plotting.md b/docs/plotting.md index eb72b0502..6dc3d13b1 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -87,7 +87,7 @@ The profit plotter shows a picture with three plots: The first graph is good to get a grip of how the overall market progresses. -The second graph will show how you algorithm works or doesnt. +The second graph will show how your algorithm works or doesn't. Perhaps you want an algorithm that steadily makes small profits, or one that acts less seldom, but makes big swings. From 55bdd2643954a38cfe6ebb144edfc04b84339894 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 May 2019 19:25:01 +0200 Subject: [PATCH 83/84] Edgecli -> Edge for Runmode and start_edge() --- freqtrade/arguments.py | 4 ++-- freqtrade/optimize/__init__.py | 4 ++-- freqtrade/state.py | 4 ++-- freqtrade/tests/optimize/test_edge_cli.py | 12 ++++++------ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index d6f0063d0..ddc0dc489 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -340,7 +340,7 @@ class Arguments(object): Builds and attaches all subcommands :return: None """ - from freqtrade.optimize import start_backtesting, start_hyperopt, start_edgecli + from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge subparsers = self.parser.add_subparsers(dest='subparser') @@ -352,7 +352,7 @@ class Arguments(object): # Add edge subcommand edge_cmd = subparsers.add_parser('edge', help='Edge module.') - edge_cmd.set_defaults(func=start_edgecli) + edge_cmd.set_defaults(func=start_edge) self.optimizer_shared_options(edge_cmd) self.edge_options(edge_cmd) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index cb01950b4..475aaa82f 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -99,7 +99,7 @@ def start_hyperopt(args: Namespace) -> None: # Same in Edge and Backtesting start() functions. -def start_edgecli(args: Namespace) -> None: +def start_edge(args: Namespace) -> None: """ Start Edge script :param args: Cli args from Arguments() @@ -107,7 +107,7 @@ def start_edgecli(args: Namespace) -> None: """ from freqtrade.optimize.edge_cli import EdgeCli # Initialize configuration - config = setup_configuration(args, RunMode.EDGECLI) + config = setup_configuration(args, RunMode.EDGE) logger.info('Starting freqtrade in Edge mode') # Initialize Edge object diff --git a/freqtrade/state.py b/freqtrade/state.py index b69c26cb5..ce2683a77 100644 --- a/freqtrade/state.py +++ b/freqtrade/state.py @@ -18,11 +18,11 @@ class State(Enum): class RunMode(Enum): """ Bot running mode (backtest, hyperopt, ...) - can be "live", "dry-run", "backtest", "edgecli", "hyperopt". + can be "live", "dry-run", "backtest", "edge", "hyperopt". """ LIVE = "live" DRY_RUN = "dry_run" BACKTEST = "backtest" - EDGECLI = "edgecli" + EDGE = "edge" HYPEROPT = "hyperopt" OTHER = "other" # Used for plotting scripts and test diff --git a/freqtrade/tests/optimize/test_edge_cli.py b/freqtrade/tests/optimize/test_edge_cli.py index dc40cc85c..5d16b0f2d 100644 --- a/freqtrade/tests/optimize/test_edge_cli.py +++ b/freqtrade/tests/optimize/test_edge_cli.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock from freqtrade.arguments import Arguments from freqtrade.edge import PairInfo -from freqtrade.optimize import start_edgecli, setup_configuration +from freqtrade.optimize import start_edge, setup_configuration from freqtrade.optimize.edge_cli import EdgeCli from freqtrade.state import RunMode from freqtrade.tests.conftest import log_has, log_has_re, patch_exchange @@ -28,8 +28,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> 'edge' ] - config = setup_configuration(get_args(args), RunMode.EDGECLI) - assert config['runmode'] == RunMode.EDGECLI + config = setup_configuration(get_args(args), RunMode.EDGE) + assert config['runmode'] == RunMode.EDGE assert 'max_open_trades' in config assert 'stake_currency' in config @@ -68,14 +68,14 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N '--stoplosses=-0.01,-0.10,-0.001' ] - config = setup_configuration(get_args(args), RunMode.EDGECLI) + config = setup_configuration(get_args(args), RunMode.EDGE) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config assert 'exchange' in config assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config - assert config['runmode'] == RunMode.EDGECLI + assert config['runmode'] == RunMode.EDGE assert log_has( 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples @@ -107,7 +107,7 @@ def test_start(mocker, fee, edge_conf, caplog) -> None: 'edge' ] args = get_args(args) - start_edgecli(args) + start_edge(args) assert log_has( 'Starting freqtrade in Edge mode', caplog.record_tuples From ea83b2b1d0542adc299513c2876963a433cea987 Mon Sep 17 00:00:00 2001 From: Misagh Date: Wed, 29 May 2019 14:17:09 +0200 Subject: [PATCH 84/84] legacy code removed. --- user_data/strategies/test_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 3cb78842f..66a5f8c09 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -51,7 +51,7 @@ class TestStrategy(IStrategy): ticker_interval = '5m' # run "populate_indicators" only for new candle - ta_on_candle = False + process_only_new_candles = False # Experimental settings (configuration will overide these if set) use_sell_signal = False