From c62356438a60ec40c7e724fa4f0e8dff898751f4 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 11 Feb 2018 12:48:06 +0200 Subject: [PATCH 1/6] loop over arrays instead of dataframes --- freqtrade/optimize/backtesting.py | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e3fb9b946..5a0770953 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -67,30 +67,29 @@ def generate_text_table( return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) -def get_sell_trade_entry(pair, row, buy_subset, ticker, trade_count_lock, args): +def get_sell_trade_entry(pair, row, partial_ticker, trade_count_lock, args): stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) trade = Trade(open_rate=row.close, - open_date=row.Index, + open_date=row.date, stake_amount=stake_amount, amount=stake_amount / row.open, fee=exchange.get_fee() ) # calculate win/lose forwards from buy point - sell_subset = ticker[ticker.index > row.Index][['close', 'sell', 'buy']] - for row2 in sell_subset.itertuples(index=True): + for row2 in partial_ticker: if max_open_trades > 0: # Increase trade_count_lock for every iteration - trade_count_lock[row2.Index] = trade_count_lock.get(row2.Index, 0) + 1 + trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1 buy_signal = row2.buy - if(should_sell(trade, row2.close, row2.Index, buy_signal, row2.sell)): + if should_sell(trade, row2.close, row2.date, buy_signal, row2.sell): return row2, (pair, trade.calc_profit_percent(rate=row2.close), trade.calc_profit(rate=row2.close), - (row2.Index - row.Index).seconds // 60 - ), row2.Index + (row2.date - row.date).seconds // 60 + ), row2.date return None @@ -107,6 +106,7 @@ def backtest(args) -> DataFrame: stoploss: use stoploss :return: DataFrame """ + headers = ['date', 'buy', 'open', 'close', 'sell'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', True) @@ -116,29 +116,29 @@ def backtest(args) -> DataFrame: trade_count_lock: dict = {} exchange._API = Bittrex({'key': '', 'secret': ''}) for pair, pair_data in processed.items(): - pair_data['buy'], pair_data['sell'] = 0, 0 - ticker = populate_sell_trend(populate_buy_trend(pair_data)) - if 'date' in ticker: - ticker.set_index('date', inplace=True) - # for each buy point + pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run + + ticker_data = populate_sell_trend(populate_buy_trend(pair_data))[headers] + ticker = [x for x in ticker_data.itertuples()] + lock_pair_until = None - headers = ['buy', 'open', 'close', 'sell'] - buy_subset = ticker[(ticker.buy == 1) & (ticker.sell == 0)][headers] - for row in buy_subset.itertuples(index=True): + for index, row in enumerate(ticker): + if row.buy == 0 or row.sell == 1: + continue # skip rows where no buy signal or that would immediately sell off + if realistic: - if lock_pair_until is not None and row.Index <= lock_pair_until: + if lock_pair_until is not None and row.date <= lock_pair_until: continue if max_open_trades > 0: # Check if max_open_trades has already been reached for the given date - if not trade_count_lock.get(row.Index, 0) < max_open_trades: + if not trade_count_lock.get(row.date, 0) < max_open_trades: continue if max_open_trades > 0: # Increase lock - trade_count_lock[row.Index] = trade_count_lock.get(row.Index, 0) + 1 + trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - ret = get_sell_trade_entry(pair, row, buy_subset, ticker, - trade_count_lock, args) + ret = get_sell_trade_entry(pair, row, ticker[index+1:], trade_count_lock, args) if ret: row2, trade_entry, next_date = ret lock_pair_until = next_date @@ -148,9 +148,9 @@ def backtest(args) -> DataFrame: # record a tuple of pair, current_profit_percent, # entry-date, duration records.append((pair, trade_entry[1], - row.Index.strftime('%s'), - row2.Index.strftime('%s'), - row.Index, trade_entry[3])) + row.date.strftime('%s'), + row2.date.strftime('%s'), + row.date, trade_entry[3])) # For now export inside backtest(), maybe change so that backtest() # returns a tuple like: (dataframe, records, logs, etc) if record and record.find('trades') >= 0: From dc105d5eae6ade13bf4718635f8eb90c6d95f590 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 11 Feb 2018 14:24:19 +0200 Subject: [PATCH 2/6] better names for row variables --- freqtrade/optimize/backtesting.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5a0770953..2541e570a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -67,29 +67,29 @@ def generate_text_table( return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) -def get_sell_trade_entry(pair, row, partial_ticker, trade_count_lock, args): +def get_sell_trade_entry(pair, buy_row, partial_ticker, trade_count_lock, args): stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) - trade = Trade(open_rate=row.close, - open_date=row.date, + trade = Trade(open_rate=buy_row.close, + open_date=buy_row.date, stake_amount=stake_amount, - amount=stake_amount / row.open, + amount=stake_amount / buy_row.open, fee=exchange.get_fee() ) # calculate win/lose forwards from buy point - for row2 in partial_ticker: + for sell_row in partial_ticker: if max_open_trades > 0: # Increase trade_count_lock for every iteration - trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1 + trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 - buy_signal = row2.buy - if should_sell(trade, row2.close, row2.date, buy_signal, row2.sell): - return row2, (pair, - trade.calc_profit_percent(rate=row2.close), - trade.calc_profit(rate=row2.close), - (row2.date - row.date).seconds // 60 - ), row2.date + buy_signal = sell_row.buy + if should_sell(trade, sell_row.close, sell_row.date, buy_signal, sell_row.sell): + return sell_row, (pair, + trade.calc_profit_percent(rate=sell_row.close), + trade.calc_profit(rate=sell_row.close), + (sell_row.date - buy_row.date).seconds // 60 + ), sell_row.date return None From 2dd2f31431991619a94dce93b82586260499ab51 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 11 Feb 2018 14:31:37 +0200 Subject: [PATCH 3/6] remove repeated condition --- freqtrade/optimize/backtesting.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2541e570a..ce5fc098b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -133,9 +133,6 @@ def backtest(args) -> DataFrame: # Check if max_open_trades has already been reached for the given date if not trade_count_lock.get(row.date, 0) < max_open_trades: continue - - if max_open_trades > 0: - # Increase lock trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 ret = get_sell_trade_entry(pair, row, ticker[index+1:], trade_count_lock, args) From 5190cd507e3964a0f8df4f3d3dccb98f79a29df1 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 11 Feb 2018 14:37:12 +0200 Subject: [PATCH 4/6] start with simpler condition --- freqtrade/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 48dfb3818..a9ca230d4 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -304,10 +304,10 @@ def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) - time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 for duration_string, threshold in strategy.minimal_roi.items(): duration = float(duration_string) - if time_diff > duration and current_profit > threshold: - return True if time_diff < duration: return False + if time_diff > duration and current_profit > threshold: + return True return False From 2ce03ab1b53bf64e48a2e46e152548590adee8e6 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 11 Feb 2018 15:02:42 +0200 Subject: [PATCH 5/6] make Strategy store roi and stoploss values as numbers to avoid later casting --- freqtrade/main.py | 5 ++--- freqtrade/optimize/hyperopt.py | 10 +++++----- freqtrade/strategy/strategy.py | 6 +++--- freqtrade/tests/optimize/test_hyperopt.py | 2 +- freqtrade/tests/strategy/test_strategy.py | 8 ++++---- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index a9ca230d4..bb218855d 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -296,14 +296,13 @@ def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) - strategy = Strategy() current_profit = trade.calc_profit_percent(current_rate) - if strategy.stoploss is not None and current_profit < float(strategy.stoploss): + if strategy.stoploss is not None and current_profit < strategy.stoploss: logger.debug('Stop loss hit.') return True # Check if time matches and current rate is above threshold time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 - for duration_string, threshold in strategy.minimal_roi.items(): - duration = float(duration_string) + for duration, threshold in strategy.minimal_roi.items(): if time_diff < duration: return False if time_diff > duration and current_profit > threshold: diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 12c061b4f..cdb1c3a6f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -225,12 +225,12 @@ def calculate_loss(total_profit: float, trade_count: int, trade_duration: float) return trade_loss + profit_loss + duration_loss -def generate_roi_table(params) -> Dict[str, float]: +def generate_roi_table(params) -> Dict[int, float]: roi_table = {} - roi_table["0"] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[str(params['roi_t3'])] = params['roi_p1'] + params['roi_p2'] - roi_table[str(params['roi_t3'] + params['roi_t2'])] = params['roi_p1'] - roi_table[str(params['roi_t3'] + params['roi_t2'] + params['roi_t1'])] = 0 + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 return roi_table diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index 9ef5a2071..27f334d5c 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -71,11 +71,11 @@ class Strategy(object): # Minimal ROI designed for the strategy self.minimal_roi = OrderedDict(sorted( - self.custom_strategy.minimal_roi.items(), - key=lambda tuple: float(tuple[0]))) # sort after converting to number + {int(key): value for (key, value) in self.custom_strategy.minimal_roi.items()}.items(), + key=lambda tuple: tuple[0])) # sort after converting to number # Optimal stoploss designed for the strategy - self.stoploss = self.custom_strategy.stoploss + self.stoploss = float(self.custom_strategy.stoploss) self.ticker_interval = self.custom_strategy.ticker_interval diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 93cb6ba8b..365869275 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -255,7 +255,7 @@ def test_roi_table_generation(): 'roi_p2': 2, 'roi_p3': 3, } - assert generate_roi_table(params) == {'0': 6, '15': 3, '25': 1, '30': 0} + assert generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} # test log_trials_result diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 890718d60..b0ce88e98 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -57,7 +57,7 @@ def test_strategy(result): strategy.init({'strategy': 'default_strategy'}) assert hasattr(strategy.custom_strategy, 'minimal_roi') - assert strategy.minimal_roi['0'] == 0.04 + assert strategy.minimal_roi[0] == 0.04 assert hasattr(strategy.custom_strategy, 'stoploss') assert strategy.stoploss == -0.10 @@ -86,7 +86,7 @@ def test_strategy_override_minimal_roi(caplog): strategy.init(config) assert hasattr(strategy.custom_strategy, 'minimal_roi') - assert strategy.minimal_roi['0'] == 0.5 + assert strategy.minimal_roi[0] == 0.5 assert ('freqtrade.strategy.strategy', logging.INFO, 'Override strategy \'minimal_roi\' with value in config file.' @@ -142,8 +142,8 @@ def test_strategy_singleton(): strategy1.init({'strategy': 'default_strategy'}) assert hasattr(strategy1.custom_strategy, 'minimal_roi') - assert strategy1.minimal_roi['0'] == 0.04 + assert strategy1.minimal_roi[0] == 0.04 strategy2 = Strategy() assert hasattr(strategy2.custom_strategy, 'minimal_roi') - assert strategy2.minimal_roi['0'] == 0.04 + assert strategy2.minimal_roi[0] == 0.04 From 9bcdc8e14b4b3b4ebc9a4ff47a46f1f00c9643c0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 11 Feb 2018 15:03:23 +0200 Subject: [PATCH 6/6] remove unnecessary condition --- freqtrade/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index bb218855d..52cccbbf1 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -303,9 +303,9 @@ def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) - # Check if time matches and current rate is above threshold time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 for duration, threshold in strategy.minimal_roi.items(): - if time_diff < duration: + if time_diff <= duration: return False - if time_diff > duration and current_profit > threshold: + if current_profit > threshold: return True return False