Merge pull request #111 from gcarq/memoryfix-hyperopt

Memory fix hyperopt

freqtrade/analyze.py

@@ -66,14 +66,13 @@ def populate_buy_trend(dataframe: DataFrame) -> DataFrame:
     :param dataframe: DataFrame
     :return: DataFrame with buy column
     """
-    dataframe.ix[
+    dataframe.loc[
         (dataframe['close'] < dataframe['sma']) &
         (dataframe['tema'] <= dataframe['blower']) &
         (dataframe['mfi'] < 25) &
         (dataframe['fastd'] < 25) &
         (dataframe['adx'] > 30),
         'buy'] = 1
-    dataframe.ix[dataframe['buy'] == 1, 'buy_price'] = dataframe['close']
 
     return dataframe
 
@@ -83,10 +82,9 @@ def populate_sell_trend(dataframe: DataFrame) -> DataFrame:
     :param dataframe: DataFrame
     :return: DataFrame with buy column
     """
-    dataframe.ix[
+    dataframe.loc[
         (crossed_above(dataframe['rsi'], 70)),
         'sell'] = 1
-    dataframe.ix[dataframe['sell'] == 1, 'sell_price'] = dataframe['close']
 
     return dataframe
 
@@ -106,6 +104,9 @@ def analyze_ticker(pair: str) -> DataFrame:
     dataframe = populate_indicators(dataframe)
     dataframe = populate_buy_trend(dataframe)
     dataframe = populate_sell_trend(dataframe)
+    # TODO: buy_price and sell_price are only used by the plotter, should probably be moved there
+    dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close']
+    dataframe.loc[dataframe['sell'] == 1, 'sell_price'] = dataframe['close']
     return dataframe
 
 

freqtrade/main.py

@@ -163,9 +163,9 @@ def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) -
         logger.debug('Stop loss hit.')
         return True
 
-    for duration, threshold in sorted(_CONF['minimal_roi'].items()):
     # Check if time matches and current rate is above threshold
     time_diff = (current_time - trade.open_date).total_seconds() / 60
+    for duration, threshold in sorted(_CONF['minimal_roi'].items()):
         if time_diff > float(duration) and current_profit > threshold:
             return True
 

freqtrade/tests/test_analyze.py

@@ -5,7 +5,7 @@ import pytest
 from pandas import DataFrame
 
 from freqtrade.analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators, \
-    get_signal, SignalType
+    get_signal, SignalType, populate_sell_trend
 
 
 @pytest.fixture
@@ -26,7 +26,11 @@ def test_dataframe_correct_length(result):
 def test_populates_buy_trend(result):
     dataframe = populate_buy_trend(populate_indicators(result))
     assert 'buy' in dataframe.columns
-    assert 'buy_price' in dataframe.columns
+
+
+def test_populates_buy_trend(result):
+    dataframe = populate_sell_trend(populate_indicators(result))
+    assert 'sell' in dataframe.columns
 
 
 def test_returns_latest_buy_signal(mocker):

freqtrade/tests/test_backtesting.py

@@ -1,4 +1,5 @@
 # pragma pylint: disable=missing-docstring
+from typing import Dict
 import logging
 import os
 
@@ -7,7 +8,8 @@ import arrow
 from pandas import DataFrame
 
 from freqtrade import exchange
-from freqtrade.analyze import analyze_ticker
+from freqtrade.analyze import parse_ticker_dataframe, populate_indicators, \
+    populate_buy_trend, populate_sell_trend
 from freqtrade.exchange import Bittrex
 from freqtrade.main import min_roi_reached
 from freqtrade.persistence import Trade
@@ -25,15 +27,22 @@ def print_pair_results(pair, results):
     print(format_results(results[results.currency == pair]))
 
 
-def backtest(backtest_conf, backdata, mocker):
+def preprocess(backdata) -> Dict[str, DataFrame]:
+    processed = {}
+    for pair, pair_data in backdata.items():
+        processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data))
+    return processed
+
+
+def backtest(backtest_conf, processed, mocker):
     trades = []
     exchange._API = Bittrex({'key': '', 'secret': ''})
-    mocked_history = mocker.patch('freqtrade.analyze.get_ticker_history')
     mocker.patch.dict('freqtrade.main._CONF', backtest_conf)
     mocker.patch('arrow.utcnow', return_value=arrow.get('2017-08-20T14:50:00'))
-    for pair, pair_data in backdata.items():
+    for pair, pair_data in processed.items():
-        mocked_history.return_value = pair_data
+        pair_data['buy'] = 0
-        ticker = analyze_ticker(pair)[['close', 'date', 'buy', 'sell']].copy()
+        pair_data['sell'] = 0
+        ticker = populate_sell_trend(populate_buy_trend(pair_data))
         # for each buy point
         for row in ticker[ticker.buy == 1].itertuples(index=True):
             trade = Trade(
@@ -50,13 +59,12 @@ def backtest(backtest_conf, backdata, mocker):
                     trades.append((pair, current_profit, row2.Index - row.Index))
                     break
     labels = ['currency', 'profit', 'duration']
-    results = DataFrame.from_records(trades, columns=labels)
+    return DataFrame.from_records(trades, columns=labels)
-    return results
 
 
 @pytest.mark.skipif(not os.environ.get('BACKTEST', False), reason="BACKTEST not set")
 def test_backtest(backtest_conf, backdata, mocker, report=True):
-    results = backtest(backtest_conf, backdata, mocker)
+    results = backtest(backtest_conf, preprocess(backdata), mocker)
 
     print('====================== BACKTESTING REPORT ================================')
     for pair in backdata:

freqtrade/tests/test_hyperopt.py

@@ -9,7 +9,7 @@ import pytest
 from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
 from pandas import DataFrame
 
-from freqtrade.tests.test_backtesting import backtest, format_results
+from freqtrade.tests.test_backtesting import backtest, format_results, preprocess
 from freqtrade.vendor.qtpylib.indicators import crossed_above
 
 logging.disable(logging.DEBUG)  # disable debug logs that slow backtesting a lot
@@ -59,7 +59,6 @@ def buy_strategy_generator(params):
         dataframe.loc[
             reduce(lambda x, y: x & y, conditions),
             'buy'] = 1
-        dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close']
 
         return dataframe
     return populate_buy_trend
@@ -67,12 +66,13 @@ def buy_strategy_generator(params):
 
 @pytest.mark.skipif(not os.environ.get('BACKTEST', False), reason="BACKTEST not set")
 def test_hyperopt(backtest_conf, backdata, mocker):
-    mocked_buy_trend = mocker.patch('freqtrade.analyze.populate_buy_trend')
+    mocked_buy_trend = mocker.patch('freqtrade.tests.test_backtesting.populate_buy_trend')
+    processed = preprocess(backdata)
 
     def optimizer(params):
         mocked_buy_trend.side_effect = buy_strategy_generator(params)
 
-        results = backtest(backtest_conf, backdata, mocker)
+        results = backtest(backtest_conf, processed, mocker)
 
         result = format_results(results)
 
@@ -146,3 +146,8 @@ def test_hyperopt(backtest_conf, backdata, mocker):
     print('Best parameters {}'.format(best))
     newlist = sorted(trials.results, key=itemgetter('loss'))
     print('Result: {}'.format(newlist[0]['result']))
+
+
+if __name__ == '__main__':
+    # for profiling with cProfile and line_profiler
+    pytest.main([__file__, '-s'])