Merge commit '1134c81aad049d4357c8f299ffc801218f3d9574' into feature/objectify

freqtrade/analyze.py

@@ -172,19 +172,17 @@ class Analyze(object):
         :return True if bot should sell at current rate
         """
         current_profit = trade.calc_profit_percent(current_rate)
-        if self.strategy.stoploss is not None and current_profit < float(self.strategy.stoploss):
+        if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss:
             self.logger.debug('Stop loss hit.')
             return True
 
         # Check if time matches and current rate is above threshold
         time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60
-        for duration_string, threshold in self.strategy.minimal_roi.items():
+        for duration, threshold in self.strategy.minimal_roi.items():
-            duration = float(duration_string)
+            if time_diff <= duration:
-            if time_diff > duration and current_profit > threshold:
-                return True
-
-            if time_diff < duration:
                 return False
+            if current_profit > threshold:
+                return True
 
         return False
 

freqtrade/configuration.py

@@ -116,7 +116,7 @@ class Configuration(object):
         if 'realistic_simulation' in self.args and self.args.realistic_simulation:
             config.update({'realistic_simulation': True})
             self.logger.info('Parameter --realistic-simulation detected ...')
-            self.logger.info('Using max_open_trades: %s ...', config.get('max_open_trades'))
+        self.logger.info('Using max_open_trades: %s ...', config.get('max_open_trades'))
 
         # If --timerange is used we add it to the configuration
         if 'timerange' in self.args and self.args.timerange:

freqtrade/misc.py

@@ -67,5 +67,5 @@ def file_dump_json(filename, data) -> None:
     :param data: JSON Data to save
     :return:
     """
-    with open(filename, 'w') as file:
+    with open(filename, 'w') as fp:
-        json.dump(data, file)
+        json.dump(data, fp, default=str)

freqtrade/optimize/backtesting.py

@@ -102,43 +102,35 @@ class Backtesting(object):
         ])
         return tabulate(tabular_data, headers=headers, floatfmt=floatfmt)
 
-    def _get_sell_trade_entry(self, pair, row, buy_subset, ticker, trade_count_lock, args):
+    def _get_sell_trade_entry(self, pair, buy_row, partial_ticker, trade_count_lock, args):
         stake_amount = args['stake_amount']
         max_open_trades = args.get('max_open_trades', 0)
         trade = Trade(
-            open_rate=row.close,
+            open_rate=buy_row.close,
-            open_date=row.Index,
+            open_date=buy_row.date,
             stake_amount=stake_amount,
-            amount=stake_amount / row.open,
+            amount=stake_amount / buy_row.open,
             fee=exchange.get_fee()
         )
 
         # calculate win/lose forwards from buy point
-        sell_subset = ticker[ticker.index > row.Index][['close', 'sell', 'buy']]
+        for sell_row in partial_ticker:
-        for row2 in sell_subset.itertuples(index=True):
             if max_open_trades > 0:
                 # Increase trade_count_lock for every iteration
-                trade_count_lock[row2.Index] = trade_count_lock.get(row2.Index, 0) + 1
+                trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1
 
-            buy_signal = row2.buy
+            buy_signal = sell_row.buy
-            if(
+            if self.analyze.should_sell(trade, sell_row.close, sell_row.date, buy_signal,
-                    self.analyze.should_sell(
+                                        sell_row.sell):
-                        trade=trade,
-                        rate=row2.close,
-                        date=row2.Index,
-                        buy=buy_signal,
-                        sell=row2.sell
-                    )
-            ):
                 return \
-                    row2, \
+                    sell_row, \
                     (
                         pair,
-                        trade.calc_profit_percent(rate=row2.close),
+                        trade.calc_profit_percent(rate=sell_row.close),
-                        trade.calc_profit(rate=row2.close),
+                        trade.calc_profit(rate=sell_row.close),
-                        (row2.Index - row.Index).seconds // 60
+                        (sell_row.date - buy_row.date).seconds // 60
-                    ),\
+                    ), \
-                    row2.Index
+                    sell_row.date
         return None
 
     def backtest(self, args) -> DataFrame:
@@ -159,6 +151,7 @@ class Backtesting(object):
             stoploss: use stoploss
         :return: DataFrame
         """
+        headers = ['date', 'buy', 'open', 'close', 'sell']
         processed = args['processed']
         max_open_trades = args.get('max_open_trades', 0)
         realistic = args.get('realistic', True)
@@ -167,37 +160,28 @@ class Backtesting(object):
         trades = []
         trade_count_lock = {}
         for pair, pair_data in processed.items():
-            pair_data['buy'], pair_data['sell'] = 0, 0
+            pair_data['buy'], pair_data['sell'] = 0, 0  # cleanup from previous run
-            ticker = self.populate_sell_trend(
+
-                self.populate_buy_trend(pair_data)
+            ticker_data = self.populate_sell_trend(self.populate_buy_trend(pair_data))[headers]
-            )
+            ticker = [x for x in ticker_data.itertuples()]
-            if 'date' in ticker:
+
-                ticker.set_index('date', inplace=True)
-            # for each buy point
             lock_pair_until = None
-            headers = ['buy', 'open', 'close', 'sell']
+            for index, row in enumerate(ticker):
-            buy_subset = ticker[(ticker.buy == 1) & (ticker.sell == 0)][headers]
+                if row.buy == 0 or row.sell == 1:
-            for row in buy_subset.itertuples(index=True):
+                    continue  # skip rows where no buy signal or that would immediately sell off
+
                 if realistic:
-                    if lock_pair_until is not None and row.Index <= lock_pair_until:
+                    if lock_pair_until is not None and row.date <= lock_pair_until:
                         continue
                 if max_open_trades > 0:
                     # Check if max_open_trades has already been reached for the given date
-                    if not trade_count_lock.get(row.Index, 0) < max_open_trades:
+                    if not trade_count_lock.get(row.date, 0) < max_open_trades:
                         continue
 
-                if max_open_trades > 0:
+                    trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
-                    # Increase lock
-                    trade_count_lock[row.Index] = trade_count_lock.get(row.Index, 0) + 1
 
-                ret = self._get_sell_trade_entry(
+                ret = self._get_sell_trade_entry(pair, row, ticker[index + 1:],
-                    pair=pair,
+                                                 trade_count_lock, args)
-                    row=row,
-                    buy_subset=buy_subset,
-                    ticker=ticker,
-                    trade_count_lock=trade_count_lock,
-                    args=args
-                )
 
                 if ret:
                     row2, trade_entry, next_date = ret
@@ -208,9 +192,9 @@ class Backtesting(object):
                         # record a tuple of pair, current_profit_percent,
                         # entry-date, duration
                         records.append((pair, trade_entry[1],
-                                        row.Index.strftime('%s'),
+                                        row.date.strftime('%s'),
-                                        row2.Index.strftime('%s'),
+                                        row2.date.strftime('%s'),
-                                        row.Index, trade_entry[3]))
+                                        row.date, trade_entry[3]))
         # For now export inside backtest(), maybe change so that backtest()
         # returns a tuple like: (dataframe, records, logs, etc)
         if record and record.find('trades') >= 0:
@@ -226,6 +210,8 @@ class Backtesting(object):
         """
         data = {}
         pairs = self.config['exchange']['pair_whitelist']
+        self.logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
+        self.logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
 
         if self.config.get('live'):
             self.logger.info('Downloading data for all pairs in whitelist ...')
@@ -233,8 +219,6 @@ class Backtesting(object):
                 data[pair] = exchange.get_ticker_history(pair, self.ticker_interval)
         else:
             self.logger.info('Using local backtesting data (using whitelist in given config) ...')
-            self.logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
-            self.logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
 
             timerange = Arguments.parse_timerange(self.config.get('timerange'))
             data = optimize.load_data(

freqtrade/optimize/hyperopt.py

@@ -240,15 +240,15 @@ class Hyperopt(Backtesting):
         return trade_loss + profit_loss + duration_loss
 
     @staticmethod
-    def generate_roi_table(params) -> Dict[str, float]:
+    def generate_roi_table(params) -> Dict[int, float]:
         """
         Generate the ROI table thqt will be used by Hyperopt
         """
         roi_table = {}
-        roi_table["0"] = params['roi_p1'] + params['roi_p2'] + params['roi_p3']
+        roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3']
-        roi_table[str(params['roi_t3'])] = params['roi_p1'] + params['roi_p2']
+        roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2']
-        roi_table[str(params['roi_t3'] + params['roi_t2'])] = params['roi_p1']
+        roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1']
-        roi_table[str(params['roi_t3'] + params['roi_t2'] + params['roi_t1'])] = 0
+        roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0
 
         return roi_table
 

freqtrade/strategy/strategy.py

@@ -58,11 +58,11 @@ class Strategy(object):
 
         # Minimal ROI designed for the strategy
         self.minimal_roi = OrderedDict(sorted(
-            self.custom_strategy.minimal_roi.items(),
+            {int(key): value for (key, value) in self.custom_strategy.minimal_roi.items()}.items(),
-            key=lambda tuple: float(tuple[0])))  # sort after converting to number
+            key=lambda tuple: tuple[0]))  # sort after converting to number
 
         # Optimal stoploss designed for the strategy
-        self.stoploss = self.custom_strategy.stoploss
+        self.stoploss = float(self.custom_strategy.stoploss)
 
         self.ticker_interval = self.custom_strategy.ticker_interval
 

freqtrade/tests/conftest.py

@@ -285,6 +285,7 @@ def ticker_history_without_bv():
     ]
 
 
+# FIX: Perhaps change result fixture to use BTC_UNITEST instead?
 @pytest.fixture
 def result():
     with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file:

freqtrade/tests/optimize/test_backtesting.py

@@ -1,12 +1,14 @@
 # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument
 
 import json
+import random
 import math
 from typing import List
 from copy import deepcopy
 from unittest.mock import MagicMock
 from arrow import Arrow
 import pandas as pd
+import numpy as np
 from freqtrade import optimize
 from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration
 from freqtrade.arguments import Arguments
@@ -96,6 +98,70 @@ def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False,
     return pairdata
 
 
+# use for mock freqtrade.exchange.get_ticker_history'
+def _load_pair_as_ticks(pair, tickfreq):
+    ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair])
+    ticks = trim_dictlist(ticks, -200)
+    return ticks[pair]
+
+
+# FIX: fixturize this?
+def _make_backtest_conf(conf=None, pair='BTC_UNITEST', record=None):
+    data = optimize.load_data(None, ticker_interval=8, pairs=[pair])
+    data = trim_dictlist(data, -200)
+    return {
+        'stake_amount': conf['stake_amount'],
+        'processed': _BACKTESTING.tickerdata_to_dataframe(data),
+        'max_open_trades': 10,
+        'realistic': True,
+        'record': record
+    }
+
+
+def _trend(signals, buy_value, sell_value):
+    n = len(signals['low'])
+    buy = np.zeros(n)
+    sell = np.zeros(n)
+    for i in range(0, len(signals['buy'])):
+        if random.random() > 0.5:  # Both buy and sell signals at same timeframe
+            buy[i] = buy_value
+            sell[i] = sell_value
+    signals['buy'] = buy
+    signals['sell'] = sell
+    return signals
+
+
+def _trend_alternate(dataframe=None):
+    signals = dataframe
+    low = signals['low']
+    n = len(low)
+    buy = np.zeros(n)
+    sell = np.zeros(n)
+    for i in range(0, len(buy)):
+        if i % 2 == 0:
+            buy[i] = 1
+        else:
+            sell[i] = 1
+    signals['buy'] = buy
+    signals['sell'] = sell
+    return dataframe
+
+
+def _run_backtest_1(fun, backtest_conf):
+    # strategy is a global (hidden as a singleton), so we
+    # emulate strategy being pure, by override/restore here
+    # if we dont do this, the override in strategy will carry over
+    # to other tests
+    old_buy = _BACKTESTING.populate_buy_trend
+    old_sell = _BACKTESTING.populate_sell_trend
+    _BACKTESTING.populate_buy_trend = fun  # Override
+    _BACKTESTING.populate_sell_trend = fun  # Override
+    results = _BACKTESTING.backtest(backtest_conf)
+    _BACKTESTING.populate_buy_trend = old_buy  # restore override
+    _BACKTESTING.populate_sell_trend = old_sell  # restore override
+    return results
+
+
 # Unit tests
 def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None:
     """
@@ -418,3 +484,125 @@ def test_backtest_pricecontours(default_conf) -> None:
     tests = [['raise', 17], ['lower', 0], ['sine', 17]]
     for [contour, numres] in tests:
         simple_backtest(default_conf, contour, numres)
+
+
+# Test backtest using offline data (testdata directory)
+def test_backtest_ticks(default_conf):
+    ticks = [1, 5]
+    fun = _BACKTESTING.populate_buy_trend
+    for tick in ticks:
+        backtest_conf = _make_backtest_conf(conf=default_conf)
+        results = _run_backtest_1(fun, backtest_conf)
+        assert not results.empty
+
+
+def test_backtest_clash_buy_sell(default_conf):
+    # Override the default buy trend function in our default_strategy
+    def fun(dataframe=None):
+        buy_value = 1
+        sell_value = 1
+        return _trend(dataframe, buy_value, sell_value)
+
+    backtest_conf = _make_backtest_conf(conf=default_conf)
+    results = _run_backtest_1(fun, backtest_conf)
+    assert results.empty
+
+
+def test_backtest_only_sell(default_conf):
+    # Override the default buy trend function in our default_strategy
+    def fun(dataframe=None):
+        buy_value = 0
+        sell_value = 1
+        return _trend(dataframe, buy_value, sell_value)
+
+    backtest_conf = _make_backtest_conf(conf=default_conf)
+    results = _run_backtest_1(fun, backtest_conf)
+    assert results.empty
+
+
+def test_backtest_alternate_buy_sell(default_conf):
+    backtest_conf = _make_backtest_conf(conf=default_conf, pair='BTC_UNITEST')
+    results = _run_backtest_1(_trend_alternate, backtest_conf)
+    assert len(results) == 3
+
+
+def test_backtest_record(default_conf, mocker):
+    names = []
+    records = []
+    mocker.patch(
+        'freqtrade.optimize.backtesting.file_dump_json',
+        new=lambda n, r: (names.append(n), records.append(r))
+    )
+    backtest_conf = _make_backtest_conf(
+        conf=default_conf,
+        pair='BTC_UNITEST',
+        record="trades"
+    )
+    results = _run_backtest_1(_trend_alternate, backtest_conf)
+    assert len(results) == 3
+    # Assert file_dump_json was only called once
+    assert names == ['backtest-result.json']
+    records = records[0]
+    # Ensure records are of correct type
+    assert len(records) == 3
+    # ('BTC_UNITEST', 0.00331158, '1510684320', '1510691700', 0, 117)
+    # Below follows just a typecheck of the schema/type of trade-records
+    oix = None
+    for (pair, profit, date_buy, date_sell, buy_index, dur) in records:
+        assert pair == 'BTC_UNITEST'
+        isinstance(profit, float)
+        # FIX: buy/sell should be converted to ints
+        isinstance(date_buy, str)
+        isinstance(date_sell, str)
+        isinstance(buy_index, pd._libs.tslib.Timestamp)
+        if oix:
+            assert buy_index > oix
+        oix = buy_index
+        assert dur > 0
+
+
+def test_backtest_start_live(default_conf, mocker, caplog):
+    default_conf['exchange']['pair_whitelist'] = ['BTC_UNITEST']
+    mocker.patch('freqtrade.exchange.get_ticker_history',
+                 new=lambda n, i: _load_pair_as_ticks(n, i))
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock())
+    mocker.patch('freqtrade.configuration.open', mocker.mock_open(
+        read_data=json.dumps(default_conf)
+    ))
+
+    args = MagicMock()
+    args.ticker_interval = 1
+    args.level = 10
+    args.live = True
+    args.datadir = None
+    args.export = None
+    args.strategy = 'default_strategy'
+    args.timerange = '-100'  # needed due to MagicMock malleability
+
+    args = [
+        '--config', 'config.json',
+        '--strategy', 'default_strategy',
+        'backtesting',
+        '--ticker-interval', '1',
+        '--live',
+        '--timerange', '-100'
+    ]
+    args = get_args(args)
+    start(args)
+    # check the logs, that will contain the backtest result
+    exists = [
+        'Parameter -i/--ticker-interval detected ...',
+        'Using ticker_interval: 1 ...',
+        'Parameter -l/--live detected ...',
+        'Using max_open_trades: 1 ...',
+        'Parameter --timerange detected: -100 ..',
+        'Parameter --datadir detected: freqtrade/tests/testdata ...',
+        'Using stake_currency: BTC ...',
+        'Using stake_amount: 0.001 ...',
+        'Downloading data for all pairs in whitelist ...',
+        'Measuring data from 2017-11-14T19:32:00+00:00 up to 2017-11-14T22:59:00+00:00 (0 days)..'
+    ]
+
+    for line in exists:
+        tt.log_has(line, caplog.record_tuples)

freqtrade/tests/optimize/test_hyperopt.py

@@ -2,6 +2,7 @@
 import os
 from copy import deepcopy
 from unittest.mock import MagicMock
+import pandas as pd
 from freqtrade.optimize.hyperopt import Hyperopt
 import freqtrade.tests.conftest as tt  # test tools
 
@@ -157,7 +158,7 @@ def test_fmin_best_results(mocker, default_conf, caplog) -> None:
         '"uptrend_long_ema": {\n        "enabled": true\n    },',
         '"uptrend_short_ema": {\n        "enabled": false\n    },',
         '"uptrend_sma": {\n        "enabled": false\n    }',
-        'ROI table:\n{\'0\': 6.0, \'3.0\': 3.0, \'5.0\': 1.0, \'6.0\': 0}',
+        'ROI table:\n{0: 6.0, 3.0: 3.0, 5.0: 1.0, 6.0: 0}',
         'Best Result:\nfoo'
     ]
     for line in exists:
@@ -275,7 +276,7 @@ def test_roi_table_generation() -> None:
     }
 
     hyperopt = _HYPEROPT
-    assert hyperopt.generate_roi_table(params) == {'0': 6, '15': 3, '25': 1, '30': 0}
+    assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0}
 
 
 def test_start_calls_fmin(mocker, default_conf) -> None:
@@ -319,3 +320,36 @@ def test_start_uses_mongotrials(mocker, default_conf) -> None:
     hyperopt.start()
     mock_mongotrials.assert_called_once()
     mock_fmin.assert_called_once()
+
+
+# test log_trials_result
+# test buy_strategy_generator def populate_buy_trend
+# test optimizer if 'ro_t1' in params
+
+def test_format_results():
+    """
+    Test Hyperopt.format_results()
+    """
+    trades = [
+        ('BTC_ETH', 2, 2, 123),
+        ('BTC_LTC', 1, 1, 123),
+        ('BTC_XRP', -1, -2, -246)
+    ]
+    labels = ['currency', 'profit_percent', 'profit_BTC', 'duration']
+    df = pd.DataFrame.from_records(trades, columns=labels)
+    x = Hyperopt.format_results(df)
+    assert x.find(' 66.67%')
+
+
+def test_signal_handler(mocker):
+    """
+    Test Hyperopt.signal_handler()
+    """
+    m = MagicMock()
+    mocker.patch('sys.exit', m)
+    mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.save_trials', m)
+    mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.log_trials_result', m)
+
+    hyperopt = _HYPEROPT
+    hyperopt.signal_handler(9, None)
+    assert m.call_count == 3

freqtrade/tests/strategy/test_strategy.py

@@ -55,7 +55,7 @@ def test_strategy(result):
     strategy = Strategy({'strategy': 'default_strategy'})
 
     assert hasattr(strategy.custom_strategy, 'minimal_roi')
-    assert strategy.minimal_roi['0'] == 0.04
+    assert strategy.minimal_roi[0] == 0.04
 
     assert hasattr(strategy.custom_strategy, 'stoploss')
     assert strategy.stoploss == -0.10
@@ -83,7 +83,7 @@ def test_strategy_override_minimal_roi(caplog):
     strategy = Strategy(config)
 
     assert hasattr(strategy.custom_strategy, 'minimal_roi')
-    assert strategy.minimal_roi['0'] == 0.5
+    assert strategy.minimal_roi[0] == 0.5
     assert ('freqtrade.strategy.strategy',
             logging.INFO,
             'Override strategy \'minimal_roi\' with value in config file.'
@@ -136,8 +136,8 @@ def test_strategy_singleton():
     strategy1 = Strategy({'strategy': 'default_strategy'})
 
     assert hasattr(strategy1.custom_strategy, 'minimal_roi')
-    assert strategy1.minimal_roi['0'] == 0.04
+    assert strategy1.minimal_roi[0] == 0.04
 
     strategy2 = Strategy()
     assert hasattr(strategy2.custom_strategy, 'minimal_roi')
-    assert strategy2.minimal_roi['0'] == 0.04
+    assert strategy2.minimal_roi[0] == 0.04

freqtrade/tests/test_analyze.py

@@ -43,6 +43,11 @@ def test_analyze_object() -> None:
     assert hasattr(Analyze, 'min_roi_reached')
 
 
+def test_dataframe_correct_length(result):
+    dataframe = Analyze.parse_ticker_dataframe(result)
+    assert len(result.index) == len(dataframe.index)
+
+
 def test_dataframe_correct_columns(result):
     assert result.columns.tolist() == \
         ['close', 'high', 'low', 'open', 'date', 'volume']