Merge branch 'develop' into volumeList_enhanced_filter

tests/config_test_comments.json

@@ -78,7 +78,7 @@
             "ZEC/BTC",
             "XLM/BTC",
             "NXT/BTC",
-            "POWR/BTC",
+            "TRX/BTC",
             "ADA/BTC",
             "XMR/BTC"
         ],

tests/data/test_btanalysis.py

@@ -2,7 +2,7 @@ from unittest.mock import MagicMock
 
 import pytest
 from arrow import Arrow
-from pandas import DataFrame, to_datetime
+from pandas import DataFrame, DateOffset, to_datetime
 
 from freqtrade.configuration import TimeRange
 from freqtrade.data.btanalysis import (BT_DATA_COLUMNS,
@@ -125,12 +125,30 @@ def test_create_cum_profit(testdatadir):
     bt_data = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
-    df = load_pair_history(pair="POWR/BTC", ticker_interval='5m',
+    df = load_pair_history(pair="TRX/BTC", ticker_interval='5m',
                            datadir=testdatadir, timerange=timerange)
 
     cum_profits = create_cum_profit(df.set_index('date'),
-                                    bt_data[bt_data["pair"] == 'POWR/BTC'],
-                                    "cum_profits")
+                                    bt_data[bt_data["pair"] == 'TRX/BTC'],
+                                    "cum_profits", timeframe="5m")
+    assert "cum_profits" in cum_profits.columns
+    assert cum_profits.iloc[0]['cum_profits'] == 0
+    assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005
+
+
+def test_create_cum_profit1(testdatadir):
+    filename = testdatadir / "backtest-result_test.json"
+    bt_data = load_backtest_data(filename)
+    # Move close-time to "off" the candle, to make sure the logic still works
+    bt_data.loc[:, 'close_time'] = bt_data.loc[:, 'close_time'] + DateOffset(seconds=20)
+    timerange = TimeRange.parse_timerange("20180110-20180112")
+
+    df = load_pair_history(pair="TRX/BTC", ticker_interval='5m',
+                           datadir=testdatadir, timerange=timerange)
+
+    cum_profits = create_cum_profit(df.set_index('date'),
+                                    bt_data[bt_data["pair"] == 'TRX/BTC'],
+                                    "cum_profits", timeframe="5m")
     assert "cum_profits" in cum_profits.columns
     assert cum_profits.iloc[0]['cum_profits'] == 0
     assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005

tests/data/test_history.py

@@ -95,6 +95,23 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N
     _clean_test_file(file)
 
 
+def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None:
+    ltfmock = mocker.patch('freqtrade.data.history.load_tickerdata_file',
+                           MagicMock(return_value=None))
+    timerange = TimeRange('date', None, 1510639620, 0)
+    history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='1m',
+                              datadir=testdatadir, timerange=timerange,
+                              startup_candles=20,
+                              )
+    assert log_has(
+        'Using indicator startup period: 20 ...', caplog
+    )
+    assert ltfmock.call_count == 1
+    assert ltfmock.call_args_list[0][1]['timerange'] != timerange
+    # startts is 20 minutes earlier
+    assert ltfmock.call_args_list[0][1]['timerange'].startts == timerange.startts - 20 * 60
+
+
 def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
                                       default_conf, testdatadir) -> None:
     """
@@ -427,6 +444,46 @@ def test_trim_tickerlist(testdatadir) -> None:
     assert not ticker
 
 
+def test_trim_dataframe(testdatadir) -> None:
+    data = history.load_data(
+        datadir=testdatadir,
+        ticker_interval='1m',
+        pairs=['UNITTEST/BTC']
+    )['UNITTEST/BTC']
+    min_date = int(data.iloc[0]['date'].timestamp())
+    max_date = int(data.iloc[-1]['date'].timestamp())
+    data_modify = data.copy()
+
+    # Remove first 30 minutes (1800 s)
+    tr = TimeRange('date', None, min_date + 1800, 0)
+    data_modify = history.trim_dataframe(data_modify, tr)
+    assert not data_modify.equals(data)
+    assert len(data_modify) < len(data)
+    assert len(data_modify) == len(data) - 30
+    assert all(data_modify.iloc[-1] == data.iloc[-1])
+    assert all(data_modify.iloc[0] == data.iloc[30])
+
+    data_modify = data.copy()
+    # Remove last 30 minutes (1800 s)
+    tr = TimeRange(None, 'date', 0, max_date - 1800)
+    data_modify = history.trim_dataframe(data_modify, tr)
+    assert not data_modify.equals(data)
+    assert len(data_modify) < len(data)
+    assert len(data_modify) == len(data) - 30
+    assert all(data_modify.iloc[0] == data.iloc[0])
+    assert all(data_modify.iloc[-1] == data.iloc[-31])
+
+    data_modify = data.copy()
+    # Remove first 25 and last 30 minutes (1800 s)
+    tr = TimeRange('date', 'date', min_date + 1500, max_date - 1800)
+    data_modify = history.trim_dataframe(data_modify, tr)
+    assert not data_modify.equals(data)
+    assert len(data_modify) < len(data)
+    assert len(data_modify) == len(data) - 55
+    # first row matches 25th original row
+    assert all(data_modify.iloc[0] == data.iloc[25])
+
+
 def test_file_dump_json_tofile(testdatadir) -> None:
     file = testdatadir / 'test_{id}.json'.format(id=str(uuid.uuid4()))
     data = {'bar': 'foo'}

tests/edge/test_edge.py

@@ -256,7 +256,7 @@ def test_edge_heartbeat_calculate(mocker, edge_conf):
 
 
 def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
-                     timerange=None, exchange=None):
+                     timerange=None, exchange=None, *args, **kwargs):
     hz = 0.1
     base = 0.001
 

tests/exchange/test_exchange.py

@@ -523,6 +523,24 @@ def test_validate_order_types_not_in_config(default_conf, mocker):
     Exchange(conf)
 
 
+def test_validate_required_startup_candles(default_conf, mocker, caplog):
+    api_mock = MagicMock()
+    mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
+
+    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
+    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock())
+
+    default_conf['startup_candle_count'] = 20
+    ex = Exchange(default_conf)
+    assert ex
+    default_conf['startup_candle_count'] = 600
+
+    with pytest.raises(OperationalException, match=r'This strategy requires 600.*'):
+        Exchange(default_conf)
+
+
 def test_exchange_has(default_conf, mocker):
     exchange = get_patched_exchange(mocker, default_conf)
     assert not exchange.exchange_has('ASDFASDF')

tests/optimize/test_backtesting.py

@@ -117,7 +117,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None:
 
 
 def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
-                     timerange=None, exchange=None, live=False):
+                     timerange=None, exchange=None, live=False, *args, **kwargs):
     tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
     pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC",
                                                        fill_missing=True)}
@@ -494,7 +494,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
     def get_timeframe(input1):
         return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
 
-    mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={}))
+    mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=None))
     mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe)
     mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)
@@ -511,10 +511,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
     default_conf['timerange'] = '20180101-20180102'
 
     backtesting = Backtesting(default_conf)
-    backtesting.start()
-    # check the logs, that will contain the backtest result
-
-    assert log_has('No data found. Terminating.', caplog)
+    with pytest.raises(OperationalException, match='No data found. Terminating.'):
+        backtesting.start()
 
 
 def test_backtest(default_conf, fee, mocker, testdatadir) -> None:
@@ -838,6 +836,8 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
+        'Loading data from 2017-11-14T20:57:00+00:00 '
+        'up to 2017-11-14T22:58:00+00:00 (0 days)..',
         'Backtesting with data from 2017-11-14T21:17:00+00:00 '
         'up to 2017-11-14T22:58:00+00:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...'
@@ -892,6 +892,8 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
+        'Loading data from 2017-11-14T20:57:00+00:00 '
+        'up to 2017-11-14T22:58:00+00:00 (0 days)..',
         'Backtesting with data from 2017-11-14T21:17:00+00:00 '
         'up to 2017-11-14T22:58:00+00:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...',

tests/optimize/test_hyperopt.py

@@ -228,7 +228,7 @@ def test_start(mocker, default_conf, caplog) -> None:
 
 def test_start_no_data(mocker, default_conf, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock(return_value={}))
+    mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=None))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -242,9 +242,8 @@ def test_start_no_data(mocker, default_conf, caplog) -> None:
         '--epochs', '5'
     ]
     args = get_args(args)
-    start_hyperopt(args)
-
-    assert log_has('No data found. Terminating.', caplog)
+    with pytest.raises(OperationalException, match='No data found. Terminating.'):
+        start_hyperopt(args)
 
 
 def test_start_filelock(mocker, default_conf, caplog) -> None:
@@ -393,7 +392,8 @@ def test_roi_table_generation(hyperopt) -> None:
 
 def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -516,7 +516,7 @@ def test_generate_optimizer(mocker, default_conf) -> None:
     default_conf.update({'hyperopt_min_trades': 1})
 
     trades = [
-        ('POWR/BTC', 0.023117, 0.000233, 100)
+        ('TRX/BTC', 0.023117, 0.000233, 100)
     ]
     labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration']
     backtest_result = pd.DataFrame.from_records(trades, columns=labels)
@@ -608,7 +608,8 @@ def test_continue_hyperopt(mocker, default_conf, caplog):
 
 def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -645,7 +646,8 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
 
 def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -682,7 +684,8 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) ->
 
 def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -728,7 +731,8 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys)
 
 def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -> None:
     mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -757,7 +761,8 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -
 
 def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -802,7 +807,8 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None:
 
 def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -853,7 +859,8 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None
 ])
 def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, method, space) -> None:
     mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))

tests/test_plotting.py

@@ -53,10 +53,10 @@ def test_init_plotscript(default_conf, mocker, testdatadir):
     assert "trades" in ret
     assert "pairs" in ret
 
-    default_conf['pairs'] = ["POWR/BTC", "ADA/BTC"]
+    default_conf['pairs'] = ["TRX/BTC", "ADA/BTC"]
     ret = init_plotscript(default_conf)
     assert "tickers" in ret
-    assert "POWR/BTC" in ret["tickers"]
+    assert "TRX/BTC" in ret["tickers"]
     assert "ADA/BTC" in ret["tickers"]
 
 
@@ -228,13 +228,13 @@ def test_add_profit(testdatadir):
     bt_data = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
-    df = history.load_pair_history(pair="POWR/BTC", ticker_interval='5m',
+    df = history.load_pair_history(pair="TRX/BTC", ticker_interval='5m',
                                    datadir=testdatadir, timerange=timerange)
     fig = generate_empty_figure()
 
     cum_profits = create_cum_profit(df.set_index('date'),
-                                    bt_data[bt_data["pair"] == 'POWR/BTC'],
-                                    "cum_profits")
+                                    bt_data[bt_data["pair"] == 'TRX/BTC'],
+                                    "cum_profits", timeframe="5m")
 
     fig1 = add_profit(fig, row=2, data=cum_profits, column='cum_profits', name='Profits')
     figure = fig1.layout.figure
@@ -247,7 +247,7 @@ def test_generate_profit_graph(testdatadir):
     filename = testdatadir / "backtest-result_test.json"
     trades = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
-    pairs = ["POWR/BTC", "ADA/BTC"]
+    pairs = ["TRX/BTC", "ADA/BTC"]
 
     tickers = history.load_data(datadir=testdatadir,
                                 pairs=pairs,
@@ -256,7 +256,7 @@ def test_generate_profit_graph(testdatadir):
                                 )
     trades = trades[trades['pair'].isin(pairs)]
 
-    fig = generate_profit_graph(pairs, tickers, trades)
+    fig = generate_profit_graph(pairs, tickers, trades, timeframe="5m")
     assert isinstance(fig, go.Figure)
 
     assert fig.layout.title.text == "Freqtrade Profit plot"

tests/test_timerange.py

@@ -1,10 +1,11 @@
 # pragma pylint: disable=missing-docstring, C0103
+import arrow
 import pytest
 
 from freqtrade.configuration import TimeRange
 
 
-def test_parse_timerange_incorrect() -> None:
+def test_parse_timerange_incorrect():
 
     assert TimeRange('date', None, 1274486400, 0) == TimeRange.parse_timerange('20100522-')
     assert TimeRange(None, 'date', 0, 1274486400) == TimeRange.parse_timerange('-20100522')
@@ -28,3 +29,37 @@ def test_parse_timerange_incorrect() -> None:
 
     with pytest.raises(Exception, match=r'Incorrect syntax.*'):
         TimeRange.parse_timerange('-')
+
+
+def test_subtract_start():
+    x = TimeRange('date', 'date', 1274486400, 1438214400)
+    x.subtract_start(300)
+    assert x.startts == 1274486400 - 300
+
+    # Do nothing if no startdate exists
+    x = TimeRange(None, 'date', 0, 1438214400)
+    x.subtract_start(300)
+    assert not x.startts
+
+    x = TimeRange('date', None, 1274486400, 0)
+    x.subtract_start(300)
+    assert x.startts == 1274486400 - 300
+
+
+def test_adjust_start_if_necessary():
+    min_date = arrow.Arrow(2017, 11, 14, 21, 15, 00)
+
+    x = TimeRange('date', 'date', 1510694100, 1510780500)
+    # Adjust by 20 candles - min_date == startts
+    x.adjust_start_if_necessary(300, 20, min_date)
+    assert x.startts == 1510694100 + (20 * 300)
+
+    x = TimeRange('date', 'date', 1510700100, 1510780500)
+    # Do nothing, startup is set and different min_date
+    x.adjust_start_if_necessary(300, 20, min_date)
+    assert x.startts == 1510694100 + (20 * 300)
+
+    x = TimeRange(None, 'date', 0, 1510780500)
+    # Adjust by 20 candles = 20 * 5m
+    x.adjust_start_if_necessary(300, 20, min_date)
+    assert x.startts == 1510694100 + (20 * 300)

tests/testdata/pairs.json

@@ -9,7 +9,7 @@
     "LTC/BTC",
     "NEO/BTC",
     "NXT/BTC",
-    "POWR/BTC",
+    "TRX/BTC",
     "STORJ/BTC",
     "QTUM/BTC",
     "WAVES/BTC",