From 0ffb184ebad3631320e7f3279746e441d3c591a0 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 14 Aug 2019 20:45:24 +0200
Subject: [PATCH 01/33] Change some docstrings and formatting from history

---
 freqtrade/data/history.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index f600615df..363495796 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -57,10 +57,8 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
     return tickerlist[start_index:stop_index]
 
 
-def load_tickerdata_file(
-        datadir: Optional[Path], pair: str,
-        ticker_interval: str,
-        timerange: Optional[TimeRange] = None) -> Optional[list]:
+def load_tickerdata_file(datadir: Optional[Path], pair: str, ticker_interval: str,
+                         timerange: Optional[TimeRange] = None) -> Optional[list]:
     """
     Load a pair from file, either .json.gz or .json
     :return: tickerlist or None if unsuccesful
@@ -68,7 +66,7 @@ def load_tickerdata_file(
     filename = pair_data_filename(datadir, pair, ticker_interval)
     pairdata = misc.file_load_json(filename)
     if not pairdata:
-        return None
+        return []
 
     if timerange:
         pairdata = trim_tickerlist(pairdata, timerange)
@@ -182,6 +180,7 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str,
                                                                            Optional[int]]:
     """
     Load cached data and choose what part of the data should be updated
+    Only used by download_pair_history().
     """
 
     since_ms = None

From 91d1061c7304a598caa1be3a38c055530920956d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 14 Aug 2019 20:48:42 +0200
Subject: [PATCH 02/33] Abstract tickerdata storing

---
 freqtrade/data/history.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index 363495796..849b882b2 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -73,6 +73,15 @@ def load_tickerdata_file(datadir: Optional[Path], pair: str, ticker_interval: st
     return pairdata
 
 
+def store_tickerdata_file(datadir: Optional[Path], pair: str,
+                          ticker_interval: str, data: list, is_zip: bool = False):
+    """
+    Stores tickerdata to file
+    """
+    filename = pair_data_filename(datadir, pair, ticker_interval)
+    misc.file_dump_json(filename, data, is_zip=is_zip)
+
+
 def load_pair_history(pair: str,
                       ticker_interval: str,
                       datadir: Optional[Path],
@@ -175,7 +184,7 @@ def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str)
     return filename
 
 
-def load_cached_data_for_updating(filename: Path, ticker_interval: str,
+def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str,
                                   timerange: Optional[TimeRange]) -> Tuple[List[Any],
                                                                            Optional[int]]:
     """
@@ -194,12 +203,10 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str,
             since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000
 
     # read the cached file
-    if filename.is_file():
-        with open(filename, "rt") as file:
-            data = misc.json_load(file)
-        # remove the last item, could be incomplete candle
-        if data:
-            data.pop()
+    data = load_tickerdata_file(datadir, pair, ticker_interval, TimeRange)
+    # remove the last item, could be incomplete candle
+    if data:
+        data.pop()
     else:
         data = []
 
@@ -238,14 +245,12 @@ def download_pair_history(datadir: Optional[Path],
         )
 
     try:
-        filename = pair_data_filename(datadir, pair, ticker_interval)
-
         logger.info(
             f'Download history data for pair: "{pair}", interval: {ticker_interval} '
             f'and store in {datadir}.'
         )
 
-        data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange)
+        data, since_ms = load_cached_data_for_updating(datadir, pair, ticker_interval, timerange)
 
         logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None')
         logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
@@ -260,7 +265,7 @@ def download_pair_history(datadir: Optional[Path],
         logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
         logger.debug("New End: %s", misc.format_ms_time(data[-1][0]))
 
-        misc.file_dump_json(filename, data)
+        store_tickerdata_file(datadir, pair, ticker_interval, data=data)
         return True
 
     except Exception as e:

From 9d3322df8c00d4d18f4b182ff0d494055c002013 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 14 Aug 2019 20:49:13 +0200
Subject: [PATCH 03/33] Adapt history-tests to new load_cached_data header

---
 freqtrade/tests/data/test_history.py | 40 +++++++---------------------
 1 file changed, 10 insertions(+), 30 deletions(-)

diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py
index 00f4738f7..a06c5aa23 100644
--- a/freqtrade/tests/data/test_history.py
+++ b/freqtrade/tests/data/test_history.py
@@ -178,53 +178,41 @@ def test_load_cached_data_for_updating(mocker) -> None:
     # timeframe starts earlier than the cached data
     # should fully update data
     timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0)
-    data, start_ts = load_cached_data_for_updating(test_filename,
-                                                   '1m',
-                                                   timerange)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == []
     assert start_ts == test_data[0][0] - 1000
 
     # same with 'line' timeframe
     num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120
-    data, start_ts = load_cached_data_for_updating(test_filename,
-                                                   '1m',
-                                                   TimeRange(None, 'line', 0, -num_lines))
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', TimeRange(None, 'line', 0, -num_lines))
     assert data == []
     assert start_ts < test_data[0][0] - 1
 
     # timeframe starts in the center of the cached data
     # should return the chached data w/o the last item
     timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0)
-    data, start_ts = load_cached_data_for_updating(test_filename,
-                                                   '1m',
-                                                   timerange)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
     # same with 'line' timeframe
     num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30
     timerange = TimeRange(None, 'line', 0, -num_lines)
-    data, start_ts = load_cached_data_for_updating(test_filename,
-                                                   '1m',
-                                                   timerange)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
     # timeframe starts after the chached data
     # should return the chached data w/o the last item
     timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0)
-    data, start_ts = load_cached_data_for_updating(test_filename,
-                                                   '1m',
-                                                   timerange)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
     # same with 'line' timeframe
     num_lines = 30
     timerange = TimeRange(None, 'line', 0, -num_lines)
-    data, start_ts = load_cached_data_for_updating(test_filename,
-                                                   '1m',
-                                                   timerange)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
@@ -232,35 +220,27 @@ def test_load_cached_data_for_updating(mocker) -> None:
     # should return the chached data w/o the last item
     num_lines = 30
     timerange = TimeRange(None, 'line', 0, -num_lines)
-    data, start_ts = load_cached_data_for_updating(test_filename,
-                                                   '1m',
-                                                   timerange)
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
     # no datafile exist
     # should return timestamp start time
     timerange = TimeRange('date', None, now_ts - 10000, 0)
-    data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'),
-                                                   '1m',
-                                                   timerange)
+    data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange)
     assert data == []
     assert start_ts == (now_ts - 10000) * 1000
 
     # same with 'line' timeframe
     num_lines = 30
     timerange = TimeRange(None, 'line', 0, -num_lines)
-    data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'),
-                                                   '1m',
-                                                   timerange)
+    data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange)
     assert data == []
     assert start_ts == (now_ts - num_lines * 60) * 1000
 
     # no datafile exist, no timeframe is set
     # should return an empty array and None
-    data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'),
-                                                   '1m',
-                                                   None)
+    data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', None)
     assert data == []
     assert start_ts is None
 

From b2a22f1afb4e10d635c33623acddd22a8dc4d9ae Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 14 Aug 2019 21:39:53 +0200
Subject: [PATCH 04/33] Fix samll errors

---
 freqtrade/data/history.py            | 4 ++--
 freqtrade/tests/data/test_history.py | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index 849b882b2..c6d731afa 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -184,7 +184,7 @@ def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str)
     return filename
 
 
-def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str,
+def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_interval: str,
                                   timerange: Optional[TimeRange]) -> Tuple[List[Any],
                                                                            Optional[int]]:
     """
@@ -203,7 +203,7 @@ def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str
             since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000
 
     # read the cached file
-    data = load_tickerdata_file(datadir, pair, ticker_interval, TimeRange)
+    data = load_tickerdata_file(datadir, pair, ticker_interval, timerange)
     # remove the last item, could be incomplete candle
     if data:
         data.pop()
diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py
index a06c5aa23..164ebe01a 100644
--- a/freqtrade/tests/data/test_history.py
+++ b/freqtrade/tests/data/test_history.py
@@ -184,7 +184,8 @@ def test_load_cached_data_for_updating(mocker) -> None:
 
     # same with 'line' timeframe
     num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120
-    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', TimeRange(None, 'line', 0, -num_lines))
+    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m',
+                                                   TimeRange(None, 'line', 0, -num_lines))
     assert data == []
     assert start_ts < test_data[0][0] - 1
 

From f3e6bcb20c166bc7287caced4e3559604a39f0c4 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 15 Aug 2019 06:35:50 +0200
Subject: [PATCH 05/33] Avoid using negative indexes

---
 freqtrade/data/history.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index c6d731afa..5471767f6 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -43,7 +43,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
             start_index += 1
 
     if timerange.stoptype == 'line':
-        start_index = len(tickerlist) + timerange.stopts
+        start_index = max(len(tickerlist) + timerange.stopts, 0)
     if timerange.stoptype == 'index':
         stop_index = timerange.stopts
     elif timerange.stoptype == 'date':

From a94a89086f7b37638a4eec19c93482db9d05a731 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 15 Aug 2019 20:09:00 +0200
Subject: [PATCH 06/33] Don't forward timerange to load_ticker_file

when loading cached data for updating.
We always want to get all data, not just a fraction (we would end up
overwriting the non-loaded part of the data).
---
 freqtrade/data/history.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index 5471767f6..14749925f 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -203,7 +203,8 @@ def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_int
             since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000
 
     # read the cached file
-    data = load_tickerdata_file(datadir, pair, ticker_interval, timerange)
+    # Intentionally don't pass timerange in - since we need to load the full dataset.
+    data = load_tickerdata_file(datadir, pair, ticker_interval)
     # remove the last item, could be incomplete candle
     if data:
         data.pop()

From 12677f2d42f8d3f789cd84b42f693bb142c6f9ad Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 15 Aug 2019 20:13:19 +0200
Subject: [PATCH 07/33] Adjust docstring to match functioning of
 load_cached_data

---
 freqtrade/data/history.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index 14749925f..af7d8cc7c 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -188,7 +188,9 @@ def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_int
                                   timerange: Optional[TimeRange]) -> Tuple[List[Any],
                                                                            Optional[int]]:
     """
-    Load cached data and choose what part of the data should be updated
+    Load cached data to download more data.
+    If timerange is passed in, checks wether data from an before the stored data will be downloaded.
+    If that's the case than what's available should be completely overwritten.
     Only used by download_pair_history().
     """
 

From 69eff890496df3cc374ae1f8eb260aa2d421db52 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 15 Aug 2019 20:28:32 +0200
Subject: [PATCH 08/33] Improve comment in test_history to explain what is
 tested

---
 freqtrade/tests/data/test_history.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py
index 164ebe01a..f238046d7 100644
--- a/freqtrade/tests/data/test_history.py
+++ b/freqtrade/tests/data/test_history.py
@@ -210,7 +210,8 @@ def test_load_cached_data_for_updating(mocker) -> None:
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
-    # same with 'line' timeframe
+    # Try loading last 30 lines.
+    # Not supported by load_cached_data_for_updating, we always need to get the full data.
     num_lines = 30
     timerange = TimeRange(None, 'line', 0, -num_lines)
     data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)

From 09286d49182fb98553dd2d137fc85b3d6b742a50 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 16 Aug 2019 13:04:07 +0200
Subject: [PATCH 09/33] file_dump_json accepts Path - so we should feed it that

---
 freqtrade/misc.py                            |  8 ++++----
 freqtrade/optimize/backtesting.py            | 12 ++++++------
 freqtrade/tests/optimize/test_backtesting.py |  5 +++--
 freqtrade/tests/test_misc.py                 |  5 +++--
 4 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/freqtrade/misc.py b/freqtrade/misc.py
index 05946e008..d01d6a254 100644
--- a/freqtrade/misc.py
+++ b/freqtrade/misc.py
@@ -5,11 +5,11 @@ import gzip
 import logging
 import re
 from datetime import datetime
+from pathlib import Path
 
 import numpy as np
 import rapidjson
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -39,7 +39,7 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray:
     return dates.dt.to_pydatetime()
 
 
-def file_dump_json(filename, data, is_zip=False) -> None:
+def file_dump_json(filename: Path, data, is_zip=False) -> None:
     """
     Dump JSON data into a file
     :param filename: file to create
@@ -49,8 +49,8 @@ def file_dump_json(filename, data, is_zip=False) -> None:
     logger.info(f'dumping json to "{filename}"')
 
     if is_zip:
-        if not filename.endswith('.gz'):
-            filename = filename + '.gz'
+        if filename.suffix != '.gz':
+            filename = filename.with_suffix('.gz')
         with gzip.open(filename, 'w') as fp:
             rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE)
     else:
diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 252175269..d321affeb 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -190,7 +190,7 @@ class Backtesting(object):
         return tabulate(tabular_data, headers=headers,  # type: ignore
                         floatfmt=floatfmt, tablefmt="pipe")
 
-    def _store_backtest_result(self, recordfilename: str, results: DataFrame,
+    def _store_backtest_result(self, recordfilename: Path, results: DataFrame,
                                strategyname: Optional[str] = None) -> None:
 
         records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
@@ -201,10 +201,10 @@ class Backtesting(object):
         if records:
             if strategyname:
                 # Inject strategyname to filename
-                recname = Path(recordfilename)
-                recordfilename = str(Path.joinpath(
-                    recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix))
-            logger.info('Dumping backtest results to %s', recordfilename)
+                recordfilename = Path.joinpath(
+                    recordfilename.parent,
+                    f'{recordfilename.stem}-{strategyname}').with_suffix(recordfilename.suffix)
+            logger.info(f'Dumping backtest results to {recordfilename}')
             file_dump_json(recordfilename, records)
 
     def _get_ticker_list(self, processed) -> Dict[str, DataFrame]:
@@ -458,7 +458,7 @@ class Backtesting(object):
         for strategy, results in all_results.items():
 
             if self.config.get('export', False):
-                self._store_backtest_result(self.config['exportfilename'], results,
+                self._store_backtest_result(Path(self.config['exportfilename']), results,
                                             strategy if len(self.strategylist) > 1 else None)
 
             print(f"Result for strategy {strategy}")
diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py
index 9ed7e7296..02e9a9c28 100644
--- a/freqtrade/tests/optimize/test_backtesting.py
+++ b/freqtrade/tests/optimize/test_backtesting.py
@@ -2,6 +2,7 @@
 
 import math
 import random
+from pathlib import Path
 from unittest.mock import MagicMock
 
 import numpy as np
@@ -785,10 +786,10 @@ def test_backtest_record(default_conf, fee, mocker):
     # reset test to test with strategy name
     names = []
     records = []
-    backtesting._store_backtest_result("backtest-result.json", results, "DefStrat")
+    backtesting._store_backtest_result(Path("backtest-result.json"), results, "DefStrat")
     assert len(results) == 4
     # Assert file_dump_json was only called once
-    assert names == ['backtest-result-DefStrat.json']
+    assert names == [Path('backtest-result-DefStrat.json')]
     records = records[0]
     # Ensure records are of correct type
     assert len(records) == 4
diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py
index 1a6b2a92d..c55083e64 100644
--- a/freqtrade/tests/test_misc.py
+++ b/freqtrade/tests/test_misc.py
@@ -1,6 +1,7 @@
 # pragma pylint: disable=missing-docstring,C0103
 
 import datetime
+from pathlib import Path
 from unittest.mock import MagicMock
 
 from freqtrade.data.converter import parse_ticker_dataframe
@@ -34,12 +35,12 @@ def test_datesarray_to_datetimearray(ticker_history_list):
 def test_file_dump_json(mocker) -> None:
     file_open = mocker.patch('freqtrade.misc.open', MagicMock())
     json_dump = mocker.patch('rapidjson.dump', MagicMock())
-    file_dump_json('somefile', [1, 2, 3])
+    file_dump_json(Path('somefile'), [1, 2, 3])
     assert file_open.call_count == 1
     assert json_dump.call_count == 1
     file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock())
     json_dump = mocker.patch('rapidjson.dump', MagicMock())
-    file_dump_json('somefile', [1, 2, 3], True)
+    file_dump_json(Path('somefile'), [1, 2, 3], True)
     assert file_open.call_count == 1
     assert json_dump.call_count == 1
 

From 84a0f9ea42518d9e1dced4e027044365ec00a01c Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Sat, 17 Aug 2019 11:43:36 +0300
Subject: [PATCH 10/33] get_pair_dataframe helper method added

---
 freqtrade/data/dataprovider.py | 46 ++++++++++++++++++++++------------
 1 file changed, 30 insertions(+), 16 deletions(-)

diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py
index b87589df7..e806f5aa7 100644
--- a/freqtrade/data/dataprovider.py
+++ b/freqtrade/data/dataprovider.py
@@ -44,36 +44,50 @@ class DataProvider():
 
     def ohlcv(self, pair: str, ticker_interval: str = None, copy: bool = True) -> DataFrame:
         """
-        get ohlcv data for the given pair as DataFrame
-        Please check `available_pairs` to verify which pairs are currently cached.
+        Get ohlcv data for the given pair as DataFrame
+        Please check `self.available_pairs` to verify which pairs are currently cached.
         :param pair: pair to get the data for
-        :param ticker_interval: ticker_interval to get pair for
-        :param copy: copy dataframe before returning.
-                     Use false only for RO operations (where the dataframe is not modified)
+        :param ticker_interval: ticker interval to get data for
+        :param copy: copy dataframe before returning if True.
+                     Use False only for read-only operations (where the dataframe is not modified)
         """
         if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
-            if ticker_interval:
-                pairtick = (pair, ticker_interval)
-            else:
-                pairtick = (pair, self._config['ticker_interval'])
+            pairtick = (pair, ticker_interval or self._config['ticker_interval'])
+            if pairtick in self.available_pairs:
+                return self._exchange.klines(pairtick, copy=copy)
 
-            return self._exchange.klines(pairtick, copy=copy)
-        else:
-            return DataFrame()
+        return DataFrame()
 
-    def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame:
+    def historic_ohlcv(self, pair: str, ticker_interval: str = None) -> DataFrame:
         """
-        get stored historic ohlcv data
+        Get stored historic ohlcv data
         :param pair: pair to get the data for
-        :param ticker_interval: ticker_interval to get pair for
+        :param ticker_interval: ticker interval to get data for
         """
         return load_pair_history(pair=pair,
-                                 ticker_interval=ticker_interval,
+                                 ticker_interval=ticker_interval or self._config['ticker_interval'],
                                  refresh_pairs=False,
                                  datadir=Path(self._config['datadir']) if self._config.get(
                                      'datadir') else None
                                  )
 
+    def get_pair_dataframe(self, pair: str, ticker_interval: str = None) -> DataFrame:
+        """
+        Return pair ohlcv data, either live or cached historical -- depending
+        on the runmode.
+        :param pair: pair to get the data for
+        :param ticker_interval: ticker interval to get data for
+        """
+        if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
+            # Get live ohlcv data.
+            data = self.ohlcv(pair=pair, ticker_interval=ticker_interval)
+        else:
+            # Get historic ohlcv data (cached on disk).
+            data = self.historic_ohlcv(pair=pair, ticker_interval=ticker_interval)
+        if len(data) == 0:
+            logger.warning(f"No data found for pair {pair}")
+        return data
+
     def ticker(self, pair: str):
         """
         Return last ticker data

From cda912bd8cc060b09168b68194b3a042b8f879ab Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Sat, 17 Aug 2019 12:59:27 +0300
Subject: [PATCH 11/33] test added

---
 freqtrade/tests/data/test_dataprovider.py | 33 +++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py
index 993f0b59b..54aab7052 100644
--- a/freqtrade/tests/data/test_dataprovider.py
+++ b/freqtrade/tests/data/test_dataprovider.py
@@ -51,6 +51,39 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history):
     assert historymock.call_args_list[0][1]["ticker_interval"] == "5m"
 
 
+def test_get_pair_dataframe(mocker, default_conf, ticker_history):
+    default_conf["runmode"] = RunMode.DRY_RUN
+    ticker_interval = default_conf["ticker_interval"]
+    exchange = get_patched_exchange(mocker, default_conf)
+    exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history
+    exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.DRY_RUN
+    assert ticker_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval))
+    assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame)
+    assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval) is not ticker_history
+    assert not dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval).empty
+    assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty
+
+    # Test with and without parameter
+    assert dp.get_pair_dataframe("UNITTEST/BTC",
+                                 ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC"))
+
+    default_conf["runmode"] = RunMode.LIVE
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.LIVE
+    assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame)
+    assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty
+
+    historymock = MagicMock(return_value=ticker_history)
+    mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock)
+    default_conf["runmode"] = RunMode.BACKTEST
+    dp = DataProvider(default_conf, exchange)
+    assert dp.runmode == RunMode.BACKTEST
+    assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame)
+    # assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty
+
+
 def test_available_pairs(mocker, default_conf, ticker_history):
     exchange = get_patched_exchange(mocker, default_conf)
 

From fce3d7586f320b1fc391d6a52b757f54ed3438d3 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Sat, 17 Aug 2019 15:13:39 +0000
Subject: [PATCH 12/33] Bump pytest from 5.0.1 to 5.1.0

Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.0.1 to 5.1.0.
- [Release notes](https://github.com/pytest-dev/pytest/releases)
- [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest/compare/5.0.1...5.1.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-dev.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 03b37417e..6436c60e4 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -7,7 +7,7 @@ flake8==3.7.8
 flake8-type-annotations==0.1.0
 flake8-tidy-imports==2.0.0
 mypy==0.720
-pytest==5.0.1
+pytest==5.1.0
 pytest-asyncio==0.10.0
 pytest-cov==2.7.1
 pytest-mock==1.10.4

From 4ce3cc66d5ea41be0bfbf4c6a1a1b64c5117efd7 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Sat, 17 Aug 2019 15:14:01 +0000
Subject: [PATCH 13/33] Bump sqlalchemy from 1.3.6 to 1.3.7

Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.3.6 to 1.3.7.
- [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases)
- [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/master/CHANGES)
- [Commits](https://github.com/sqlalchemy/sqlalchemy/commits)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 651be7611..91da6e45c 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,7 +1,7 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
 ccxt==1.18.1043
-SQLAlchemy==1.3.6
+SQLAlchemy==1.3.7
 python-telegram-bot==11.1.0
 arrow==0.14.5
 cachetools==3.1.1

From e0335705b2a95387f32d692f6417f55ccaf070b3 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 17 Aug 2019 17:19:02 +0200
Subject: [PATCH 14/33] Add dependabot config yaml

---
 .dependabot/config.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 .dependabot/config.yml

diff --git a/.dependabot/config.yml b/.dependabot/config.yml
new file mode 100644
index 000000000..66b91e99f
--- /dev/null
+++ b/.dependabot/config.yml
@@ -0,0 +1,17 @@
+version: 1
+
+update_configs:
+  - package_manager: "python"
+    directory: "/"
+    update_schedule: "weekly"
+    allowed_updates:
+    - match:
+        update_type: "all"
+    target_branch: "develop"
+
+  - package_manager: "docker"
+    directory: "/"
+    update_schedule: "daily"
+    allowed_updates:
+    - match:
+        update_type: "all"

From 9143ea13adbeeee2fad130debaf1d0ebbd9e8a2c Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Sat, 17 Aug 2019 15:26:07 +0000
Subject: [PATCH 15/33] Bump ccxt from 1.18.1043 to 1.18.1063

Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.18.1043 to 1.18.1063.
- [Release notes](https://github.com/ccxt/ccxt/releases)
- [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ccxt/ccxt/compare/1.18.1043...1.18.1063)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 91da6e45c..4666fc053 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,6 +1,6 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.18.1043
+ccxt==1.18.1063
 SQLAlchemy==1.3.7
 python-telegram-bot==11.1.0
 arrow==0.14.5

From 351740fc80761176891fb7bbe463672c540e8384 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 17 Aug 2019 17:27:14 +0200
Subject: [PATCH 16/33] Change pyup to every month (should ideally not find
 anything ...)

---
 .pyup.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pyup.yml b/.pyup.yml
index b1b721113..7ab3e5eca 100644
--- a/.pyup.yml
+++ b/.pyup.yml
@@ -14,7 +14,7 @@ pin: True
 # update schedule
 # default: empty
 # allowed: "every day", "every week", ..
-schedule: "every week"
+schedule: "every month"
 
 
 search: False

From 0e87cc8c84c72a75d7889f02fb584f46036400b0 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 17 Aug 2019 19:30:03 +0200
Subject: [PATCH 17/33] Remove pyup.yml

---
 .pyup.yml | 37 -------------------------------------
 1 file changed, 37 deletions(-)
 delete mode 100644 .pyup.yml

diff --git a/.pyup.yml b/.pyup.yml
deleted file mode 100644
index 7ab3e5eca..000000000
--- a/.pyup.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-# autogenerated pyup.io config file
-# see https://pyup.io/docs/configuration/ for all available options
-
-# configure updates globally
-# default: all
-# allowed: all, insecure, False
-update: all
-
-# configure dependency pinning globally
-# default: True
-# allowed: True, False
-pin: True
-
-# update schedule
-# default: empty
-# allowed: "every day", "every week", ..
-schedule: "every month"
-
-
-search: False
-# Specify requirement files by hand, default is empty
-# default: empty
-# allowed: list
-requirements:
-  - requirements.txt
-  - requirements-dev.txt
-  - requirements-plot.txt
-  - requirements-common.txt
-
-
-# configure the branch prefix the bot is using
-# default: pyup-
-branch_prefix: pyup/
-
-# allow to close stale PRs
-# default: True
-close_prs: True

From 7fa6d804ce5a7352ec92f6114279acc90118aab6 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 17 Aug 2019 19:48:55 +0200
Subject: [PATCH 18/33] Add note explaining how / when docker images are
 rebuild

---
 docs/docker.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/docker.md b/docs/docker.md
index 615d31796..923dec1e2 100644
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -26,6 +26,10 @@ To update the image, simply run the above commands again and restart your runnin
 
 Should you require additional libraries, please [build the image yourself](#build-your-own-docker-image).
 
+!!! Note Docker image update frequency
+    The official docker images with tags `master`, `develop` and `latest` are automatically rebuild once a week to keep the base image uptodate.
+    In addition to that, every merge to `develop` will trigger a rebuild for `develop` and `latest`.
+
 ### Prepare the configuration files
 
 Even though you will use docker, you'll still need some files from the github repository.

From 8a2a8ab8b5da8392857898684cf8d983e3cd1881 Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Sun, 18 Aug 2019 12:47:19 +0300
Subject: [PATCH 19/33] docstring for ohlcv improved

---
 freqtrade/data/dataprovider.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py
index e806f5aa7..b67aba045 100644
--- a/freqtrade/data/dataprovider.py
+++ b/freqtrade/data/dataprovider.py
@@ -45,7 +45,7 @@ class DataProvider():
     def ohlcv(self, pair: str, ticker_interval: str = None, copy: bool = True) -> DataFrame:
         """
         Get ohlcv data for the given pair as DataFrame
-        Please check `self.available_pairs` to verify which pairs are currently cached.
+        Please use the `available_pairs` method to verify which pairs are currently cached.
         :param pair: pair to get the data for
         :param ticker_interval: ticker interval to get data for
         :param copy: copy dataframe before returning if True.

From 310e4387065bf1407c8226a0bec9f79dc62e98ec Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Sun, 18 Aug 2019 12:55:31 +0300
Subject: [PATCH 20/33] logging message improved

---
 freqtrade/data/dataprovider.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py
index b67aba045..b904ba985 100644
--- a/freqtrade/data/dataprovider.py
+++ b/freqtrade/data/dataprovider.py
@@ -85,7 +85,7 @@ class DataProvider():
             # Get historic ohlcv data (cached on disk).
             data = self.historic_ohlcv(pair=pair, ticker_interval=ticker_interval)
         if len(data) == 0:
-            logger.warning(f"No data found for pair {pair}")
+            logger.warning(f"No data found for ({pair}, {ticker_interval}).")
         return data
 
     def ticker(self, pair: str):

From 407a3bca6222bd820d55656e73398dc4e6bef102 Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Sun, 18 Aug 2019 13:00:37 +0300
Subject: [PATCH 21/33] implementation of ohlcv optimized

---
 freqtrade/data/dataprovider.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py
index b904ba985..5b71c21a8 100644
--- a/freqtrade/data/dataprovider.py
+++ b/freqtrade/data/dataprovider.py
@@ -52,11 +52,10 @@ class DataProvider():
                      Use False only for read-only operations (where the dataframe is not modified)
         """
         if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
-            pairtick = (pair, ticker_interval or self._config['ticker_interval'])
-            if pairtick in self.available_pairs:
-                return self._exchange.klines(pairtick, copy=copy)
-
-        return DataFrame()
+            return self._exchange.klines((pair, ticker_interval or self._config['ticker_interval']),
+                                         copy=copy)
+        else:
+            return DataFrame()
 
     def historic_ohlcv(self, pair: str, ticker_interval: str = None) -> DataFrame:
         """

From d300964691977d34bc3618a8e75bb48b6ce863ae Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Sun, 18 Aug 2019 13:06:21 +0300
Subject: [PATCH 22/33] code formatting in test_dataprovider.py

---
 freqtrade/tests/data/test_dataprovider.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py
index 54aab7052..2272f69a3 100644
--- a/freqtrade/tests/data/test_dataprovider.py
+++ b/freqtrade/tests/data/test_dataprovider.py
@@ -13,6 +13,7 @@ def test_ohlcv(mocker, default_conf, ticker_history):
     exchange = get_patched_exchange(mocker, default_conf)
     exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history
     exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history
+
     dp = DataProvider(default_conf, exchange)
     assert dp.runmode == RunMode.DRY_RUN
     assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", ticker_interval))
@@ -37,11 +38,9 @@ def test_ohlcv(mocker, default_conf, ticker_history):
 
 
 def test_historic_ohlcv(mocker, default_conf, ticker_history):
-
     historymock = MagicMock(return_value=ticker_history)
     mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock)
 
-    # exchange = get_patched_exchange(mocker, default_conf)
     dp = DataProvider(default_conf, None)
     data = dp.historic_ohlcv("UNITTEST/BTC", "5m")
     assert isinstance(data, DataFrame)
@@ -57,6 +56,7 @@ def test_get_pair_dataframe(mocker, default_conf, ticker_history):
     exchange = get_patched_exchange(mocker, default_conf)
     exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history
     exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history
+
     dp = DataProvider(default_conf, exchange)
     assert dp.runmode == RunMode.DRY_RUN
     assert ticker_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval))
@@ -86,12 +86,11 @@ def test_get_pair_dataframe(mocker, default_conf, ticker_history):
 
 def test_available_pairs(mocker, default_conf, ticker_history):
     exchange = get_patched_exchange(mocker, default_conf)
-
     ticker_interval = default_conf["ticker_interval"]
     exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history
     exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history
-    dp = DataProvider(default_conf, exchange)
 
+    dp = DataProvider(default_conf, exchange)
     assert len(dp.available_pairs) == 2
     assert dp.available_pairs == [
         ("XRP/BTC", ticker_interval),

From 8e96ac876597b686e66df3e4a238f324a9be95d8 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 18 Aug 2019 15:45:30 +0200
Subject: [PATCH 23/33] Split exception tests for create_order

---
 freqtrade/tests/exchange/test_exchange.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 6b833054d..123f1fcfd 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -779,7 +779,13 @@ def test_sell_prod(default_conf, mocker, exchange_name):
     with pytest.raises(DependencyException):
         api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200)
+        exchange.sell(pair='ETH/BTC', ordertype='limit', amount=1, rate=200)
+
+    # Market orders don't require price, so the behaviour is slightly different
+    with pytest.raises(DependencyException):
+        api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+        exchange.sell(pair='ETH/BTC', ordertype='market', amount=1, rate=200)
 
     with pytest.raises(TemporaryError):
         api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No Connection"))

From ee7ba96e8524558d8d7ed068fb939d18aac0a711 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 18 Aug 2019 15:46:10 +0200
Subject: [PATCH 24/33] Don't do calculations in exception handlers when one
 element can be None

fixes #2011
---
 freqtrade/exchange/exchange.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index 5119e0fcd..67a79178f 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -408,12 +408,12 @@ class Exchange(object):
         except ccxt.InsufficientFunds as e:
             raise DependencyException(
                 f'Insufficient funds to create {ordertype} {side} order on market {pair}.'
-                f'Tried to {side} amount {amount} at rate {rate} (total {rate * amount}).'
+                f'Tried to {side} amount {amount} at rate {rate}.'
                 f'Message: {e}') from e
         except ccxt.InvalidOrder as e:
             raise DependencyException(
                 f'Could not create {ordertype} {side} order on market {pair}.'
-                f'Tried to {side} amount {amount} at rate {rate} (total {rate * amount}).'
+                f'Tried to {side} amount {amount} at rate {rate}.'
                 f'Message: {e}') from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(

From 045ac1019e37359bf6d04f9989af50493122b434 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 18 Aug 2019 15:48:20 +0200
Subject: [PATCH 25/33] Split test for buy-orders too

---
 freqtrade/exchange/exchange.py            | 2 +-
 freqtrade/tests/exchange/test_exchange.py | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index 67a79178f..6e281b9b2 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -472,7 +472,7 @@ class Exchange(object):
 
         order = self.create_order(pair, ordertype, 'sell', amount, rate, params)
         logger.info('stoploss limit order added for %s. '
-                    'stop price: %s. limit: %s' % (pair, stop_price, rate))
+                    'stop price: %s. limit: %s', pair, stop_price, rate)
         return order
 
     @retrier
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 123f1fcfd..397326258 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -656,7 +656,13 @@ def test_buy_prod(default_conf, mocker, exchange_name):
     with pytest.raises(DependencyException):
         api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.buy(pair='ETH/BTC', ordertype=order_type,
+        exchange.buy(pair='ETH/BTC', ordertype='limit',
+                     amount=1, rate=200, time_in_force=time_in_force)
+
+    with pytest.raises(DependencyException):
+        api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+        exchange.buy(pair='ETH/BTC', ordertype='market',
                      amount=1, rate=200, time_in_force=time_in_force)
 
     with pytest.raises(TemporaryError):

From ddfadbb69ee2bf18191d94d053d3ef2301286030 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 18 Aug 2019 16:10:10 +0200
Subject: [PATCH 26/33] Validate configuration consistency after loading
 strategy

---
 freqtrade/configuration/__init__.py           |  1 +
 .../{json_schema.py => config_validation.py}  | 35 ++++++++++++++++-
 freqtrade/configuration/configuration.py      | 38 +++----------------
 freqtrade/freqtradebot.py                     |  2 +
 freqtrade/tests/test_configuration.py         | 14 +++----
 5 files changed, 47 insertions(+), 43 deletions(-)
 rename freqtrade/configuration/{json_schema.py => config_validation.py} (53%)

diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py
index 7b476d173..ac59421a7 100644
--- a/freqtrade/configuration/__init__.py
+++ b/freqtrade/configuration/__init__.py
@@ -1,3 +1,4 @@
 from freqtrade.configuration.arguments import Arguments  # noqa: F401
 from freqtrade.configuration.timerange import TimeRange  # noqa: F401
 from freqtrade.configuration.configuration import Configuration  # noqa: F401
+from freqtrade.configuration.config_validation import validate_config_consistency  # noqa: F401
diff --git a/freqtrade/configuration/json_schema.py b/freqtrade/configuration/config_validation.py
similarity index 53%
rename from freqtrade/configuration/json_schema.py
rename to freqtrade/configuration/config_validation.py
index 4c6f4a4a0..bda60f90b 100644
--- a/freqtrade/configuration/json_schema.py
+++ b/freqtrade/configuration/config_validation.py
@@ -4,7 +4,7 @@ from typing import Any, Dict
 from jsonschema import Draft4Validator, validators
 from jsonschema.exceptions import ValidationError, best_match
 
-from freqtrade import constants
+from freqtrade import constants, OperationalException
 
 
 logger = logging.getLogger(__name__)
@@ -51,3 +51,36 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]:
         raise ValidationError(
             best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message
         )
+
+
+def validate_config_consistency(conf: Dict[str, Any]) -> None:
+    """
+    Validate the configuration consistency.
+    Should be ran after loading both configuration and strategy,
+    since strategies can set certain configuration settings too.
+    :param conf: Config in JSON format
+    :return: Returns None if everything is ok, otherwise throw an OperationalException
+    """
+    # validating trailing stoploss
+    _validate_trailing_stoploss(conf)
+
+
+def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
+
+    # Skip if trailing stoploss is not activated
+    if not conf.get('trailing_stop', False):
+        return
+
+    tsl_positive = float(conf.get('trailing_stop_positive', 0))
+    tsl_offset = float(conf.get('trailing_stop_positive_offset', 0))
+    tsl_only_offset = conf.get('trailing_only_offset_is_reached', False)
+
+    if tsl_only_offset:
+        if tsl_positive == 0.0:
+            raise OperationalException(
+                f'The config trailing_only_offset_is_reached needs '
+                'trailing_stop_positive_offset to be more than 0 in your config.')
+    if tsl_positive > 0 and 0 < tsl_offset <= tsl_positive:
+        raise OperationalException(
+            f'The config trailing_stop_positive_offset needs '
+            'to be greater than trailing_stop_positive_offset in your config.')
diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py
index c95246fc0..486857153 100644
--- a/freqtrade/configuration/configuration.py
+++ b/freqtrade/configuration/configuration.py
@@ -6,10 +6,11 @@ import warnings
 from argparse import Namespace
 from typing import Any, Callable, Dict, List, Optional
 
-from freqtrade import OperationalException, constants
+from freqtrade import constants
 from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.create_datadir import create_datadir
-from freqtrade.configuration.json_schema import validate_config_schema
+from freqtrade.configuration.config_validation import (validate_config_schema,
+                                                       validate_config_consistency)
 from freqtrade.configuration.load_config import load_config_file
 from freqtrade.loggers import setup_logging
 from freqtrade.misc import deep_merge_dicts
@@ -77,8 +78,6 @@ class Configuration(object):
         # Load all configs
         config: Dict[str, Any] = Configuration.from_files(self.args.config)
 
-        self._validate_config_consistency(config)
-
         self._process_common_options(config)
 
         self._process_optimize_options(config)
@@ -87,6 +86,8 @@ class Configuration(object):
 
         self._process_runmode(config)
 
+        validate_config_consistency(config)
+
         return config
 
     def _process_logging_options(self, config: Dict[str, Any]) -> None:
@@ -285,35 +286,6 @@ class Configuration(object):
 
         config.update({'runmode': self.runmode})
 
-    def _validate_config_consistency(self, conf: Dict[str, Any]) -> None:
-        """
-        Validate the configuration consistency
-        :param conf: Config in JSON format
-        :return: Returns None if everything is ok, otherwise throw an OperationalException
-        """
-        # validating trailing stoploss
-        self._validate_trailing_stoploss(conf)
-
-    def _validate_trailing_stoploss(self, conf: Dict[str, Any]) -> None:
-
-        # Skip if trailing stoploss is not activated
-        if not conf.get('trailing_stop', False):
-            return
-
-        tsl_positive = float(conf.get('trailing_stop_positive', 0))
-        tsl_offset = float(conf.get('trailing_stop_positive_offset', 0))
-        tsl_only_offset = conf.get('trailing_only_offset_is_reached', False)
-
-        if tsl_only_offset:
-            if tsl_positive == 0.0:
-                raise OperationalException(
-                    f'The config trailing_only_offset_is_reached needs '
-                    'trailing_stop_positive_offset to be more than 0 in your config.')
-        if tsl_positive > 0 and 0 < tsl_offset <= tsl_positive:
-            raise OperationalException(
-                f'The config trailing_stop_positive_offset needs '
-                'to be greater than trailing_stop_positive_offset in your config.')
-
     def _args_to_config(self, config: Dict[str, Any], argname: str,
                         logstring: str, logfun: Optional[Callable] = None,
                         deprecated_msg: Optional[str] = None) -> None:
diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 68b45d96f..af29604f5 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -16,6 +16,7 @@ from freqtrade import (DependencyException, OperationalException, InvalidOrderEx
 from freqtrade.data.converter import order_book_to_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
+from freqtrade.configuration import validate_config_consistency
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
 from freqtrade.persistence import Trade
 from freqtrade.rpc import RPCManager, RPCMessageType
@@ -50,6 +51,7 @@ class FreqtradeBot(object):
         self.config = config
 
         self.strategy: IStrategy = StrategyResolver(self.config).strategy
+        validate_config_consistency(config)
 
         self.rpc: RPCManager = RPCManager(self)
 
diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py
index 8cbd02ece..3a0077ef2 100644
--- a/freqtrade/tests/test_configuration.py
+++ b/freqtrade/tests/test_configuration.py
@@ -2,7 +2,6 @@
 import json
 import logging
 import warnings
-from argparse import Namespace
 from copy import deepcopy
 from pathlib import Path
 from unittest.mock import MagicMock
@@ -11,10 +10,10 @@ import pytest
 from jsonschema import Draft4Validator, ValidationError, validate
 
 from freqtrade import OperationalException, constants
-from freqtrade.configuration import Arguments, Configuration
+from freqtrade.configuration import Arguments, Configuration, validate_config_consistency
 from freqtrade.configuration.check_exchange import check_exchange
+from freqtrade.configuration.config_validation import validate_config_schema
 from freqtrade.configuration.create_datadir import create_datadir
-from freqtrade.configuration.json_schema import validate_config_schema
 from freqtrade.configuration.load_config import load_config_file
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
 from freqtrade.loggers import _set_loggers
@@ -625,21 +624,18 @@ def test_validate_tsl(default_conf):
     with pytest.raises(OperationalException,
                        match=r'The config trailing_only_offset_is_reached needs '
                        'trailing_stop_positive_offset to be more than 0 in your config.'):
-        configuration = Configuration(Namespace())
-        configuration._validate_config_consistency(default_conf)
+        validate_config_consistency(default_conf)
 
     default_conf['trailing_stop_positive_offset'] = 0.01
     default_conf['trailing_stop_positive'] = 0.015
     with pytest.raises(OperationalException,
                        match=r'The config trailing_stop_positive_offset needs '
                        'to be greater than trailing_stop_positive_offset in your config.'):
-        configuration = Configuration(Namespace())
-        configuration._validate_config_consistency(default_conf)
+        validate_config_consistency(default_conf)
 
     default_conf['trailing_stop_positive'] = 0.01
     default_conf['trailing_stop_positive_offset'] = 0.015
-    Configuration(Namespace())
-    configuration._validate_config_consistency(default_conf)
+    validate_config_consistency(default_conf)
 
 
 def test_load_config_test_comments() -> None:

From 611850bf91a343d056259f3bb6ab7bc9b683c59f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 18 Aug 2019 16:19:24 +0200
Subject: [PATCH 27/33] Add edge/dynamic_whitelist validation

---
 freqtrade/configuration/config_validation.py | 16 ++++++++++++++++
 freqtrade/tests/test_configuration.py        | 16 ++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py
index bda60f90b..92846b704 100644
--- a/freqtrade/configuration/config_validation.py
+++ b/freqtrade/configuration/config_validation.py
@@ -63,6 +63,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None:
     """
     # validating trailing stoploss
     _validate_trailing_stoploss(conf)
+    _validate_edge(conf)
 
 
 def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
@@ -84,3 +85,18 @@ def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
         raise OperationalException(
             f'The config trailing_stop_positive_offset needs '
             'to be greater than trailing_stop_positive_offset in your config.')
+
+
+def _validate_edge(conf: Dict[str, Any]) -> None:
+    """
+    Edge and Dynamic whitelist should not both be enabled, since edge overrides dynamic whitelists.
+    """
+
+    if not conf.get('edge', {}).get('enabled'):
+        return
+
+    if conf.get('pairlist', {}).get('method') == 'VolumePairList':
+        raise OperationalException(
+            "Edge and VolumePairList are incompatible, "
+            "Edge will override whatever pairs VolumePairlist selects."
+        )
diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py
index 3a0077ef2..19d2a28ee 100644
--- a/freqtrade/tests/test_configuration.py
+++ b/freqtrade/tests/test_configuration.py
@@ -638,6 +638,22 @@ def test_validate_tsl(default_conf):
     validate_config_consistency(default_conf)
 
 
+def test_validate_edge(edge_conf):
+    edge_conf.update({"pairlist": {
+        "method": "VolumePairList",
+    }})
+
+    with pytest.raises(OperationalException,
+                       match="Edge and VolumePairList are incompatible, "
+                       "Edge will override whatever pairs VolumePairlist selects."):
+        validate_config_consistency(edge_conf)
+
+    edge_conf.update({"pairlist": {
+        "method": "StaticPairList",
+    }})
+    validate_config_consistency(edge_conf)
+
+
 def test_load_config_test_comments() -> None:
     """
     Load config with comments

From b6462cd51f707f4160fa77d8bf2eb36c73a7695e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 18 Aug 2019 16:22:18 +0200
Subject: [PATCH 28/33] Add explaining comment

---
 freqtrade/freqtradebot.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index af29604f5..e5ecef8bf 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -51,6 +51,8 @@ class FreqtradeBot(object):
         self.config = config
 
         self.strategy: IStrategy = StrategyResolver(self.config).strategy
+
+        # Check config consistency here since strategies can set certain options
         validate_config_consistency(config)
 
         self.rpc: RPCManager = RPCManager(self)

From d785d7637085edcaf382a26df2d71cb8612f6a7e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 18 Aug 2019 18:06:36 +0200
Subject: [PATCH 29/33] make VolumePairlist less verbose

no need to print the full whitelist on every iteration
---
 freqtrade/constants.py               | 1 -
 freqtrade/pairlist/VolumePairList.py | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/constants.py b/freqtrade/constants.py
index 9b73adcfe..bd3ba21fd 100644
--- a/freqtrade/constants.py
+++ b/freqtrade/constants.py
@@ -5,7 +5,6 @@ bot constants
 """
 DEFAULT_CONFIG = 'config.json'
 DEFAULT_EXCHANGE = 'bittrex'
-DYNAMIC_WHITELIST = 20  # pairs
 PROCESS_THROTTLE_SECS = 5  # sec
 DEFAULT_TICKER_INTERVAL = 5  # min
 HYPEROPT_EPOCH = 100  # epochs
diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py
index 9a2e2eac4..b9b7977ab 100644
--- a/freqtrade/pairlist/VolumePairList.py
+++ b/freqtrade/pairlist/VolumePairList.py
@@ -55,7 +55,6 @@ class VolumePairList(IPairList):
         # Generate dynamic whitelist
         self._whitelist = self._gen_pair_whitelist(
             self._config['stake_currency'], self._sort_key)[:self._number_pairs]
-        logger.info(f"Searching pairs: {self._whitelist}")
 
     @cached(TTLCache(maxsize=1, ttl=1800))
     def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]:
@@ -92,4 +91,6 @@ class VolumePairList(IPairList):
                     valid_tickers.remove(t)
 
         pairs = [s['symbol'] for s in valid_tickers]
+        logger.info(f"Searching pairs: {self._whitelist}")
+
         return pairs

From ea4db0ffb6dd0007b4df471a3e3f8f90d9e3599d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 18 Aug 2019 18:06:57 +0200
Subject: [PATCH 30/33] Pass object-name to loader to fix logging

---
 freqtrade/resolvers/iresolver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py
index 841c3cf43..310c54015 100644
--- a/freqtrade/resolvers/iresolver.py
+++ b/freqtrade/resolvers/iresolver.py
@@ -29,7 +29,8 @@ class IResolver(object):
         """
 
         # Generate spec based on absolute path
-        spec = importlib.util.spec_from_file_location('unknown', str(module_path))
+        # Pass object_name as first argument to have logging print a reasonable name.
+        spec = importlib.util.spec_from_file_location(object_name, str(module_path))
         module = importlib.util.module_from_spec(spec)
         try:
             spec.loader.exec_module(module)  # type: ignore # importlib does not use typehints

From a4ede02cedd22ddc1147f1fcffd224b554af970d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 18 Aug 2019 19:38:23 +0200
Subject: [PATCH 31/33] Gracefully handle problems with dry-run orders

---
 freqtrade/exchange/exchange.py            | 9 +++++++--
 freqtrade/tests/exchange/test_exchange.py | 3 +++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index 5119e0fcd..7ae40381d 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -696,8 +696,13 @@ class Exchange(object):
     @retrier
     def get_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
-            order = self._dry_run_open_orders[order_id]
-            return order
+            try:
+                order = self._dry_run_open_orders[order_id]
+                return order
+            except KeyError as e:
+                # Gracefully handle errors with dry-run orders.
+                raise InvalidOrderException(
+                    f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e
         try:
             return self._api.fetch_order(order_id, pair)
         except ccxt.InvalidOrder as e:
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 6b833054d..6a7dfa04b 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -1328,6 +1328,9 @@ def test_get_order(default_conf, mocker, exchange_name):
     print(exchange.get_order('X', 'TKN/BTC'))
     assert exchange.get_order('X', 'TKN/BTC').myid == 123
 
+    with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'):
+        exchange.get_order('Y', 'TKN/BTC')
+
     default_conf['dry_run'] = False
     api_mock = MagicMock()
     api_mock.fetch_order = MagicMock(return_value=456)

From 9ad9ce0da1b46a0822cba4f100257a177211530f Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 19 Aug 2019 10:52:53 +0000
Subject: [PATCH 32/33] Bump ccxt from 1.18.1063 to 1.18.1068

Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.18.1063 to 1.18.1068.
- [Release notes](https://github.com/ccxt/ccxt/releases)
- [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ccxt/ccxt/compare/1.18.1063...1.18.1068)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 4666fc053..3d80c3ef5 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,6 +1,6 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.18.1063
+ccxt==1.18.1068
 SQLAlchemy==1.3.7
 python-telegram-bot==11.1.0
 arrow==0.14.5

From 70b1a05d976aad8a7c808ba7ee8a27156081b7b8 Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Tue, 20 Aug 2019 01:32:02 +0300
Subject: [PATCH 33/33] example in the docs changed

---
 docs/strategy-customization.md | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md
index 0d08bdd02..d71ebfded 100644
--- a/docs/strategy-customization.md
+++ b/docs/strategy-customization.md
@@ -274,27 +274,24 @@ Please always check the mode of operation to select the correct method to get da
 
 #### Possible options for DataProvider
 
-- `available_pairs` - Property with tuples listing cached pairs with their intervals. (pair, interval)
-- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for all pairs in the whitelist, returns DataFrame or empty DataFrame
-- `historic_ohlcv(pair, ticker_interval)` - Data stored on disk
+- `available_pairs` - Property with tuples listing cached pairs with their intervals (pair, interval).
+- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for the pair, returns DataFrame or empty DataFrame.
+- `historic_ohlcv(pair, ticker_interval)` - Returns historical data stored on disk.
+- `get_pair_dataframe(pair, ticker_interval)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes).
 - `runmode` - Property containing the current runmode.
 
-#### ohlcv / historic_ohlcv
+#### Example: fetch live ohlcv / historic data for the first informative pair
 
 ``` python
 if self.dp:
-    if self.dp.runmode in ('live', 'dry_run'):
-        if (f'{self.stake_currency}/BTC', self.ticker_interval) in self.dp.available_pairs:
-            data_eth = self.dp.ohlcv(pair='{self.stake_currency}/BTC',
-                                     ticker_interval=self.ticker_interval)
-    else:
-        # Get historic ohlcv data (cached on disk).
-        history_eth = self.dp.historic_ohlcv(pair='{self.stake_currency}/BTC',
-                                             ticker_interval='1h')
+    inf_pair, inf_timeframe = self.informative_pairs()[0]
+    informative = self.dp.get_pair_dataframe(pair=inf_pair,
+                                             ticker_interval=inf_timeframe)
 ```
 
 !!! Warning Warning about backtesting
-    Be carefull when using dataprovider in backtesting. `historic_ohlcv()` provides the full time-range in one go,
+    Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()`
+    for the backtesting runmode) provides the full time-range in one go,
     so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode).
 
 !!! Warning Warning in hyperopt