From e2b83624a397b85807d23f7b5353ade9727e54a3 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 17 May 2019 19:05:36 +0300 Subject: [PATCH 01/23] data/history cleanup --- freqtrade/data/history.py | 38 ++++++++++++++++------------ freqtrade/tests/data/test_history.py | 29 ++++++++++++++++----- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 4dba1b760..86d3c3071 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -90,13 +90,8 @@ def load_pair_history(pair: str, :return: DataFrame with ohlcv data """ - # If the user force the refresh of pairs + # The user forced the refresh of pairs if refresh_pairs: - if not exchange: - raise OperationalException("Exchange needs to be initialized when " - "calling load_data with refresh_pairs=True") - - logger.info('Download data for pair and store them in %s', datadir) download_pair_history(datadir=datadir, exchange=exchange, pair=pair, @@ -115,10 +110,11 @@ def load_pair_history(pair: str, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing) else: - logger.warning('No data for pair: "%s", Interval: %s. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' - 'script to download the data', - pair, ticker_interval) + logger.warning( + f'No history data for pair: "{pair}", interval: {ticker_interval}. ' + 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'script to download the data' + ) return None @@ -190,7 +186,7 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str, def download_pair_history(datadir: Optional[Path], - exchange: Exchange, + exchange: Optional[Exchange], pair: str, ticker_interval: str = '5m', timerange: Optional[TimeRange] = None) -> bool: @@ -201,18 +197,26 @@ def download_pair_history(datadir: Optional[Path], the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data + :param pair: pair to download :param ticker_interval: ticker interval :param timerange: range of time to download :return: bool with success state - """ + if not exchange: + raise OperationalException( + "Exchange needs to be initialized when downloading pair history data" + ) + try: path = make_testdata_path(datadir) filepair = pair.replace("/", "_") filename = path.joinpath(f'{filepair}-{ticker_interval}.json') - logger.info('Download the pair: "%s", Interval: %s', pair, ticker_interval) + logger.info( + f'Download history data for pair: "{pair}", interval: {ticker_interval} ' + f'and store in {datadir}.' + ) data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange) @@ -231,7 +235,9 @@ def download_pair_history(datadir: Optional[Path], misc.file_dump_json(filename, data) return True - except BaseException: - logger.info('Failed to download the pair: "%s", Interval: %s', - pair, ticker_interval) + + except Exception: + logger.error( + f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}.' + ) return False diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 14ec99042..15442f577 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -59,7 +59,11 @@ def _clean_test_file(file: str) -> None: def test_load_data_30min_ticker(mocker, caplog, default_conf) -> None: ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='30m', datadir=None) assert isinstance(ld, DataFrame) - assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 30m', caplog.record_tuples) + assert not log_has( + 'Download history data for pair: "UNITTEST/BTC", interval: 30m ' + 'and store in None.', + caplog.record_tuples + ) def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: @@ -67,7 +71,7 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: assert not isinstance(ld, DataFrame) assert ld is None assert log_has( - 'No data for pair: "UNITTEST/BTC", Interval: 7m. ' + 'No history data for pair: "UNITTEST/BTC", interval: 7m. ' 'Use --refresh-pairs-cached option or download_backtest_data.py ' 'script to download the data', caplog.record_tuples @@ -80,7 +84,11 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: _backup_file(file, copy_file=True) history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC']) assert os.path.isfile(file) is True - assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 1m', caplog.record_tuples) + assert not log_has( + 'Download history data for pair: "UNITTEST/BTC", interval: 1m ' + 'and store in None.', + caplog.record_tuples + ) _clean_test_file(file) @@ -100,7 +108,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau pair='MEME/BTC') assert os.path.isfile(file) is False assert log_has( - 'No data for pair: "MEME/BTC", Interval: 1m. ' + 'No history data for pair: "MEME/BTC", interval: 1m. ' 'Use --refresh-pairs-cached option or download_backtest_data.py ' 'script to download the data', caplog.record_tuples @@ -113,7 +121,11 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau exchange=exchange, pair='MEME/BTC') assert os.path.isfile(file) is True - assert log_has('Download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) + assert log_has( + 'Download history data for pair: "MEME/BTC", interval: 1m ' + 'and store in None.', + caplog.record_tuples + ) with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): history.load_pair_history(datadir=None, ticker_interval='1m', @@ -293,7 +305,7 @@ def test_download_pair_history2(mocker, default_conf) -> None: def test_download_backtesting_data_exception(ticker_history, mocker, caplog, default_conf) -> None: mocker.patch('freqtrade.exchange.Exchange.get_history', - side_effect=BaseException('File Error')) + side_effect=Exception('File Error')) exchange = get_patched_exchange(mocker, default_conf) @@ -308,7 +320,10 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) - assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) + assert log_has( + 'Failed to download history data for pair: "MEME/BTC", interval: 1m.', + caplog.record_tuples + ) def test_load_tickerdata_file() -> None: From 8d8b4a69b759c0774e913abc26c82105c18d6f3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 May 2019 09:03:56 +0200 Subject: [PATCH 02/23] Clearly warn about using future data during strategy development --- docs/bot-optimization.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 5e080eab1..97166a736 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -53,6 +53,12 @@ file as reference.** It is therefore best to use vectorized operations (across the whole dataframe, not loops) and avoid index referencing (`df.iloc[-1]`), but instead use `df.shift()` to get to the previous candle. +!!! Warning Using future data + Since backtesting passes the full time interval to the `populate_*()` methods, the strategy author + needs to take care to avoid having the strategy utilize data from the future. + Samples for usage of future data are `dataframe.shift(-1)`, `dataframe.resample("1h")` (this uses the left border of the interval, so moves data from an hour to the start of the hour). + They all use data which is not available during regular operations, so these strategies will perform well during backtesting, but will fail / perform badly dry-run tests. + ### Customize Indicators Buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `populate_indicators()` from your strategy file. From f93e6ad0f6461fee649cfaeeed2f2400ada8d0e1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 May 2019 09:07:43 +0200 Subject: [PATCH 03/23] Rename strategy customization file --- docs/bot-usage.md | 4 ++-- docs/{bot-optimization.md => strategy-customization.md} | 0 mkdocs.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename docs/{bot-optimization.md => strategy-customization.md} (100%) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 2b2fef640..cb98e1ea5 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -103,7 +103,7 @@ If the bot does not find your strategy file, it will display in an error message the reason (File not found, or errors in your code). Learn more about strategy file in -[optimize your bot](bot-optimization.md). +[Strategy Customization](strategy-customization.md). ### How to use **--strategy-path**? @@ -296,4 +296,4 @@ in [misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc. ## Next step The optimal strategy of the bot will change with time depending of the market trends. The next step is to -[optimize your bot](bot-optimization.md). +[Strategy Customization](strategy-customization.md). diff --git a/docs/bot-optimization.md b/docs/strategy-customization.md similarity index 100% rename from docs/bot-optimization.md rename to docs/strategy-customization.md diff --git a/mkdocs.yml b/mkdocs.yml index ecac265c1..9932ff316 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,7 +3,7 @@ nav: - About: index.md - Installation: installation.md - Configuration: configuration.md - - Custom Strategy: bot-optimization.md + - Strategy Customization: strategy-customization.md - Stoploss: stoploss.md - Start the bot: bot-usage.md - Control the bot: From fc96da869a4d0b5051eefd8e437b088dc35cf941 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 May 2019 16:07:16 +0200 Subject: [PATCH 04/23] Fix grammar messup --- docs/strategy-customization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 97166a736..51540f690 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -57,7 +57,7 @@ file as reference.** Since backtesting passes the full time interval to the `populate_*()` methods, the strategy author needs to take care to avoid having the strategy utilize data from the future. Samples for usage of future data are `dataframe.shift(-1)`, `dataframe.resample("1h")` (this uses the left border of the interval, so moves data from an hour to the start of the hour). - They all use data which is not available during regular operations, so these strategies will perform well during backtesting, but will fail / perform badly dry-run tests. + They all use data which is not available during regular operations, so these strategies will perform well during backtesting, but will fail / perform badly in dry-runs. ### Customize Indicators From e7b9bc6808be9dc54a94e43831a60a55dcc04013 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 20 May 2019 12:27:30 +0300 Subject: [PATCH 05/23] minor: remove noisy useless debug message --- freqtrade/persistence.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 6088dba72..e64e0b89c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -238,7 +238,6 @@ class Trade(_DECL_BASE): """ Adjust the max_rate and min_rate. """ - logger.debug("Adjusting min/max rates") self.max_rate = max(current_price, self.max_rate or self.open_rate) self.min_rate = min(current_price, self.min_rate or self.open_rate) From 703fdb2bc637efa73e1515afcf8628969cb7a3aa Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:07 +0000 Subject: [PATCH 06/23] Update scipy from 1.2.1 to 1.3.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 78585f8f5..da87f56d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ numpy==1.16.3 pandas==0.24.2 -scipy==1.2.1 +scipy==1.3.0 From de95e50804addaafb785f1392a0a10075f523a0b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:08 +0000 Subject: [PATCH 07/23] Update ccxt from 1.18.523 to 1.18.551 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 4f7309a6a..8139d2cbb 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.523 +ccxt==1.18.551 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.1 From 3404bb1865b31823bd978ec3084f66c6955f63e2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:09 +0000 Subject: [PATCH 08/23] Update arrow from 0.13.1 to 0.13.2 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 8139d2cbb..4ec3a0092 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -3,7 +3,7 @@ ccxt==1.18.551 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 -arrow==0.13.1 +arrow==0.13.2 cachetools==3.1.0 requests==2.21.0 urllib3==1.24.2 # pyup: ignore From 34c7ac8926079673cbf92f7a204f08daff38231b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:10 +0000 Subject: [PATCH 09/23] Update requests from 2.21.0 to 2.22.0 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 4ec3a0092..deea30faf 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -5,7 +5,7 @@ SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.2 cachetools==3.1.0 -requests==2.21.0 +requests==2.22.0 urllib3==1.24.2 # pyup: ignore wrapt==1.11.1 scikit-learn==0.21.0 From 5b24ac78981946bf334d19e28c14a180add4431c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:11 +0000 Subject: [PATCH 10/23] Update scikit-learn from 0.21.0 to 0.21.1 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index deea30faf..4ba4426b7 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -8,7 +8,7 @@ cachetools==3.1.0 requests==2.22.0 urllib3==1.24.2 # pyup: ignore wrapt==1.11.1 -scikit-learn==0.21.0 +scikit-learn==0.21.1 joblib==0.13.2 jsonschema==3.0.1 TA-Lib==0.4.17 From 04e13eed7dbc006bd17ea50dab6da94317a39750 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 May 2019 15:36:13 +0000 Subject: [PATCH 11/23] Update filelock from 3.0.10 to 3.0.12 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 4ba4426b7..3f755b8c0 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -17,7 +17,7 @@ coinmarketcap==5.0.3 # Required for hyperopt scikit-optimize==0.5.2 -filelock==3.0.10 +filelock==3.0.12 # find first, C search in arrays py_find_1st==1.1.3 From 96a34f753b0be9a86e09eb9fd00aeace08302bfd Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 May 2019 19:48:12 +0200 Subject: [PATCH 12/23] Adapt test to new output from arrow --- freqtrade/tests/rpc/test_rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 6ce543f3d..c3fcd62fb 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -119,7 +119,7 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: freqtradebot.create_trade() result = rpc._rpc_status_table() - assert 'just now' in result['Since'].all() + assert 'instantly' in result['Since'].all() assert 'ETH/BTC' in result['Pair'].all() assert '-0.59%' in result['Profit'].all() @@ -128,7 +128,7 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: # invalidate ticker cache rpc._freqtrade.exchange._cached_ticker = {} result = rpc._rpc_status_table() - assert 'just now' in result['Since'].all() + assert 'instantly' in result['Since'].all() assert 'ETH/BTC' in result['Pair'].all() assert 'nan%' in result['Profit'].all() From 11dce9128193184623d9a4896d503c351ef37faf Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 21 May 2019 20:49:02 +0300 Subject: [PATCH 13/23] data/history minor cleanup --- freqtrade/data/history.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 86d3c3071..27e68b533 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -63,12 +63,8 @@ def load_tickerdata_file( Load a pair from file, either .json.gz or .json :return tickerlist or None if unsuccesful """ - path = make_testdata_path(datadir) - pair_s = pair.replace('/', '_') - file = path.joinpath(f'{pair_s}-{ticker_interval}.json') - - pairdata = misc.file_load_json(file) - + filename = pair_data_filename(datadir, pair, ticker_interval) + pairdata = misc.file_load_json(filename) if not pairdata: return None @@ -142,11 +138,18 @@ def load_data(datadir: Optional[Path], return result -def make_testdata_path(datadir: Optional[Path]) -> Path: +def make_datadir_path(datadir: Optional[Path]) -> Path: """Return the path where testdata files are stored""" return datadir or (Path(__file__).parent.parent / "tests" / "testdata").resolve() +def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) -> Path: + path = make_datadir_path(datadir) + pair_s = pair.replace("/", "_") + filename = path.joinpath(f'{pair_s}-{ticker_interval}.json') + return filename + + def load_cached_data_for_updating(filename: Path, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: @@ -209,9 +212,7 @@ def download_pair_history(datadir: Optional[Path], ) try: - path = make_testdata_path(datadir) - filepair = pair.replace("/", "_") - filename = path.joinpath(f'{filepair}-{ticker_interval}.json') + filename = pair_data_filename(datadir, pair, ticker_interval) logger.info( f'Download history data for pair: "{pair}", interval: {ticker_interval} ' @@ -236,8 +237,9 @@ def download_pair_history(datadir: Optional[Path], misc.file_dump_json(filename, data) return True - except Exception: + except Exception as e: logger.error( - f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}.' + f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}. ' + f'Error: {e}' ) return False From 7cb753754b039898c3cfae2dd8049068cda6b4a2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 21 May 2019 20:49:19 +0300 Subject: [PATCH 14/23] tests adjusted --- freqtrade/tests/data/test_btanalysis.py | 4 ++-- freqtrade/tests/data/test_history.py | 7 ++++--- freqtrade/tests/test_misc.py | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index dd7cbe0d9..4c0426b93 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -2,12 +2,12 @@ import pytest from pandas import DataFrame from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data -from freqtrade.data.history import make_testdata_path +from freqtrade.data.history import make_datadir_path def test_load_backtest_data(): - filename = make_testdata_path(None) / "backtest-result_test.json" + filename = make_datadir_path(None) / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"] diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 15442f577..a37b42351 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -16,7 +16,7 @@ from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, load_tickerdata_file, - make_testdata_path, + make_datadir_path, trim_tickerlist) from freqtrade.misc import file_dump_json from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -136,7 +136,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau def test_testdata_path() -> None: - assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) + assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_datadir_path(None)) def test_load_cached_data_for_updating(mocker) -> None: @@ -321,7 +321,8 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def _clean_test_file(file1_1) _clean_test_file(file1_5) assert log_has( - 'Failed to download history data for pair: "MEME/BTC", interval: 1m.', + 'Failed to download history data for pair: "MEME/BTC", interval: 1m. ' + 'Error: File Error', caplog.record_tuples ) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 2da6b8718..c7bcf7edf 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, file_load_json, format_ms_time, shorten_date) -from freqtrade.data.history import load_tickerdata_file, make_testdata_path +from freqtrade.data.history import load_tickerdata_file, pair_data_filename from freqtrade.strategy.default_strategy import DefaultStrategy @@ -60,13 +60,13 @@ def test_file_dump_json(mocker) -> None: def test_file_load_json(mocker) -> None: # 7m .json does not exist - ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-7m.json')) + ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '7m')) assert not ret # 1m json exists (but no .gz exists) - ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-1m.json')) + ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '1m')) assert ret # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json - ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-8m.json')) + ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '8m')) assert ret From 98eeec31451c844bbdabc0a8c8dacd2474122596 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 22 May 2019 14:04:58 +0300 Subject: [PATCH 15/23] renaming of make_testdata_path reverted --- freqtrade/data/history.py | 4 ++-- freqtrade/tests/data/test_btanalysis.py | 4 ++-- freqtrade/tests/data/test_history.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 27e68b533..3bec63926 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -138,13 +138,13 @@ def load_data(datadir: Optional[Path], return result -def make_datadir_path(datadir: Optional[Path]) -> Path: +def make_testdata_path(datadir: Optional[Path]) -> Path: """Return the path where testdata files are stored""" return datadir or (Path(__file__).parent.parent / "tests" / "testdata").resolve() def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) -> Path: - path = make_datadir_path(datadir) + path = make_testdata_path(datadir) pair_s = pair.replace("/", "_") filename = path.joinpath(f'{pair_s}-{ticker_interval}.json') return filename diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index 4c0426b93..dd7cbe0d9 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -2,12 +2,12 @@ import pytest from pandas import DataFrame from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data -from freqtrade.data.history import make_datadir_path +from freqtrade.data.history import make_testdata_path def test_load_backtest_data(): - filename = make_datadir_path(None) / "backtest-result_test.json" + filename = make_testdata_path(None) / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"] diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index a37b42351..0d4210d3a 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -16,7 +16,7 @@ from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, load_tickerdata_file, - make_datadir_path, + make_testdata_path, trim_tickerlist) from freqtrade.misc import file_dump_json from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -136,7 +136,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau def test_testdata_path() -> None: - assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_datadir_path(None)) + assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) def test_load_cached_data_for_updating(mocker) -> None: From 2c9a519c5e359ba2fea353bfd29155c06c8a2d3b Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 22 May 2019 14:21:36 +0300 Subject: [PATCH 16/23] edge: handle properly the 'No trades' case --- freqtrade/edge/__init__.py | 1 + freqtrade/optimize/edge_cli.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4801c6cb3..5c7252d88 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -139,6 +139,7 @@ class Edge(): # If no trade found then exit if len(trades) == 0: + logger.info("No trades created.") return False # Fill missing, calculable columns, profit, duration , abs etc. diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 9b628cf2e..818c1e050 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -73,9 +73,10 @@ class EdgeCli(object): floatfmt=floatfmt, tablefmt="pipe") def start(self) -> None: - self.edge.calculate() - print('') # blank like for readability - print(self._generate_edge_table(self.edge._cached_pairs)) + result = self.edge.calculate() + if result: + print('') # blank like for readability + print(self._generate_edge_table(self.edge._cached_pairs)) def setup_configuration(args: Namespace) -> Dict[str, Any]: From 406e266bb4ea12caaaaf86b6ddde5369e7450fea Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 22 May 2019 14:34:35 +0300 Subject: [PATCH 17/23] typo in comment fixed --- freqtrade/optimize/edge_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 818c1e050..d37b930b8 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -75,7 +75,7 @@ class EdgeCli(object): def start(self) -> None: result = self.edge.calculate() if result: - print('') # blank like for readability + print('') # blank line for readability print(self._generate_edge_table(self.edge._cached_pairs)) From 6e1da13920eb3b1ed7ce501cf0c13f5c11a667df Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 22 May 2019 17:19:11 +0300 Subject: [PATCH 18/23] Log message changed --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 5c7252d88..053be6bc3 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -139,7 +139,7 @@ class Edge(): # If no trade found then exit if len(trades) == 0: - logger.info("No trades created.") + logger.info("No trades found.") return False # Fill missing, calculable columns, profit, duration , abs etc. From 7b074765ab059193ec75cacd8739d5e7e5ea6204 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 May 2019 19:48:22 +0200 Subject: [PATCH 19/23] Improve edge tests - cleanup test file --- freqtrade/tests/edge/test_edge.py | 128 ++++++++++++++++++++---------- 1 file changed, 87 insertions(+), 41 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index af8674188..a14e3282e 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -10,10 +10,11 @@ import numpy as np import pytest from pandas import DataFrame, to_datetime +from freqtrade import OperationalException from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.strategy.interface import SellType -from freqtrade.tests.conftest import get_patched_freqtradebot +from freqtrade.tests.conftest import get_patched_freqtradebot, log_has from freqtrade.tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset) @@ -30,7 +31,50 @@ ticker_start_time = arrow.get(2018, 10, 3) ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} +# Helpers for this test file + +def _validate_ohlc(buy_ohlc_sell_matrice): + for index, ohlc in enumerate(buy_ohlc_sell_matrice): + # if not high < open < low or not high < close < low + if not ohlc[3] >= ohlc[2] >= ohlc[4] or not ohlc[3] >= ohlc[5] >= ohlc[4]: + raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!') + return True + + +def _build_dataframe(buy_ohlc_sell_matrice): + _validate_ohlc(buy_ohlc_sell_matrice) + tickers = [] + for ohlc in buy_ohlc_sell_matrice: + ticker = { + 'date': ticker_start_time.shift( + minutes=( + ohlc[0] * + ticker_interval_in_minute)).timestamp * + 1000, + 'buy': ohlc[1], + 'open': ohlc[2], + 'high': ohlc[3], + 'low': ohlc[4], + 'close': ohlc[5], + 'sell': ohlc[6]} + tickers.append(ticker) + + frame = DataFrame(tickers) + frame['date'] = to_datetime(frame['date'], + unit='ms', + utc=True, + infer_datetime_format=True) + + return frame + + +def _time_on_candle(number): + return np.datetime64(ticker_start_time.shift( + minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') + + +# End helper functions # Open trade should be removed from the end tc0 = BTContainer(data=[ # D O H L C V B S @@ -203,46 +247,6 @@ def test_nonexisting_stake_amount(mocker, edge_conf): assert edge.stake_amount('N/O', 1, 2, 1) == 0.15 -def _validate_ohlc(buy_ohlc_sell_matrice): - for index, ohlc in enumerate(buy_ohlc_sell_matrice): - # if not high < open < low or not high < close < low - if not ohlc[3] >= ohlc[2] >= ohlc[4] or not ohlc[3] >= ohlc[5] >= ohlc[4]: - raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!') - return True - - -def _build_dataframe(buy_ohlc_sell_matrice): - _validate_ohlc(buy_ohlc_sell_matrice) - tickers = [] - for ohlc in buy_ohlc_sell_matrice: - ticker = { - 'date': ticker_start_time.shift( - minutes=( - ohlc[0] * - ticker_interval_in_minute)).timestamp * - 1000, - 'buy': ohlc[1], - 'open': ohlc[2], - 'high': ohlc[3], - 'low': ohlc[4], - 'close': ohlc[5], - 'sell': ohlc[6]} - tickers.append(ticker) - - frame = DataFrame(tickers) - frame['date'] = to_datetime(frame['date'], - unit='ms', - utc=True, - infer_datetime_format=True) - - return frame - - -def _time_on_candle(number): - return np.datetime64(ticker_start_time.shift( - minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') - - def test_edge_heartbeat_calculate(mocker, edge_conf): freqtrade = get_patched_freqtradebot(mocker, edge_conf) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) @@ -298,6 +302,40 @@ def test_edge_process_downloaded_data(mocker, edge_conf): assert edge._last_updated <= arrow.utcnow().timestamp + 2 +def test_edge_process_no_data(mocker, edge_conf, caplog): + edge_conf['datadir'] = None + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + + assert not edge.calculate() + assert len(edge._cached_pairs) == 0 + assert log_has("No data found. Edge is stopped ...", caplog.record_tuples) + assert edge._last_updated == 0 + + +def test_edge_process_no_trades(mocker, edge_conf, caplog): + edge_conf['datadir'] = None + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.data.history.load_data', mocked_load_data) + # Return empty + mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', MagicMock(return_value=[])) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + + assert not edge.calculate() + assert len(edge._cached_pairs) == 0 + assert log_has("No trades found.", caplog.record_tuples) + + +def test_edge_init_error(mocker, edge_conf,): + edge_conf['stake_amount'] = 0.5 + mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + with pytest.raises(OperationalException, match='Edge works only with unlimited stake amount'): + get_patched_freqtradebot(mocker, edge_conf) + + def test_process_expectancy(mocker, edge_conf): edge_conf['edge']['min_trade_number'] = 2 freqtrade = get_patched_freqtradebot(mocker, edge_conf) @@ -360,3 +398,11 @@ def test_process_expectancy(mocker, edge_conf): assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384 assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0 assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 + + # Pop last item so no trade is profitable + trades.pop() + trades_df = DataFrame(trades) + trades_df = edge._fill_calculable_fields(trades_df) + final = edge._process_expectancy(trades_df) + assert len(final) == 0 + assert isinstance(final, dict) From 253025c0feb173ccb304ca3d38dcfc2ac7b28477 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 May 2019 19:53:42 +0200 Subject: [PATCH 20/23] Add tests for check_int_positive --- freqtrade/tests/test_arguments.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 0952d1c5d..d71502abb 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -185,3 +185,12 @@ def test_testdata_dl_options() -> None: assert args.export == 'export/folder' assert args.days == 30 assert args.exchange == 'binance' + + +def test_check_int_positive() -> None: + assert Arguments.check_int_positive(2) == 2 + assert Arguments.check_int_positive(6) == 6 + with pytest.raises(argparse.ArgumentTypeError): + Arguments.check_int_positive(-6) + + assert Arguments.check_int_positive(2.5) == 2 From 7bbe8b24832099ca7bac9508373c232c4497f683 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 May 2019 06:22:27 +0200 Subject: [PATCH 21/23] Add a few more testcases for check_int_positive --- freqtrade/tests/test_arguments.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index d71502abb..455f3dbc6 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring, C0103 - import argparse import pytest @@ -194,3 +193,9 @@ def test_check_int_positive() -> None: Arguments.check_int_positive(-6) assert Arguments.check_int_positive(2.5) == 2 + assert Arguments.check_int_positive("3") == 3 + with pytest.raises(argparse.ArgumentTypeError): + Arguments.check_int_positive("3.5") + + with pytest.raises(argparse.ArgumentTypeError): + Arguments.check_int_positive("DeadBeef") From c3e93e7593b7f092e80464ebad25918420246bd3 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 24 May 2019 23:08:56 +0300 Subject: [PATCH 22/23] fix reduce() TypeError in hyperopts --- docs/hyperopt.md | 7 ++++--- freqtrade/optimize/default_hyperopt.py | 14 ++++++++------ user_data/hyperopts/sample_hyperopt.py | 14 ++++++++------ 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index b4e42de16..79ea4771b 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -122,9 +122,10 @@ So let's write the buy strategy using these values: dataframe['macd'], dataframe['macdsignal'] )) - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 return dataframe diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 721848d2e..7f1cb2435 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -70,9 +70,10 @@ class DefaultHyperOpts(IHyperOpt): dataframe['close'], dataframe['sar'] )) - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 return dataframe @@ -129,9 +130,10 @@ class DefaultHyperOpts(IHyperOpt): dataframe['sar'], dataframe['close'] )) - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 return dataframe diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 54f65a7e6..7cb55378e 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -79,9 +79,10 @@ class SampleHyperOpts(IHyperOpt): dataframe['close'], dataframe['sar'] )) - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 return dataframe @@ -138,9 +139,10 @@ class SampleHyperOpts(IHyperOpt): dataframe['sar'], dataframe['close'] )) - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 return dataframe From 469c0b6a558d3194026c97ec637f47d1e4e06eae Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 May 2019 13:16:00 +0200 Subject: [PATCH 23/23] Adjust check_int_positive tests --- freqtrade/arguments.py | 2 +- freqtrade/tests/test_arguments.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 327915b61..5afa8fa06 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -405,7 +405,7 @@ class Arguments(object): raise Exception('Incorrect syntax for timerange "%s"' % text) @staticmethod - def check_int_positive(value) -> int: + def check_int_positive(value: str) -> int: try: uint = int(value) if uint <= 0: diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 455f3dbc6..ecd108b5e 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -187,13 +187,17 @@ def test_testdata_dl_options() -> None: def test_check_int_positive() -> None: - assert Arguments.check_int_positive(2) == 2 - assert Arguments.check_int_positive(6) == 6 - with pytest.raises(argparse.ArgumentTypeError): - Arguments.check_int_positive(-6) - assert Arguments.check_int_positive(2.5) == 2 assert Arguments.check_int_positive("3") == 3 + assert Arguments.check_int_positive("1") == 1 + assert Arguments.check_int_positive("100") == 100 + + with pytest.raises(argparse.ArgumentTypeError): + Arguments.check_int_positive("-2") + + with pytest.raises(argparse.ArgumentTypeError): + Arguments.check_int_positive("0") + with pytest.raises(argparse.ArgumentTypeError): Arguments.check_int_positive("3.5")