From 646437363676156533ba00bb4dd6079b8ebc52d0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 27 Dec 2017 10:19:45 +0100 Subject: [PATCH 1/4] Update pymarketcap from 3.3.139 to 3.3.141 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b37a15c79..e36b66dc4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.2 -pymarketcap==3.3.139 +pymarketcap==3.3.141 # Required for plotting data #matplotlib==2.1.0 From f48f5d0f31de205e274303ed5aef14d595b7ab1b Mon Sep 17 00:00:00 2001 From: kryofly Date: Thu, 28 Dec 2017 15:58:02 +0100 Subject: [PATCH 2/4] tests for dataframe, whitelist and backtesting --- freqtrade/tests/optimize/test_backtesting.py | 104 +++++++++++++++++++ freqtrade/tests/test_acl_pair.py | 83 +++++++++++++++ freqtrade/tests/test_dataframe.py | 27 +++++ 3 files changed, 214 insertions(+) create mode 100644 freqtrade/tests/test_acl_pair.py create mode 100644 freqtrade/tests/test_dataframe.py diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 99f95d9bf..e4405fb4c 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212 +import math import os import pandas as pd from freqtrade import exchange, optimize @@ -120,3 +121,106 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): if os.path.isfile(file2): os.remove(file2) + + +def trim_dataframe(df, num): + new = dict() + for pair, pair_data in df.items(): + new[pair] = pair_data[-num:] # last 50 rows + return new + +def load_data_test(what): + data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) + data = trim_dataframe(data, -40) + pair = data['BTC_UNITEST'] + + # Depending on the what parameter we now adjust the + # loaded data: + # pair :: [{'O': 0.123, 'H': 0.123, 'L': 0.123, 'C': 0.123, 'V': 123.123, 'T': '2017-11-04T23:02:00', 'BV': 0.123}] + if what == 'raise': + o = h = l = c = 0.001 + l -= 0.0001 + h += 0.0001 + for frame in pair: + o += 0.0001 + h += 0.0001 + l += 0.0001 + c += 0.0001 + o = round(o,9) # round to satoshis + h = round(h,9) + l = round(l,9) + c = round(c,9) + frame['O'] = o + frame['H'] = h + frame['L'] = l + frame['C'] = c + if what == 'lower': + o = h = l = c = 0.001 + l -= 0.0001 + h += 0.0001 + for frame in pair: + o -= 0.0001 + h -= 0.0001 + l -= 0.0001 + c -= 0.0001 + o = round(o,9) # round to satoshis + h = round(h,9) + l = round(l,9) + c = round(c,9) + frame['O'] = o + frame['H'] = h + frame['L'] = l + frame['C'] = c + if what == 'sine': + i = 0 + o = h = l = c = (2 + math.sin(i/10)) / 1000 + h += 0.0001 + l -= 0.0001 + for frame in pair: + o = (2 + math.sin(i/10)) / 1000 + h = (2 + math.sin(i/10)) / 1000 + 0.0001 + l = (2 + math.sin(i/10)) / 1000 - 0.0001 + c = (2 + math.sin(i/10)) / 1000 - 0.000001 + + o = round(o,9) # round to satoshis + h = round(h,9) + l = round(l,9) + c = round(c,9) + frame['O'] = o + frame['H'] = h + frame['L'] = l + frame['C'] = c + i += 1 + return data + +def simple_backtest(config, contour, num_results): + data = load_data_test(contour) + processed = optimize.preprocess(data) + assert isinstance(processed, dict) + results = backtest(config['stake_amount'], processed, 1, True) + # results :: + assert len(results) == num_results + +# Test backtest on offline data +# loaded by freqdata/optimize/__init__.py::load_data() + +def test_backtest(default_conf, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) + results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) + num_resutls = len(results) + assert num_resutls > 0 + +def test_processed(default_conf, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + data = load_data_test('raise') + processed = optimize.preprocess(data) + +def test_raise(default_conf, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + tests = [['raise', 359], ['lower', 0], ['sine', 1734]] + for [contour, numres] in tests: + simple_backtest(default_conf, contour, numres) + + + diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py new file mode 100644 index 000000000..b1e57755c --- /dev/null +++ b/freqtrade/tests/test_acl_pair.py @@ -0,0 +1,83 @@ + +# whitelist, blacklist, filtering, all of that will +# eventually become some rules to run on a generic ACL engine + +# perhaps try to anticipate that by using some python package + +import pytest +from unittest.mock import MagicMock +import copy + +from freqtrade.main import refresh_whitelist +#from freqtrade.exchange import Exchanges +from freqtrade import exchange + +# "deep equal" +def assert_list_equal (l1, l2): + for pair in l1: + assert pair in l2 + for pair in l2: + assert pair in l1 + +def whitelist_conf(): + return { + "stake_currency":"BTC", + "exchange": { + "pair_whitelist": [ + "BTC_ETH", + "BTC_TKN", + "BTC_TRST", + "BTC_SWT", + "BTC_BCC" + ], + }, + } + +def get_health(): + return [{'Currency': 'ETH', + 'IsActive': True + }, + {'Currency': 'TKN', + 'IsActive': True + }] + +def get_health_empty(): + return [] + +# below three test could be merged into a single +# test that ran randomlly generated health lists + +def test_refresh_whitelist(mocker): + conf = whitelist_conf() + mocker.patch.dict('freqtrade.main._CONF', conf) + mocker.patch.multiple('freqtrade.main.exchange', + get_wallet_health=get_health) + # no argument: use the whitelist provided by config + refresh_whitelist() + whitelist = ['BTC_ETH', 'BTC_TKN'] + pairslist = conf['exchange']['pair_whitelist'] + # Ensure all except those in whitelist are removed + assert_list_equal(whitelist, pairslist) + +def test_refresh_whitelist_dynamic(mocker): + conf = whitelist_conf() + mocker.patch.dict('freqtrade.main._CONF', conf) + mocker.patch.multiple('freqtrade.main.exchange', + get_wallet_health=get_health) + # argument: use the whitelist dynamically by exchange-volume + whitelist = ['BTC_ETH', 'BTC_TKN'] + refresh_whitelist(whitelist) + pairslist = conf['exchange']['pair_whitelist'] + assert_list_equal(whitelist, pairslist) + +def test_refresh_whitelist_dynamic_empty(mocker): + conf = whitelist_conf() + mocker.patch.dict('freqtrade.main._CONF', conf) + mocker.patch.multiple('freqtrade.main.exchange', + get_wallet_health=get_health_empty) + # argument: use the whitelist dynamically by exchange-volume + whitelist = [] + conf['exchange']['pair_whitelist'] = [] + refresh_whitelist(whitelist) + pairslist = conf['exchange']['pair_whitelist'] + assert_list_equal(whitelist, pairslist) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py new file mode 100644 index 000000000..58d1474fd --- /dev/null +++ b/freqtrade/tests/test_dataframe.py @@ -0,0 +1,27 @@ + +import pytest +import pandas + +from freqtrade import analyze +import freqtrade.optimize +from pandas import DataFrame + +_pairs = ['BTC_ETH'] + +def load_dataframe_pair(pairs): + ld = freqtrade.optimize.load_data(ticker_interval=5, pairs=pairs) + assert isinstance(ld, dict) + assert isinstance(pairs[0], str) + dataframe = ld[pairs[0]] + dataframe = analyze.analyze_ticker(dataframe) + return dataframe + +def test_dataframe_load(): + dataframe = load_dataframe_pair(_pairs) + assert isinstance(dataframe, pandas.core.frame.DataFrame) + +def test_dataframe_columns_exists(): + dataframe = load_dataframe_pair(_pairs) + assert 'high' in dataframe.columns + assert 'low' in dataframe.columns + assert 'close' in dataframe.columns From ab112581a71fa42848561fe0b3893832255baf8f Mon Sep 17 00:00:00 2001 From: kryofly Date: Thu, 28 Dec 2017 20:05:33 +0100 Subject: [PATCH 3/4] tests: anal stretching to accomodate flake8 --- freqtrade/tests/optimize/test_backtesting.py | 88 ++++++++++---------- freqtrade/tests/test_acl_pair.py | 24 +++--- freqtrade/tests/test_dataframe.py | 10 +-- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index e4405fb4c..528feafb1 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,5 +1,3 @@ -# pragma pylint: disable=missing-docstring,W0212 - import math import os import pandas as pd @@ -126,9 +124,10 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): def trim_dataframe(df, num): new = dict() for pair, pair_data in df.items(): - new[pair] = pair_data[-num:] # last 50 rows + new[pair] = pair_data[-num:] # last 50 rows return new + def load_data_test(what): data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) data = trim_dataframe(data, -40) @@ -136,63 +135,66 @@ def load_data_test(what): # Depending on the what parameter we now adjust the # loaded data: - # pair :: [{'O': 0.123, 'H': 0.123, 'L': 0.123, 'C': 0.123, 'V': 123.123, 'T': '2017-11-04T23:02:00', 'BV': 0.123}] + # pair :: [{'O': 0.123, 'H': 0.123, 'L': 0.123, + # 'C': 0.123, 'V': 123.123, + # 'T': '2017-11-04T23:02:00', 'BV': 0.123}] if what == 'raise': - o = h = l = c = 0.001 - l -= 0.0001 + o = 0.001 + h = 0.001 + ll = 0.001 + c = 0.001 + ll -= 0.0001 h += 0.0001 for frame in pair: o += 0.0001 h += 0.0001 - l += 0.0001 + ll += 0.0001 c += 0.0001 - o = round(o,9) # round to satoshis - h = round(h,9) - l = round(l,9) - c = round(c,9) - frame['O'] = o - frame['H'] = h - frame['L'] = l - frame['C'] = c + # save prices rounded to satoshis + frame['O'] = round(o, 9) + frame['H'] = round(h, 9) + frame['L'] = round(ll, 9) + frame['C'] = round(c, 9) if what == 'lower': - o = h = l = c = 0.001 - l -= 0.0001 + o = 0.001 + h = 0.001 + ll = 0.001 + c = 0.001 + ll -= 0.0001 h += 0.0001 for frame in pair: o -= 0.0001 h -= 0.0001 - l -= 0.0001 + ll -= 0.0001 c -= 0.0001 - o = round(o,9) # round to satoshis - h = round(h,9) - l = round(l,9) - c = round(c,9) - frame['O'] = o - frame['H'] = h - frame['L'] = l - frame['C'] = c + # save prices rounded to satoshis + frame['O'] = round(o, 9) + frame['H'] = round(h, 9) + frame['L'] = round(ll, 9) + frame['C'] = round(c, 9) if what == 'sine': i = 0 - o = h = l = c = (2 + math.sin(i/10)) / 1000 + o = (2 + math.sin(i/10)) / 1000 + h = o + ll = o + c = o h += 0.0001 - l -= 0.0001 + ll -= 0.0001 for frame in pair: - o = (2 + math.sin(i/10)) / 1000 + o = (2 + math.sin(i/10)) / 1000 h = (2 + math.sin(i/10)) / 1000 + 0.0001 - l = (2 + math.sin(i/10)) / 1000 - 0.0001 + ll = (2 + math.sin(i/10)) / 1000 - 0.0001 c = (2 + math.sin(i/10)) / 1000 - 0.000001 - o = round(o,9) # round to satoshis - h = round(h,9) - l = round(l,9) - c = round(c,9) - frame['O'] = o - frame['H'] = h - frame['L'] = l - frame['C'] = c + # save prices rounded to satoshis + frame['O'] = round(o, 9) + frame['H'] = round(h, 9) + frame['L'] = round(ll, 9) + frame['C'] = round(c, 9) i += 1 return data + def simple_backtest(config, contour, num_results): data = load_data_test(contour) processed = optimize.preprocess(data) @@ -204,23 +206,23 @@ def simple_backtest(config, contour, num_results): # Test backtest on offline data # loaded by freqdata/optimize/__init__.py::load_data() -def test_backtest(default_conf, mocker): + +def test_backtest2(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) num_resutls = len(results) assert num_resutls > 0 + def test_processed(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) data = load_data_test('raise') - processed = optimize.preprocess(data) + assert optimize.preprocess(data) + def test_raise(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) tests = [['raise', 359], ['lower', 0], ['sine', 1734]] for [contour, numres] in tests: simple_backtest(default_conf, contour, numres) - - - diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index b1e57755c..8748d97b8 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -1,27 +1,20 @@ +from freqtrade.main import refresh_whitelist # whitelist, blacklist, filtering, all of that will # eventually become some rules to run on a generic ACL engine - # perhaps try to anticipate that by using some python package -import pytest -from unittest.mock import MagicMock -import copy -from freqtrade.main import refresh_whitelist -#from freqtrade.exchange import Exchanges -from freqtrade import exchange - -# "deep equal" -def assert_list_equal (l1, l2): +def assert_list_equal(l1, l2): for pair in l1: assert pair in l2 for pair in l2: assert pair in l1 + def whitelist_conf(): return { - "stake_currency":"BTC", + "stake_currency": "BTC", "exchange": { "pair_whitelist": [ "BTC_ETH", @@ -33,13 +26,15 @@ def whitelist_conf(): }, } + def get_health(): return [{'Currency': 'ETH', 'IsActive': True - }, + }, {'Currency': 'TKN', 'IsActive': True - }] + }] + def get_health_empty(): return [] @@ -47,6 +42,7 @@ def get_health_empty(): # below three test could be merged into a single # test that ran randomlly generated health lists + def test_refresh_whitelist(mocker): conf = whitelist_conf() mocker.patch.dict('freqtrade.main._CONF', conf) @@ -59,6 +55,7 @@ def test_refresh_whitelist(mocker): # Ensure all except those in whitelist are removed assert_list_equal(whitelist, pairslist) + def test_refresh_whitelist_dynamic(mocker): conf = whitelist_conf() mocker.patch.dict('freqtrade.main._CONF', conf) @@ -70,6 +67,7 @@ def test_refresh_whitelist_dynamic(mocker): pairslist = conf['exchange']['pair_whitelist'] assert_list_equal(whitelist, pairslist) + def test_refresh_whitelist_dynamic_empty(mocker): conf = whitelist_conf() mocker.patch.dict('freqtrade.main._CONF', conf) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index 58d1474fd..916985406 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -1,13 +1,11 @@ - -import pytest import pandas from freqtrade import analyze import freqtrade.optimize -from pandas import DataFrame _pairs = ['BTC_ETH'] + def load_dataframe_pair(pairs): ld = freqtrade.optimize.load_data(ticker_interval=5, pairs=pairs) assert isinstance(ld, dict) @@ -16,12 +14,14 @@ def load_dataframe_pair(pairs): dataframe = analyze.analyze_ticker(dataframe) return dataframe + def test_dataframe_load(): dataframe = load_dataframe_pair(_pairs) assert isinstance(dataframe, pandas.core.frame.DataFrame) + def test_dataframe_columns_exists(): dataframe = load_dataframe_pair(_pairs) - assert 'high' in dataframe.columns - assert 'low' in dataframe.columns + assert 'high' in dataframe.columns + assert 'low' in dataframe.columns assert 'close' in dataframe.columns From 37613fc056705e3fa6435a630cf922bac4fa7a67 Mon Sep 17 00:00:00 2001 From: kryofly Date: Fri, 29 Dec 2017 17:53:58 +0100 Subject: [PATCH 4/4] flake8 --- freqtrade/tests/optimize/test_backtesting.py | 73 -------------------- 1 file changed, 73 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 3acd4b142..0d153a22e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,7 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212 import math -import os import pandas as pd from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex @@ -50,78 +49,6 @@ def test_backtest_1min_ticker_interval(default_conf, mocker): assert not results.empty -def test_backtest_with_new_pair(default_conf, ticker_history, mocker): - mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) - - optimize.load_data(ticker_interval=1, pairs=['BTC_MEME']) - file = 'freqtrade/tests/testdata/BTC_MEME-1.json' - assert os.path.isfile(file) is True - - # delete file freshly downloaded - if os.path.isfile(file): - os.remove(file) - - -def test_testdata_path(): - assert os.path.join('freqtrade', 'tests', 'testdata') in testdata_path() - - -def test_download_pairs(default_conf, ticker_history, mocker): - mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) - - file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json' - file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json' - file2_1 = 'freqtrade/tests/testdata/BTC_CFI-1.json' - file2_5 = 'freqtrade/tests/testdata/BTC_CFI-5.json' - - assert download_pairs(pairs=['BTC-MEME', 'BTC-CFI']) is True - - assert os.path.isfile(file1_1) is True - assert os.path.isfile(file1_5) is True - assert os.path.isfile(file2_1) is True - assert os.path.isfile(file2_5) is True - - # delete files freshly downloaded - if os.path.isfile(file1_1): - os.remove(file1_1) - - if os.path.isfile(file1_5): - os.remove(file1_5) - - if os.path.isfile(file2_1): - os.remove(file2_1) - - if os.path.isfile(file2_5): - os.remove(file2_5) - - -def test_download_backtesting_testdata(default_conf, ticker_history, mocker): - mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) - - # Download a 1 min ticker file - file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json' - download_backtesting_testdata(pair="BTC-XEL", interval=1) - assert os.path.isfile(file1) is True - - if os.path.isfile(file1): - os.remove(file1) - - # Download a 5 min ticker file - file2 = 'freqtrade/tests/testdata/BTC_STORJ-5.json' - download_backtesting_testdata(pair="BTC-STORJ", interval=5) - assert os.path.isfile(file2) is True - - if os.path.isfile(file2): - os.remove(file2) - - def trim_dataframe(df, num): new = dict() for pair, pair_data in df.items():