From ca8cab0ce9e59abd4dc3aa9b02a640a035e01f19 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 6 Jan 2018 11:44:41 +0200 Subject: [PATCH 01/19] Hyperopt to handle SIGINT by saving/reading the trials file --- .gitignore | 2 ++ freqtrade/optimize/hyperopt.py | 47 +++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 672dd1f6d..c81b55222 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,5 @@ target/ .venv .idea .vscode + +hyperopt_trials.pickle diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index cf46b96ad..febaa196d 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,6 +4,9 @@ import json import logging import sys +import pickle +import signal +import os from functools import reduce from math import exp from operator import itemgetter @@ -43,6 +46,10 @@ EXPECTED_MAX_PROFIT = 3.85 PROCESSED = None # optimize.preprocess(optimize.load_data()) OPTIMIZE_CONFIG = hyperopt_optimize_conf() +# Hyperopt Trials +TRIALS_FILE = 'freqtrade/optimize/hyperopt_trials.pickle' +TRIALS = Trials() + # Monkey patch config from freqtrade import main # noqa main._CONF = OPTIMIZE_CONFIG @@ -99,6 +106,19 @@ SPACE = { } +def save_trials(trials, trials_path=TRIALS_FILE): + "Save hyperopt trials to file" + logger.info('Saving Trials to \'{}\''.format(trials_path)) + pickle.dump(trials, open(trials_path, 'wb')) + + +def read_trials(trials_path=TRIALS_FILE): + "Read hyperopt trials file" + logger.info('Reading Trials from \'{}\''.format(trials_path)) + trials = pickle.load(open(trials_path, 'rb')) + # os.remove(trials_path) + return trials + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -216,7 +236,12 @@ def buy_strategy_generator(params): def start(args): +<<<<<<< HEAD global TOTAL_TRIES, PROCESSED, SPACE +======= + global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES + +>>>>>>> Hyperopt to handle SIGINT by saving/reading the trials file TOTAL_TRIES = args.epochs exchange._API = Bittrex({'key': '', 'secret': ''}) @@ -238,9 +263,14 @@ def start(args): logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!') db_name = 'freqtrade_hyperopt' - trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') + TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') else: - trials = Trials() + logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, signal_handler) + # read trials file if we have one + if os.path.exists(TRIALS_FILE): + TRIALS = read_trials() + _CURRENT_TRIES = len([result for result in TRIALS.results if result['status'] == 'ok']) try: best_parameters = fmin( @@ -248,7 +278,7 @@ def start(args): space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, - trials=trials + trials=TRIALS ) results = sorted(trials.results, key=itemgetter('loss')) @@ -265,3 +295,14 @@ def start(args): logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) logger.info('Best Result:\n%s', best_result) + + # Store trials result to file to resume next time + save_trials(TRIALS) + + +def signal_handler(sig, frame): + "Hyperopt SIGINT handler" + logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) + + save_trials(TRIALS) + sys.exit(0) From a48840509b4d168e44750e96bf6277dbbe53bb5d Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 6 Jan 2018 15:34:21 +0200 Subject: [PATCH 02/19] Hyperopt: use results from previous runs --- freqtrade/optimize/hyperopt.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index febaa196d..c520ed81d 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -119,6 +119,7 @@ def read_trials(trials_path=TRIALS_FILE): # os.remove(trials_path) return trials + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -270,7 +271,11 @@ def start(args): # read trials file if we have one if os.path.exists(TRIALS_FILE): TRIALS = read_trials() - _CURRENT_TRIES = len([result for result in TRIALS.results if result['status'] == 'ok']) + _CURRENT_TRIES = len(TRIALS.results) + TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES + logger.info( + 'Continuing with trials. Current: {}, Total: {}' + .format(_CURRENT_TRIES, TOTAL_TRIES)) try: best_parameters = fmin( From b35fa4c9f6ba29c4c2333251df77276301cf449c Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 6 Jan 2018 15:53:22 +0200 Subject: [PATCH 03/19] hyperopt: show the best results so far --- freqtrade/optimize/hyperopt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c520ed81d..dc611fcbc 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -120,6 +120,11 @@ def read_trials(trials_path=TRIALS_FILE): return trials +def log_trials_result(trials): + vals = trials.best_trial['misc']['vals'] + results = trials.best_trial['result']['result'] + logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -310,4 +315,5 @@ def signal_handler(sig, frame): logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) save_trials(TRIALS) + log_trials_result(TRIALS) sys.exit(0) From 1647e7a0c1357fc663d52386381563687f45554c Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 9 Jan 2018 11:37:27 +0200 Subject: [PATCH 04/19] update fix failing tests, unitest that resume hyperopt functionality works --- freqtrade/optimize/hyperopt.py | 12 ++--- freqtrade/tests/optimize/test_hyperopt.py | 66 +++++++++++++++++++++-- 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index dc611fcbc..4162e0758 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data TARGET_TRADES = 1100 -TOTAL_TRIES = None +TOTAL_TRIES = 0 _CURRENT_TRIES = 0 CURRENT_BEST_LOSS = 100 @@ -116,7 +116,7 @@ def read_trials(trials_path=TRIALS_FILE): "Read hyperopt trials file" logger.info('Reading Trials from \'{}\''.format(trials_path)) trials = pickle.load(open(trials_path, 'rb')) - # os.remove(trials_path) + os.remove(trials_path) return trials @@ -125,6 +125,7 @@ def log_trials_result(trials): results = trials.best_trial['result']['result'] logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -242,12 +243,8 @@ def buy_strategy_generator(params): def start(args): -<<<<<<< HEAD - global TOTAL_TRIES, PROCESSED, SPACE -======= - global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES + global TOTAL_TRIES, PROCESSED, SPACE, TRIALS, _CURRENT_TRIES ->>>>>>> Hyperopt to handle SIGINT by saving/reading the trials file TOTAL_TRIES = args.epochs exchange._API = Bittrex({'key': '', 'secret': ''}) @@ -276,6 +273,7 @@ def start(args): # read trials file if we have one if os.path.exists(TRIALS_FILE): TRIALS = read_trials() + _CURRENT_TRIES = len(TRIALS.results) TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES logger.info( diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index a309af7fe..99a1f4a3c 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,5 +1,8 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 - +import pickle +import os +import pytest +import freqtrade.optimize.hyperopt from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ log_results @@ -27,16 +30,35 @@ def test_loss_calculation_has_limited_profit(): def create_trials(mocker): + """ + When creating trials, mock the hyperopt Trials so that *by default* + - we don't create any pickle'd files in the filesystem + - we might have a pickle'd file so make sure that we return + false when looking for it + """ + mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', + return_value='freqtrade/tests/optimize/ut_trials.pickle') + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', + return_value=False) + mocker.patch('freqtrade.optimize.hyperopt.save_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.read_trials', + return_value=None) return mocker.Mock( results=[{ 'loss': 1, - 'result': 'foo' - }] + 'result': 'foo', + 'status': 'ok' + }], + best_trial={'misc': {'vals': {'adx': 999}}} ) def test_start_calls_fmin(mocker): - mocker.patch('freqtrade.optimize.hyperopt.Trials', return_value=create_trials(mocker)) + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials) + mocker.patch('freqtrade.optimize.hyperopt.sorted', + return_value=trials.results) mocker.patch('freqtrade.optimize.preprocess') mocker.patch('freqtrade.optimize.load_data') mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) @@ -141,3 +163,39 @@ def test_fmin_throw_value_error(mocker, caplog): for line in exists: assert line in caplog.text + + +def test_resuming_previous_hyperopt_results_succeeds(mocker): + import freqtrade.optimize.hyperopt as hyperopt + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.TRIALS', + return_value=trials) + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', + return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.len', + return_value=len(trials.results)) + mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials', + return_value=trials) + mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.sorted', + return_value=trials.results) + mocker.patch('freqtrade.optimize.preprocess') + mocker.patch('freqtrade.optimize.load_data') + mocker.patch('freqtrade.optimize.hyperopt.fmin', + return_value={}) + args = mocker.Mock(epochs=1, + config='config.json.example', + mongodb=False) + + start(args) + + mock_read.assert_called_once() + mock_save.assert_called_once() + + current_tries = hyperopt._CURRENT_TRIES + total_tries = hyperopt.TOTAL_TRIES + + assert current_tries == len(trials.results) + assert total_tries == (current_tries + len(trials.results)) + From fe2b0c28621c53ded0ebe85ae1c4eb59c2661236 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 9 Jan 2018 12:19:44 +0200 Subject: [PATCH 05/19] add unittest to save and read trials file --- freqtrade/tests/optimize/test_hyperopt.py | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 99a1f4a3c..3e03d26c0 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,10 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import pickle -import os -import pytest -import freqtrade.optimize.hyperopt from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ - log_results + log_results, save_trials, read_trials def test_loss_calculation_prefer_correct_trade_count(): @@ -44,6 +40,8 @@ def create_trials(mocker): return_value=None) mocker.patch('freqtrade.optimize.hyperopt.read_trials', return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.os.remove', + return_value=True) return mocker.Mock( results=[{ 'loss': 1, @@ -199,3 +197,27 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker): assert current_tries == len(trials.results) assert total_tries == (current_tries + len(trials.results)) + +def test_save_trials_saves_trials(mocker): + trials = create_trials(mocker) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', + return_value=None) + trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', + return_value='ut_trials.pickle') + mocker.patch('freqtrade.optimize.hyperopt.open', + return_value=trials_path) + save_trials(trials, trials_path) + + mock_dump.assert_called_once_with(trials, trials_path) + + +def test_read_trials_returns_trials_file(mocker): + trials = create_trials(mocker) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', + return_value=trials) + mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', + return_value=mock_load) + + assert read_trials() == trials + mock_open.assert_called_once() + mock_load.assert_called_once() From ffae0b2cd54eab2d6eea2eae026194ca9d3d9657 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 9 Jan 2018 12:37:56 +0200 Subject: [PATCH 06/19] hyperopt: prettyfie best values when receiving SIGINT, use the global TRIALS --- freqtrade/optimize/hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4162e0758..03f833c4a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -121,7 +121,7 @@ def read_trials(trials_path=TRIALS_FILE): def log_trials_result(trials): - vals = trials.best_trial['misc']['vals'] + vals = json.dumps(trials.best_trial['misc']['vals'], indent=4) results = trials.best_trial['result']['result'] logger.info('Best result:\n%s\nwith values:\n%s', results, vals) @@ -289,7 +289,7 @@ def start(args): trials=TRIALS ) - results = sorted(trials.results, key=itemgetter('loss')) + results = sorted(TRIALS.results, key=itemgetter('loss')) best_result = results[0]['result'] except ValueError: From e67c652988cf55b85cb06771dc513222c6eb0ea8 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Wed, 10 Jan 2018 11:50:00 +0200 Subject: [PATCH 07/19] use os.path.join, fix docstrings --- freqtrade/optimize/hyperopt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 03f833c4a..5d342b831 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -47,7 +47,7 @@ PROCESSED = None # optimize.preprocess(optimize.load_data()) OPTIMIZE_CONFIG = hyperopt_optimize_conf() # Hyperopt Trials -TRIALS_FILE = 'freqtrade/optimize/hyperopt_trials.pickle' +TRIALS_FILE = os.path.join('freqtrade', 'optimize', 'hyperopt_trials.pickle') TRIALS = Trials() # Monkey patch config @@ -107,13 +107,13 @@ SPACE = { def save_trials(trials, trials_path=TRIALS_FILE): - "Save hyperopt trials to file" + """Save hyperopt trials to file""" logger.info('Saving Trials to \'{}\''.format(trials_path)) pickle.dump(trials, open(trials_path, 'wb')) def read_trials(trials_path=TRIALS_FILE): - "Read hyperopt trials file" + """Read hyperopt trials file""" logger.info('Reading Trials from \'{}\''.format(trials_path)) trials = pickle.load(open(trials_path, 'rb')) os.remove(trials_path) @@ -309,7 +309,7 @@ def start(args): def signal_handler(sig, frame): - "Hyperopt SIGINT handler" + """Hyperopt SIGINT handler""" logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) save_trials(TRIALS) From 1b6b0ad9d2e7f26e9e05cbcce5f7c7b35f1043e0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 9 Jan 2018 12:59:06 +0200 Subject: [PATCH 08/19] autopep8 --- freqtrade/main.py | 22 +++--- freqtrade/optimize/backtesting.py | 26 +++---- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/rpc/telegram.py | 2 +- freqtrade/tests/optimize/test_optimize.py | 2 +- freqtrade/tests/test_acl_pair.py | 90 +++++++++++------------ freqtrade/tests/test_main.py | 84 ++++++++++----------- freqtrade/tests/test_misc.py | 2 +- 8 files changed, 115 insertions(+), 115 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index efd0d7c44..58f3c7aee 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -197,11 +197,11 @@ def execute_sell(trade: Trade, limit: float) -> None: profit_trade = trade.calc_profit(rate=limit) message = '*{exchange}:* Selling [{pair}]({pair_url}) with limit `{limit:.8f}`'.format( - exchange=trade.exchange, - pair=trade.pair.replace('_', '/'), - pair_url=exchange.get_pair_detail_url(trade.pair), - limit=limit - ) + exchange=trade.exchange, + pair=trade.pair.replace('_', '/'), + pair_url=exchange.get_pair_detail_url(trade.pair), + limit=limit + ) # For regular case, when the configuration exists if 'stake_currency' in _CONF and 'fiat_display_currency' in _CONF: @@ -213,12 +213,12 @@ def execute_sell(trade: Trade, limit: float) -> None: ) message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ '` / {profit_fiat:.3f} {fiat})`'.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade, - coin=_CONF['stake_currency'], - profit_fiat=profit_fiat, - fiat=_CONF['fiat_display_currency'], + gain="profit" if fmt_exp_profit > 0 else "loss", + profit_percent=fmt_exp_profit, + profit_coin=profit_trade, + coin=_CONF['stake_currency'], + profit_fiat=profit_fiat, + fiat=_CONF['fiat_display_currency'], ) # Because telegram._forcesell does not have the configuration # Ignore the FIAT value and does not show the stake_currency as well diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 315f960d8..fe3ebd4f1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -122,20 +122,20 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], if min_roi_reached(trade, row2.close, row2.date) or \ (row2.sell == 1 and use_sell_signal) or \ current_profit_percent <= stoploss: - current_profit_btc = trade.calc_profit(rate=row2.close) - lock_pair_until = row2.Index + current_profit_btc = trade.calc_profit(rate=row2.close) + lock_pair_until = row2.Index - trades.append( - ( - pair, - current_profit_percent, - current_profit_btc, - row2.Index - row.Index, - current_profit_btc > 0, - current_profit_btc < 0 - ) + trades.append( + ( + pair, + current_profit_percent, + current_profit_btc, + row2.Index - row.Index, + current_profit_btc > 0, + current_profit_btc < 0 ) - break + ) + break labels = ['currency', 'profit_percent', 'profit_BTC', 'duration', 'profit', 'loss'] return DataFrame.from_records(trades, columns=labels) @@ -193,6 +193,6 @@ def start(args): use_sell_signal=config.get('experimental', {}).get('use_sell_signal', False) ) logger.info( - '\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa + '\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa generate_text_table(data, results, config['stake_currency'], args.ticker_interval) ) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5d342b831..725bf37b0 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -194,7 +194,7 @@ def format_results(results: DataFrame): results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), results.duration.mean() * 5, - ) + ) def buy_strategy_generator(params): diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4d7957f59..55f1942f3 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -255,7 +255,7 @@ def _daily(bot: Bot, update: Update) -> None: ), symbol=_CONF['fiat_display_currency'] ) - ] + ] for key, value in profit_days.items() ] stats = tabulate(stats, diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index a5892f278..57c41c9c6 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -6,7 +6,7 @@ from shutil import copyfile from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ - download_backtesting_testdata, load_tickerdata_file + download_backtesting_testdata, load_tickerdata_file # Change this if modifying BTC_UNITEST testdatafile _btc_unittest_length = 13681 diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index be0bd47e8..d1e42d444 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -25,51 +25,51 @@ def whitelist_conf(): def get_market_summaries(): return [{ - "MarketName": "BTC-TKN", - "High": 0.00000919, - "Low": 0.00000820, - "Volume": 74339.61396015, - "Last": 0.00000820, - "BaseVolume": 1664, - "TimeStamp": "2014-07-09T07:19:30.15", - "Bid": 0.00000820, - "Ask": 0.00000831, - "OpenBuyOrders": 15, - "OpenSellOrders": 15, - "PrevDay": 0.00000821, - "Created": "2014-03-20T06:00:00", - "DisplayMarketName": "" - }, { - "MarketName": "BTC-ETH", - "High": 0.00000072, - "Low": 0.00000001, - "Volume": 166340678.42280999, - "Last": 0.00000005, - "BaseVolume": 42, - "TimeStamp": "2014-07-09T07:21:40.51", - "Bid": 0.00000004, - "Ask": 0.00000005, - "OpenBuyOrders": 18, - "OpenSellOrders": 18, - "PrevDay": 0.00000002, - "Created": "2014-05-30T07:57:49.637", - "DisplayMarketName": "" - }, { - "MarketName": "BTC-BLK", - "High": 0.00000072, - "Low": 0.00000001, - "Volume": 166340678.42280999, - "Last": 0.00000005, - "BaseVolume": 3, - "TimeStamp": "2014-07-09T07:21:40.51", - "Bid": 0.00000004, - "Ask": 0.00000005, - "OpenBuyOrders": 18, - "OpenSellOrders": 18, - "PrevDay": 0.00000002, - "Created": "2014-05-30T07:57:49.637", - "DisplayMarketName": "" - } + "MarketName": "BTC-TKN", + "High": 0.00000919, + "Low": 0.00000820, + "Volume": 74339.61396015, + "Last": 0.00000820, + "BaseVolume": 1664, + "TimeStamp": "2014-07-09T07:19:30.15", + "Bid": 0.00000820, + "Ask": 0.00000831, + "OpenBuyOrders": 15, + "OpenSellOrders": 15, + "PrevDay": 0.00000821, + "Created": "2014-03-20T06:00:00", + "DisplayMarketName": "" + }, { + "MarketName": "BTC-ETH", + "High": 0.00000072, + "Low": 0.00000001, + "Volume": 166340678.42280999, + "Last": 0.00000005, + "BaseVolume": 42, + "TimeStamp": "2014-07-09T07:21:40.51", + "Bid": 0.00000004, + "Ask": 0.00000005, + "OpenBuyOrders": 18, + "OpenSellOrders": 18, + "PrevDay": 0.00000002, + "Created": "2014-05-30T07:57:49.637", + "DisplayMarketName": "" + }, { + "MarketName": "BTC-BLK", + "High": 0.00000072, + "Low": 0.00000001, + "Volume": 166340678.42280999, + "Last": 0.00000005, + "BaseVolume": 3, + "TimeStamp": "2014-07-09T07:21:40.51", + "Bid": 0.00000004, + "Ask": 0.00000005, + "OpenBuyOrders": 18, + "OpenSellOrders": 18, + "PrevDay": 0.00000002, + "Created": "2014-05-30T07:57:49.637", + "DisplayMarketName": "" + } ] diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index cceb555f7..ab39cebc9 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -624,54 +624,54 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker): def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': True, - } + default_conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': True, + } - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.min_roi_reached', return_value=False) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value='mocked_limit_buy')) - init(default_conf, create_engine('sqlite://')) - create_trade(0.001) + init(default_conf, create_engine('sqlite://')) + create_trade(0.001) - trade = Trade.query.first() - trade.update(limit_buy_order) - assert handle_trade(trade) is False + trade = Trade.query.first() + trade.update(limit_buy_order) + assert handle_trade(trade) is False def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': False, - } + default_conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': False, + } - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.min_roi_reached', return_value=False) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value='mocked_limit_buy')) - init(default_conf, create_engine('sqlite://')) - create_trade(0.001) + init(default_conf, create_engine('sqlite://')) + create_trade(0.001) - trade = Trade.query.first() - trade.update(limit_buy_order) - assert handle_trade(trade) is True + trade = Trade.query.first() + trade.update(limit_buy_order) + assert handle_trade(trade) is True diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 0b85000cb..16bb8d232 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -8,7 +8,7 @@ import pytest from jsonschema import ValidationError from freqtrade.misc import throttle, parse_args, load_config,\ - parse_args_common + parse_args_common def test_throttle(): From 0abc30401c2ac1b2aa73b638db06defe82223d64 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 9 Jan 2018 13:06:28 +0200 Subject: [PATCH 09/19] linter fixes and cleanups --- freqtrade/misc.py | 15 ++++++++------- freqtrade/tests/test_misc.py | 8 +++----- scripts/plot_dataframe.py | 4 ++-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index afc4334e8..c018b7fbb 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -57,8 +57,8 @@ def load_config(path: str) -> Dict: try: validate(conf, CONF_SCHEMA) return conf - except ValidationError as ex: - logger.fatal('Invalid configuration. See config.json.example. Reason: %s', ex) + except ValidationError as exception: + logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception) raise ValidationError( best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message ) @@ -81,7 +81,7 @@ def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: return result -def parse_args_common(args: List[str], description: str): +def common_args_parser(description: str): """ Parses given common arguments and returns them as a parsed object. """ @@ -117,11 +117,11 @@ def parse_args(args: List[str], description: str): Parses given arguments and returns an argparse Namespace instance. Returns None if a sub command has been selected and executed. """ - parser = parse_args_common(args, description) + parser = common_args_parser(description) parser.add_argument( '--dry-run-db', - help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" instead of memory DB. Work only if dry_run is \ - enabled.', # noqa + help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \ + instead of memory DB. Work only if dry_run is enabled.', action='store_true', dest='dry_run_db', ) @@ -135,7 +135,8 @@ def parse_args(args: List[str], description: str): ) parser.add_argument( '--dynamic-whitelist', - help='dynamically generate and update whitelist based on 24h BaseVolume (Default 20 currencies)', # noqa + help='dynamically generate and update whitelist \ + based on 24h BaseVolume (Default 20 currencies)', # noqa dest='dynamic_whitelist', const=20, type=int, diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 16bb8d232..9de40e8d5 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -8,7 +8,7 @@ import pytest from jsonschema import ValidationError from freqtrade.misc import throttle, parse_args, load_config,\ - parse_args_common + common_args_parser def test_throttle(): @@ -39,12 +39,10 @@ def test_throttle_with_assets(): assert result == -1 -# Parse common command-line-arguments -# used for all tools - +# Parse common command-line-arguments. Used for all tools def test_parse_args_none(): - args = parse_args_common([], '') + args = common_args_parser('') assert isinstance(args, argparse.ArgumentParser) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 44961891c..ce636a4b5 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -6,11 +6,11 @@ import matplotlib # Install PYQT5 manually if you want to test this helper func matplotlib.use("Qt5Agg") import matplotlib.pyplot as plt from freqtrade import exchange, analyze -from freqtrade.misc import parse_args_common +from freqtrade.misc import common_args_parser def plot_parse_args(args ): - parser = parse_args_common(args, 'Graph utility') + parser = common_args_parser(args, 'Graph utility') parser.add_argument( '-p', '--pair', help = 'What currency pair', From 86db6c90840ae40e8eaebc63c0ad75bc1b060626 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 09:51:36 +0200 Subject: [PATCH 10/19] sort imports --- freqtrade/analyze.py | 4 ++-- freqtrade/exchange/bittrex.py | 5 +++-- freqtrade/exchange/interface.py | 2 +- freqtrade/fiat_convert.py | 1 + freqtrade/main.py | 16 ++++++++-------- freqtrade/misc.py | 6 +++--- freqtrade/optimize/backtesting.py | 7 +++---- freqtrade/optimize/hyperopt.py | 3 ++- freqtrade/persistence.py | 5 +++-- freqtrade/rpc/telegram.py | 12 ++++++------ freqtrade/tests/conftest.py | 4 ++-- freqtrade/tests/test_analyze.py | 5 +++-- freqtrade/tests/test_dataframe.py | 2 +- freqtrade/tests/test_fiat_convert.py | 5 +++-- freqtrade/tests/test_main.py | 13 ++++++------- freqtrade/tests/test_misc.py | 6 +++--- 16 files changed, 50 insertions(+), 46 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 4b16550c5..f85c46248 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -4,14 +4,14 @@ Functions to analyze ticker data with indicators and produce buy and sell signal import logging from datetime import timedelta from enum import Enum -from typing import List, Dict +from typing import Dict, List import arrow import talib.abstract as ta from pandas import DataFrame, to_datetime -from freqtrade.exchange import get_ticker_history import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.exchange import get_ticker_history logger = logging.getLogger(__name__) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 4883db037..5a8a602d2 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -1,7 +1,8 @@ import logging -from typing import List, Dict, Optional +from typing import Dict, List, Optional -from bittrex.bittrex import Bittrex as _Bittrex, API_V2_0, API_V1_1 +from bittrex.bittrex import Bittrex as _Bittrex +from bittrex.bittrex import API_V1_1, API_V2_0 from requests.exceptions import ContentDecodingError from freqtrade import OperationalException diff --git a/freqtrade/exchange/interface.py b/freqtrade/exchange/interface.py index 1be84abe5..6121a98b3 100644 --- a/freqtrade/exchange/interface.py +++ b/freqtrade/exchange/interface.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Dict, Optional +from typing import Dict, List, Optional class Exchange(ABC): diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 3f0b6330b..0132e531d 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -1,5 +1,6 @@ import logging import time + from pymarketcap import Pymarketcap logger = logging.getLogger(__name__) diff --git a/freqtrade/main.py b/freqtrade/main.py index 58f3c7aee..513f33b3b 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -5,21 +5,21 @@ import logging import sys import time import traceback -import arrow from datetime import datetime -from typing import Dict, Optional, List +from typing import Dict, List, Optional +import arrow import requests from requests.adapters import TimeoutSauce from cachetools import cached, TTLCache -from freqtrade import __version__, exchange, persistence, rpc, DependencyException, \ - OperationalException -from freqtrade.analyze import get_signal, SignalType -from freqtrade.misc import State, get_state, update_state, parse_args, throttle, \ - load_config -from freqtrade.persistence import Trade +from freqtrade import (DependencyException, OperationalException, __version__, + exchange, persistence, rpc) +from freqtrade.analyze import SignalType, get_signal from freqtrade.fiat_convert import CryptoToFiatConverter +from freqtrade.misc import (State, get_state, load_config, parse_args, + throttle, update_state) +from freqtrade.persistence import Trade logger = logging.getLogger('freqtrade') diff --git a/freqtrade/misc.py b/freqtrade/misc.py index c018b7fbb..66e8b6c6b 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -3,10 +3,10 @@ import enum import json import logging import time -from typing import Any, Callable, List, Dict +from typing import Any, Callable, Dict, List -from jsonschema import validate, Draft4Validator -from jsonschema.exceptions import best_match, ValidationError +from jsonschema import Draft4Validator, validate +from jsonschema.exceptions import ValidationError, best_match from wrapt import synchronized from freqtrade import __version__ diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fe3ebd4f1..6d600b303 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1,20 +1,19 @@ # pragma pylint: disable=missing-docstring,W0212 - import logging -from typing import Tuple, Dict +from typing import Dict, Tuple import arrow from pandas import DataFrame, Series from tabulate import tabulate +import freqtrade.misc as misc +import freqtrade.optimize as optimize from freqtrade import exchange from freqtrade.analyze import populate_buy_trend, populate_sell_trend from freqtrade.exchange import Bittrex from freqtrade.main import min_roi_reached -import freqtrade.misc as misc from freqtrade.optimize import preprocess -import freqtrade.optimize as optimize from freqtrade.persistence import Trade logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 725bf37b0..71ddd33c6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,10 +11,11 @@ from functools import reduce from math import exp from operator import itemgetter -from hyperopt import fmin, tpe, hp, Trials, STATUS_OK, STATUS_FAIL, space_eval +from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from hyperopt.mongoexp import MongoTrials from pandas import DataFrame +from freqtrade import main # noqa from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.misc import load_config diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index d50c9acb4..0cac3bbe9 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -1,10 +1,11 @@ import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Optional, Dict +from typing import Dict, Optional import arrow -from sqlalchemy import Boolean, Column, DateTime, Float, Integer, String, create_engine +from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, + create_engine) from sqlalchemy.engine import Engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 55f1942f3..009714682 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1,21 +1,21 @@ import logging import re +from datetime import datetime, timedelta from decimal import Decimal -from datetime import timedelta, datetime -from typing import Callable, Any +from typing import Any, Callable import arrow from pandas import DataFrame from sqlalchemy import and_, func, text from tabulate import tabulate -from telegram import ParseMode, Bot, Update, ReplyKeyboardMarkup +from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater -from freqtrade import exchange, __version__ -from freqtrade.misc import get_state, State, update_state -from freqtrade.persistence import Trade +from freqtrade import __version__, exchange from freqtrade.fiat_convert import CryptoToFiatConverter +from freqtrade.misc import State, get_state, update_state +from freqtrade.persistence import Trade # Remove noisy log messages logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 788585345..2be8039ca 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -2,10 +2,10 @@ from datetime import datetime from unittest.mock import MagicMock -import pytest import arrow +import pytest from jsonschema import validate -from telegram import Message, Chat, Update +from telegram import Chat, Message, Update from freqtrade.misc import CONF_SCHEMA diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 040c45f26..d1afc4200 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -6,8 +6,9 @@ import arrow import pytest from pandas import DataFrame -from freqtrade.analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators, \ - get_signal, SignalType, populate_sell_trend +from freqtrade.analyze import (SignalType, get_signal, parse_ticker_dataframe, + populate_buy_trend, populate_indicators, + populate_sell_trend) @pytest.fixture diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index cd8af2c52..f9230a03f 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -1,7 +1,7 @@ import pandas -from freqtrade import analyze import freqtrade.optimize +from freqtrade import analyze _pairs = ['BTC_ETH'] diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 4aa9b1c4f..ddc1c8e29 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -1,10 +1,11 @@ # pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 import time -import pytest from unittest.mock import MagicMock -from freqtrade.fiat_convert import CryptoToFiatConverter, CryptoFiat +import pytest + +from freqtrade.fiat_convert import CryptoFiat, CryptoToFiatConverter def test_pair_convertion_object(): diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index ab39cebc9..ccb3f0534 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -1,22 +1,21 @@ # pragma pylint: disable=missing-docstring,C0103 import copy +import logging from unittest.mock import MagicMock +import arrow import pytest import requests -import logging -import arrow from sqlalchemy import create_engine +import freqtrade.main as main from freqtrade import DependencyException, OperationalException from freqtrade.analyze import SignalType from freqtrade.exchange import Exchanges -from freqtrade.main import create_trade, handle_trade, init, \ - get_target_bid, _process, execute_sell, check_handle_timedout -from freqtrade.misc import get_state, State +from freqtrade.main import (_process, check_handle_timedout, create_trade, + execute_sell, get_target_bid, handle_trade, init) +from freqtrade.misc import State, get_state from freqtrade.persistence import Trade -import freqtrade.main as main - # Test that main() can start backtesting or hyperopt. # and also ensure we can pass some specific arguments diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 9de40e8d5..244ff31e7 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -1,14 +1,14 @@ # pragma pylint: disable=missing-docstring,C0103 +import argparse import json import time -import argparse from copy import deepcopy import pytest from jsonschema import ValidationError -from freqtrade.misc import throttle, parse_args, load_config,\ - common_args_parser +from freqtrade.misc import (common_args_parser, load_config, parse_args, + throttle) def test_throttle(): From 64530c61967b4506b464da6339f8f2743131a2ba Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 10:13:41 +0200 Subject: [PATCH 11/19] remove unused variables --- freqtrade/tests/test_main.py | 10 +++++----- freqtrade/tests/test_persistence.py | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index ccb3f0534..b25f4ac6a 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -268,7 +268,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): assert trade.close_date is not None -def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog): +def test_handle_trade_roi(default_conf, ticker, mocker, caplog): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -300,7 +300,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog) assert ('freqtrade', logging.DEBUG, 'Executing sell due to ROI ...') in caplog.record_tuples -def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker, caplog): +def test_handle_trade_experimental(default_conf, ticker, mocker, caplog): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -352,7 +352,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mo handle_trade(trade) -def test_check_handle_timedout_buy(default_conf, ticker, health, limit_buy_order_old, mocker): +def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) cancel_order_mock = MagicMock() mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) @@ -384,7 +384,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, health, limit_buy_order assert len(trades) == 0 -def test_check_handle_timedout_sell(default_conf, ticker, health, limit_sell_order_old, mocker): +def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) cancel_order_mock = MagicMock() mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) @@ -417,7 +417,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, health, limit_sell_ord def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, - health, mocker): + mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) cancel_order_mock = MagicMock() mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index c2e2c13ea..70797f960 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,9 +1,10 @@ # pragma pylint: disable=missing-docstring +import os + import pytest -import os from freqtrade.exchange import Exchanges -from freqtrade.persistence import init, Trade +from freqtrade.persistence import Trade, init def test_init_create_session(default_conf, mocker): From 8fb404b0f8181c8399a80a3177364197be3cbdaa Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 11:15:44 +0200 Subject: [PATCH 12/19] ignore talib.abstract in pylint --- .pylintrc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index 65ed64830..dce99c067 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,10 +1,10 @@ [MASTER] -extension-pkg-whitelist=numpy,talib +extension-pkg-whitelist=numpy,talib,talib.abstract [BASIC] good-names=logger ignore=vendor [TYPECHECK] -ignored-modules=numpy,talib +ignored-modules=numpy,talib,talib.abstract From 6a433282dca7d236ece2990857c287bbf24aa425 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 11:17:04 +0200 Subject: [PATCH 13/19] fix literal comparison --- freqtrade/tests/test_misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 244ff31e7..63cfba627 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -85,12 +85,12 @@ def test_parse_args_invalid(): def test_parse_args_dynamic_whitelist(): args = parse_args(['--dynamic-whitelist'], '') - assert args.dynamic_whitelist is 20 + assert args.dynamic_whitelist == 20 def test_parse_args_dynamic_whitelist_10(): args = parse_args(['--dynamic-whitelist', '10'], '') - assert args.dynamic_whitelist is 10 + assert args.dynamic_whitelist == 10 def test_parse_args_dynamic_whitelist_invalid_values(): From 0d6051e6f9e778b40c7cca739952d98bba9e2291 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 11:17:16 +0200 Subject: [PATCH 14/19] formatting --- freqtrade/tests/test_acl_pair.py | 113 ++++++++++++++++--------------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index d1e42d444..7c42c676e 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -7,17 +7,17 @@ from freqtrade.main import refresh_whitelist, gen_pair_whitelist def whitelist_conf(): return { - "stake_currency": "BTC", - "exchange": { - "pair_whitelist": [ - "BTC_ETH", - "BTC_TKN", - "BTC_TRST", - "BTC_SWT", - "BTC_BCC" + 'stake_currency': 'BTC', + 'exchange': { + 'pair_whitelist': [ + 'BTC_ETH', + 'BTC_TKN', + 'BTC_TRST', + 'BTC_SWT', + 'BTC_BCC' ], - "pair_blacklist": [ - "BTC_BLK" + 'pair_blacklist': [ + 'BTC_BLK' ], }, } @@ -25,52 +25,51 @@ def whitelist_conf(): def get_market_summaries(): return [{ - "MarketName": "BTC-TKN", - "High": 0.00000919, - "Low": 0.00000820, - "Volume": 74339.61396015, - "Last": 0.00000820, - "BaseVolume": 1664, - "TimeStamp": "2014-07-09T07:19:30.15", - "Bid": 0.00000820, - "Ask": 0.00000831, - "OpenBuyOrders": 15, - "OpenSellOrders": 15, - "PrevDay": 0.00000821, - "Created": "2014-03-20T06:00:00", - "DisplayMarketName": "" + 'MarketName': 'BTC-TKN', + 'High': 0.00000919, + 'Low': 0.00000820, + 'Volume': 74339.61396015, + 'Last': 0.00000820, + 'BaseVolume': 1664, + 'TimeStamp': '2014-07-09T07:19:30.15', + 'Bid': 0.00000820, + 'Ask': 0.00000831, + 'OpenBuyOrders': 15, + 'OpenSellOrders': 15, + 'PrevDay': 0.00000821, + 'Created': '2014-03-20T06:00:00', + 'DisplayMarketName': '' }, { - "MarketName": "BTC-ETH", - "High": 0.00000072, - "Low": 0.00000001, - "Volume": 166340678.42280999, - "Last": 0.00000005, - "BaseVolume": 42, - "TimeStamp": "2014-07-09T07:21:40.51", - "Bid": 0.00000004, - "Ask": 0.00000005, - "OpenBuyOrders": 18, - "OpenSellOrders": 18, - "PrevDay": 0.00000002, - "Created": "2014-05-30T07:57:49.637", - "DisplayMarketName": "" + 'MarketName': 'BTC-ETH', + 'High': 0.00000072, + 'Low': 0.00000001, + 'Volume': 166340678.42280999, + 'Last': 0.00000005, + 'BaseVolume': 42, + 'TimeStamp': '2014-07-09T07:21:40.51', + 'Bid': 0.00000004, + 'Ask': 0.00000005, + 'OpenBuyOrders': 18, + 'OpenSellOrders': 18, + 'PrevDay': 0.00000002, + 'Created': '2014-05-30T07:57:49.637', + 'DisplayMarketName': '' }, { - "MarketName": "BTC-BLK", - "High": 0.00000072, - "Low": 0.00000001, - "Volume": 166340678.42280999, - "Last": 0.00000005, - "BaseVolume": 3, - "TimeStamp": "2014-07-09T07:21:40.51", - "Bid": 0.00000004, - "Ask": 0.00000005, - "OpenBuyOrders": 18, - "OpenSellOrders": 18, - "PrevDay": 0.00000002, - "Created": "2014-05-30T07:57:49.637", - "DisplayMarketName": "" - } - ] + 'MarketName': 'BTC-BLK', + 'High': 0.00000072, + 'Low': 0.00000001, + 'Volume': 166340678.42280999, + 'Last': 0.00000005, + 'BaseVolume': 3, + 'TimeStamp': '2014-07-09T07:21:40.51', + 'Bid': 0.00000004, + 'Ask': 0.00000005, + 'OpenBuyOrders': 18, + 'OpenSellOrders': 18, + 'PrevDay': 0.00000002, + 'Created': '2014-05-30T07:57:49.637', + 'DisplayMarketName': '' + }] def get_health(): @@ -95,7 +94,8 @@ def test_refresh_market_pair_not_in_whitelist(mocker): mocker.patch.dict('freqtrade.main._CONF', conf) mocker.patch.multiple('freqtrade.main.exchange', get_wallet_health=get_health) - refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist'] + ['BTC_XXX']) + refreshedwhitelist = refresh_whitelist( + conf['exchange']['pair_whitelist'] + ['BTC_XXX']) # List ordered by BaseVolume whitelist = ['BTC_ETH', 'BTC_TKN'] # Ensure all except those in whitelist are removed @@ -123,7 +123,8 @@ def test_refresh_whitelist_dynamic(mocker): get_market_summaries=get_market_summaries) # argument: use the whitelist dynamically by exchange-volume whitelist = ['BTC_TKN', 'BTC_ETH'] - refreshedwhitelist = refresh_whitelist(gen_pair_whitelist(conf['stake_currency'])) + refreshedwhitelist = refresh_whitelist( + gen_pair_whitelist(conf['stake_currency'])) assert whitelist == refreshedwhitelist From 02fcbbb6d2dea62733a203079fc0efa8d2c99a17 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 11:25:45 +0200 Subject: [PATCH 15/19] few flake8 fixes --- .../tests/exchange/test_exchange_bittrex.py | 30 +++++++++---------- freqtrade/tests/test_main.py | 7 +++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index 704ca0f6e..e01e7fa02 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -32,7 +32,7 @@ def _stub_config(): 'secret': ''} -class Fake_bittrex(): +class FakeBittrex(): def __init__(self, success=True): self.success = True # Believe in yourself self.result = None @@ -145,7 +145,7 @@ def test_exchange_bittrex_fee(): def test_exchange_bittrex_buy_good(mocker): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() uuid = wb.buy('BTC_ETH', 1, 1) assert uuid == fb.fake_buysell_limit(1, 2, 3)['result']['uuid'] @@ -156,7 +156,7 @@ def test_exchange_bittrex_buy_good(mocker): def test_exchange_bittrex_sell_good(mocker): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() uuid = wb.sell('BTC_ETH', 1, 1) assert uuid == fb.fake_buysell_limit(1, 2, 3)['result']['uuid'] @@ -167,7 +167,7 @@ def test_exchange_bittrex_sell_good(mocker): def test_exchange_bittrex_get_balance(mocker): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() bal = wb.get_balance('BTC_ETH') assert bal == fb.fake_get_balance(1)['result']['Balance'] @@ -178,7 +178,7 @@ def test_exchange_bittrex_get_balance(mocker): def test_exchange_bittrex_get_balances(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() bals = wb.get_balances() assert bals == fb.fake_get_balances()['result'] @@ -189,7 +189,7 @@ def test_exchange_bittrex_get_balances(): def test_exchange_bittrex_get_ticker(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() # Poll ticker, which updates the cache tick = wb.get_ticker('BTC_ETH') @@ -210,7 +210,7 @@ def test_exchange_bittrex_get_ticker(): def test_exchange_bittrex_get_ticker_bad(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() fb.result = {'success': True, 'result': {'Bid': 1}} # incomplete result with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'): @@ -222,15 +222,15 @@ def test_exchange_bittrex_get_ticker_bad(): wb.get_ticker('BTC_ETH') -def test_exchange_bittrex_get_ticker_historyOne(): +def test_exchange_bittrex_get_ticker_history_one(): wb = make_wrap_bittrex() - Fake_bittrex() + FakeBittrex() assert wb.get_ticker_history('BTC_ETH', 1) def test_exchange_bittrex_get_ticker_history(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() assert wb.get_ticker_history('BTC_ETH', 5) with pytest.raises(ValueError, match=r'.*Cannot parse tick_interval.*'): wb.get_ticker_history('BTC_ETH', 2) @@ -253,7 +253,7 @@ def test_exchange_bittrex_get_ticker_history(): def test_exchange_bittrex_get_order(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() order = wb.get_order('someUUID') assert order['id'] == 'ABC123' fb.success = False @@ -263,7 +263,7 @@ def test_exchange_bittrex_get_order(): def test_exchange_bittrex_cancel_order(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() wb.cancel_order('someUUID') with pytest.raises(btx.OperationalException, match=r'no such order'): fb.success = False @@ -284,7 +284,7 @@ def test_exchange_get_pair_detail_url(): def test_exchange_get_markets(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() x = wb.get_markets() assert x == ['__'] with pytest.raises(btx.OperationalException, match=r'market gone'): @@ -294,7 +294,7 @@ def test_exchange_get_markets(): def test_exchange_get_market_summaries(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() assert ['sum'] == wb.get_market_summaries() with pytest.raises(btx.OperationalException, match=r'no summary'): fb.success = False @@ -303,7 +303,7 @@ def test_exchange_get_market_summaries(): def test_exchange_get_wallet_health(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() x = wb.get_wallet_health() assert x[0]['Currency'] == 'BTC_ETH' with pytest.raises(btx.OperationalException, match=r'bad health'): diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index b25f4ac6a..049ed4284 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -17,11 +17,12 @@ from freqtrade.main import (_process, check_handle_timedout, create_trade, from freqtrade.misc import State, get_state from freqtrade.persistence import Trade -# Test that main() can start backtesting or hyperopt. -# and also ensure we can pass some specific arguments -# argument parsing is done in test_misc.py + def test_parse_args_backtesting(mocker): + """ Test that main() can start backtesting or hyperopt. + and also ensure we can pass some specific arguments + argument parsing is done in test_misc.py """ backtesting_mock = mocker.patch( 'freqtrade.optimize.backtesting.start', MagicMock()) with pytest.raises(SystemExit, match=r'0'): From c11102cf4a1a8b7d08ce3d036cf7d5093bf4522a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 11:29:41 +0200 Subject: [PATCH 16/19] another run of autopep8 --- freqtrade/optimize/hyperopt_conf.py | 8 ++++---- freqtrade/tests/conftest.py | 8 ++++---- freqtrade/tests/test_main.py | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt_conf.py b/freqtrade/optimize/hyperopt_conf.py index cbd95973a..8e044e549 100644 --- a/freqtrade/optimize/hyperopt_conf.py +++ b/freqtrade/optimize/hyperopt_conf.py @@ -15,10 +15,10 @@ def hyperopt_optimize_conf() -> dict: 'stake_currency': 'BTC', 'stake_amount': 0.01, "minimal_roi": { - '40': 0.0, - '30': 0.01, - '20': 0.02, - '0': 0.04, + '40': 0.0, + '30': 0.01, + '20': 0.02, + '0': 0.04, }, 'stoploss': -0.10, "bid_strategy": { diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 2be8039ca..c779aa726 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -20,10 +20,10 @@ def default_conf(): "fiat_display_currency": "USD", "dry_run": True, "minimal_roi": { - "40": 0.0, - "30": 0.01, - "20": 0.02, - "0": 0.04 + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 }, "stoploss": -0.10, "unfilledtimeout": 600, diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 049ed4284..97bef2257 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -18,7 +18,6 @@ from freqtrade.misc import State, get_state from freqtrade.persistence import Trade - def test_parse_args_backtesting(mocker): """ Test that main() can start backtesting or hyperopt. and also ensure we can pass some specific arguments From 11cbb9188bab7b500c9de04da6110b31b1fa9380 Mon Sep 17 00:00:00 2001 From: Anton Ermak Date: Thu, 11 Jan 2018 12:24:05 +0700 Subject: [PATCH 17/19] Set timeout for bittrex only --- freqtrade/exchange/bittrex.py | 18 ++++++++++++++++++ freqtrade/main.py | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 4883db037..8e256a936 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -1,4 +1,5 @@ import logging +import requests from typing import List, Dict, Optional from bittrex.bittrex import Bittrex as _Bittrex, API_V2_0, API_V1_1 @@ -13,6 +14,21 @@ _API: _Bittrex = None _API_V2: _Bittrex = None _EXCHANGE_CONF: dict = {} +# API socket timeout +API_TIMEOUT = 60 + + +def custom_requests(request_url, apisign): + """ + Set timeout for requests + """ + print("Dispatch", request_url, apisign) + return requests.get( + request_url, + headers={"apisign": apisign}, + timeout=API_TIMEOUT + ).json() + class Bittrex(Exchange): """ @@ -31,12 +47,14 @@ class Bittrex(Exchange): api_secret=_EXCHANGE_CONF['secret'], calls_per_second=1, api_version=API_V1_1, + dispatch=custom_requests ) _API_V2 = _Bittrex( api_key=_EXCHANGE_CONF['key'], api_secret=_EXCHANGE_CONF['secret'], calls_per_second=1, api_version=API_V2_0, + dispatch=custom_requests ) self.cached_ticker = {} diff --git a/freqtrade/main.py b/freqtrade/main.py index efd0d7c44..5e8680b85 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -10,7 +10,6 @@ from datetime import datetime from typing import Dict, Optional, List import requests -from requests.adapters import TimeoutSauce from cachetools import cached, TTLCache from freqtrade import __version__, exchange, persistence, rpc, DependencyException, \ @@ -25,23 +24,6 @@ logger = logging.getLogger('freqtrade') _CONF = {} -DEFAULT_TIMEOUT = 120 - - -# Set requests default timeout (fix for #127) -class DefaultTimeout(TimeoutSauce): - def __init__(self, *args, **kwargs): - connect = kwargs.get('connect', DEFAULT_TIMEOUT) - read = kwargs.get('read', connect) - if connect is None: - connect = DEFAULT_TIMEOUT - if read is None: - read = connect - super(DefaultTimeout, self).__init__(connect=connect, read=read) - - -requests.adapters.TimeoutSauce = DefaultTimeout - def refresh_whitelist(whitelist: List[str]) -> List[str]: """ From bb91fdbaf93cc7fa1fab0e87eb555f1acc71bb3d Mon Sep 17 00:00:00 2001 From: Anton Ermak Date: Thu, 11 Jan 2018 13:26:49 +0700 Subject: [PATCH 18/19] oops, print removed --- freqtrade/exchange/bittrex.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 8e256a936..7a08acdbc 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -22,7 +22,6 @@ def custom_requests(request_url, apisign): """ Set timeout for requests """ - print("Dispatch", request_url, apisign) return requests.get( request_url, headers={"apisign": apisign}, From 3a902289f11e8d30fa747ee34e40a159333d4ab2 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 11 Jan 2018 13:58:06 +0200 Subject: [PATCH 19/19] testdata path to use os.path.join (#360) --- freqtrade/misc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 66e8b6c6b..3d70f6b25 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -3,6 +3,7 @@ import enum import json import logging import time +import os from typing import Any, Callable, Dict, List from jsonschema import Draft4Validator, validate @@ -129,7 +130,7 @@ def parse_args(args: List[str], description: str): '-dd', '--datadir', help='path to backtest data (default freqdata/tests/testdata', dest='datadir', - default='freqtrade/tests/testdata', + default=os.path.join('freqtrade', 'tests', 'testdata'), type=str, metavar='PATH', )