From b7e297ebda06cf654e3b0a6e0228261d5f3efaf1 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 11:25:12 +0200 Subject: [PATCH 01/11] remove unused loop variable --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a21bdb61a..8e244bdda 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -26,7 +26,7 @@ def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow] :return: tuple containing min_date, max_date """ all_dates = Series([]) - for pair, pair_data in data.items(): + for pair_data in data.values(): all_dates = all_dates.append(pair_data['date']) all_dates.sort_values(inplace=True) return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) From a7a7c3712168c01bb8b35dec738cd9b7d0821084 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 11:25:35 +0200 Subject: [PATCH 02/11] add day counter to timeframe --- freqtrade/optimize/backtesting.py | 5 ++++- freqtrade/tests/optimize/test_backtesting.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8e244bdda..a78c9e197 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -212,7 +212,10 @@ def start(args): preprocessed = optimize.tickerdata_to_dataframe(data) # Print timeframe min_date, max_date = get_timeframe(preprocessed) - logger.info('Measuring data from %s up to %s ...', min_date.isoformat(), max_date.isoformat()) + logger.info('Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date-min_date).days) # Execute backtest and print results sell_profit_only = config.get('experimental', {}).get('sell_profit_only', False) use_sell_signal = config.get('experimental', {}).get('use_sell_signal', False) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f88cdd9b9..f5db4d037 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -193,7 +193,8 @@ def test_backtest_start(default_conf, mocker, caplog): # check the logs, that will contain the backtest result exists = ['Using max_open_trades: 1 ...', 'Using stake_amount: 0.001 ...', - 'Measuring data from 2017-11-14T21:17:00+00:00 up to 2017-11-14T22:59:00+00:00 ...'] + 'Measuring data from 2017-11-14T21:17:00+00:00 ' + 'up to 2017-11-14T22:59:00+00:00 (0 days)..'] for line in exists: assert ('freqtrade.optimize.backtesting', logging.INFO, From f33923c7842ad0c440d5d5f9bd028331ea4b9ac3 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 11:48:34 +0200 Subject: [PATCH 03/11] fix typings for hyperopt code --- freqtrade/optimize/hyperopt.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4c73dc76f..6f63c159f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -10,7 +10,7 @@ import sys from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, List +from typing import Dict, Any, Callable import numpy import talib.abstract as ta @@ -35,7 +35,7 @@ logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) logger = logging.getLogger(__name__) -# set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data +# set TARGET_TRADES to suit your number concurrent trades so its realistic to the number of days TARGET_TRADES = 600 TOTAL_TRIES = 0 _CURRENT_TRIES = 0 @@ -225,7 +225,7 @@ def calculate_loss(total_profit: float, trade_count: int, trade_duration: float) return trade_loss + profit_loss + duration_loss -def generate_roi_table(params): +def generate_roi_table(params) -> Dict[str, float]: roi_table = {} roi_table["0"] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] roi_table[str(params['roi_t3'])] = params['roi_p1'] + params['roi_p2'] @@ -235,7 +235,7 @@ def generate_roi_table(params): return roi_table -def roi_space() -> List[Dict]: +def roi_space() -> Dict[str, Any]: return { 'roi_t1': hp.quniform('roi_t1', 10, 220, 10), 'roi_t2': hp.quniform('roi_t2', 10, 120, 10), @@ -246,13 +246,13 @@ def roi_space() -> List[Dict]: } -def stoploss_space() -> Dict: +def stoploss_space() -> Dict[str, Any]: return { 'stoploss': hp.uniform('stoploss', -0.5, -0.02), } -def indicator_space() -> List[Dict]: +def indicator_space() -> Dict[str, Any]: """ Define your Hyperopt space for searching strategy parameters """ @@ -312,11 +312,11 @@ def indicator_space() -> List[Dict]: } -def hyperopt_space() -> List[Dict]: +def hyperopt_space() -> Dict[str, Any]: return {**indicator_space(), **roi_space(), **stoploss_space()} -def buy_strategy_generator(params) -> None: +def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """ From 95ab7c84bcfa837e0709481cdfdd9325b48850b9 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:41:41 +0200 Subject: [PATCH 04/11] remove unnecessary else --- freqtrade/optimize/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 08630af40..a0eb5b8a2 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -22,8 +22,8 @@ def trim_tickerlist(tickerlist, timerange): return tickerlist[0:start] elif stype == ('index', 'index'): return tickerlist[start:stop] - else: - return tickerlist + + return tickerlist def load_tickerdata_file(datadir, pair, ticker_interval, From 5505845c6fd0210bceceef2729a2dad094ae1754 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:48:53 +0200 Subject: [PATCH 05/11] remove unused method parameter --- freqtrade/main.py | 4 ++-- freqtrade/tests/test_main.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 3bf945b99..302d4cab0 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -55,7 +55,7 @@ def refresh_whitelist(whitelist: List[str]) -> List[str]: return final_list -def process_maybe_execute_buy(conf, interval): +def process_maybe_execute_buy(interval): """ Tries to execute a buy trade in a safe way :return: True if executed @@ -115,7 +115,7 @@ def _process(interval: int, nb_assets: Optional[int] = 0) -> bool: # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() if len(trades) < _CONF['max_open_trades']: - state_changed = process_maybe_execute_buy(_CONF, interval) + state_changed = process_maybe_execute_buy(interval) for trade in trades: state_changed |= process_maybe_execute_sell(trade, interval) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index cf0b5283b..225f5e28c 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -50,9 +50,9 @@ def test_main_start_hyperopt(mocker): def test_process_maybe_execute_buy(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.create_trade', return_value=True) - assert main.process_maybe_execute_buy(default_conf, int(default_conf['ticker_interval'])) + assert main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) mocker.patch('freqtrade.main.create_trade', return_value=False) - assert not main.process_maybe_execute_buy(default_conf, int(default_conf['ticker_interval'])) + assert not main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) def test_process_maybe_execute_sell(default_conf, mocker): @@ -71,7 +71,7 @@ def test_process_maybe_execute_sell(default_conf, mocker): def test_process_maybe_execute_buy_exception(default_conf, mocker, caplog): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.create_trade', MagicMock(side_effect=DependencyException)) - main.process_maybe_execute_buy(default_conf, int(default_conf['ticker_interval'])) + main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) tt.log_has('Unable to create trade:', caplog.record_tuples) From 42919e8864c6b1ff53bd867c6a82ceb7dd460413 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:49:14 +0200 Subject: [PATCH 06/11] give type hint for _CONF --- freqtrade/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 302d4cab0..c10fb2c5b 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -6,7 +6,7 @@ import sys import time import traceback from datetime import datetime -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Any import arrow import requests @@ -23,7 +23,7 @@ from freqtrade.strategy.strategy import Strategy logger = logging.getLogger('freqtrade') -_CONF = {} +_CONF: Dict[str, Any] = {} def refresh_whitelist(whitelist: List[str]) -> List[str]: From e14007ced42bb12d248201cc18b1eb5cb3d9b05b Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:52:39 +0200 Subject: [PATCH 07/11] sort imports --- freqtrade/tests/optimize/test_optimize.py | 1 - freqtrade/tests/test_analyze.py | 4 ++-- freqtrade/tests/test_main.py | 2 +- freqtrade/tests/test_misc.py | 6 +++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 02b675cf9..22bffc1ff 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -2,7 +2,6 @@ import os import logging -# from unittest.mock import MagicMock from shutil import copyfile from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index f77a2d71a..516bb3607 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -1,13 +1,13 @@ # pragma pylint: disable=missing-docstring,W0621 +import datetime import json from unittest.mock import MagicMock -import freqtrade.tests.conftest as tt # test tools import arrow -import datetime import pytest from pandas import DataFrame +import freqtrade.tests.conftest as tt # test tools from freqtrade.analyze import (get_signal, parse_ticker_dataframe, populate_buy_trend, populate_indicators, populate_sell_trend) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 225f5e28c..0c9e594a2 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -2,7 +2,6 @@ import copy import logging from unittest.mock import MagicMock -import freqtrade.tests.conftest as tt # test tools import arrow import pytest @@ -10,6 +9,7 @@ import requests from sqlalchemy import create_engine import freqtrade.main as main +import freqtrade.tests.conftest as tt # test tools from freqtrade import DependencyException, OperationalException from freqtrade.exchange import Exchanges from freqtrade.main import (_process, check_handle_timedout, create_trade, diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index fb768b89c..52a8c3991 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -3,13 +3,13 @@ import argparse import json import time from copy import deepcopy +from unittest.mock import MagicMock import pytest -from unittest.mock import MagicMock from jsonschema import ValidationError -from freqtrade.misc import (common_args_parser, load_config, parse_args, - throttle, file_dump_json, parse_timerange) +from freqtrade.misc import (common_args_parser, file_dump_json, load_config, + parse_args, parse_timerange, throttle) def test_throttle(): From 0ff56c6e8d4e70d24343a967656fd40470088225 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:54:15 +0200 Subject: [PATCH 08/11] use uppercase constant --- freqtrade/tests/optimize/test_optimize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 22bffc1ff..b2820f367 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -9,7 +9,7 @@ from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ download_backtesting_testdata, load_tickerdata_file # Change this if modifying BTC_UNITEST testdatafile -_btc_unittest_length = 13681 +_BTC_UNITTEST_LENGTH = 13681 def _backup_file(file: str, copy_file: bool = False) -> None: @@ -209,7 +209,7 @@ def test_download_backtesting_testdata2(default_conf, mocker): def test_load_tickerdata_file(): assert not load_tickerdata_file(None, 'BTC_UNITEST', 7) tickerdata = load_tickerdata_file(None, 'BTC_UNITEST', 1) - assert _btc_unittest_length == len(tickerdata) + assert _BTC_UNITTEST_LENGTH == len(tickerdata) def test_init(default_conf, mocker): From a5690e707d3b0942a114bdee1c352cc833b38c58 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:56:56 +0200 Subject: [PATCH 09/11] remove unused parameters --- freqtrade/tests/optimize/test_optimize.py | 2 +- freqtrade/tests/test_main.py | 8 ++++---- freqtrade/tests/test_misc.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index b2820f367..c69284c32 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -198,7 +198,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): _clean_test_file(file2) -def test_download_backtesting_testdata2(default_conf, mocker): +def test_download_backtesting_testdata2(mocker): tick = [{'T': 'bar'}, {'T': 'foo'}] mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 0c9e594a2..8ac06ee69 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -256,7 +256,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, mocker): create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) -def test_create_trade_no_signal(default_conf, ticker, mocker): +def test_create_trade_no_signal(default_conf, mocker): default_conf['dry_run'] = True mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.get_signal', MagicMock(return_value=(False, False))) @@ -308,7 +308,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): assert trade.close_date is not None -def test_handle_overlpapping_signals(default_conf, ticker, mocker, caplog): +def test_handle_overlpapping_signals(default_conf, ticker, mocker): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -471,7 +471,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mo assert len(trades) == 0 -def test_handle_timedout_limit_buy(default_conf, mocker): +def test_handle_timedout_limit_buy(mocker): cancel_order = MagicMock() mocker.patch('freqtrade.exchange.cancel_order', cancel_order) Trade.session = MagicMock() @@ -519,7 +519,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, assert trade_sell.is_open is True -def test_handle_timedout_limit_sell(default_conf, mocker): +def test_handle_timedout_limit_sell(mocker): cancel_order = MagicMock() mocker.patch('freqtrade.exchange.cancel_order', cancel_order) trade = MagicMock() diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 52a8c3991..96743dfab 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -124,7 +124,7 @@ def test_parse_args_backtesting_custom(): assert call_args.refresh_pairs is True -def test_parse_args_hyperopt_custom(mocker): +def test_parse_args_hyperopt_custom(): args = ['-c', 'test_conf.json', 'hyperopt', '--epochs', '20'] call_args = parse_args(args, '') assert call_args.config == 'test_conf.json' @@ -134,7 +134,7 @@ def test_parse_args_hyperopt_custom(mocker): assert call_args.func is not None -def test_file_dump_json(default_conf, mocker): +def test_file_dump_json(mocker): file_open = mocker.patch('freqtrade.misc.open', MagicMock()) json_dump = mocker.patch('json.dump', MagicMock()) file_dump_json('somefile', [1, 2, 3]) From 1eebbebed100e3c55a0529e32e27466c9d2edffc Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:57:09 +0200 Subject: [PATCH 10/11] fix assert order --- freqtrade/tests/optimize/test_optimize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index c69284c32..8a18b58e3 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -224,4 +224,4 @@ def test_tickerdata_to_dataframe(): tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) tickerlist = {'BTC_UNITEST': tick} data = optimize.tickerdata_to_dataframe(tickerlist) - assert 100 == len(data['BTC_UNITEST']) + assert len(data['BTC_UNITEST']) == 100 From 67995a2f490150014ad338492f7c00a249927524 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 19:01:54 +0200 Subject: [PATCH 11/11] remove unnecessary else statements --- freqtrade/main.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index c10fb2c5b..efcb27543 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -64,12 +64,12 @@ def process_maybe_execute_buy(interval): # Create entity and execute trade if create_trade(float(_CONF['stake_amount']), interval): return True - else: - logger.info( - 'Checked all whitelisted currencies. ' - 'Found no suitable entry positions for buying. Will keep looking ...' - ) - return False + + logger.info( + 'Checked all whitelisted currencies. ' + 'Found no suitable entry positions for buying. Will keep looking ...' + ) + return False except DependencyException as exception: logger.warning('Unable to create trade: %s', exception) return False @@ -159,16 +159,16 @@ def handle_timedout_limit_buy(trade: Trade, order: Dict) -> bool: rpc.send_msg('*Timeout:* Unfilled buy order for {} cancelled'.format( trade.pair.replace('_', '/'))) return True - else: - # if trade is partially complete, edit the stake details for the trade - # and close the order - trade.amount = order['amount'] - order['remaining'] - trade.stake_amount = trade.amount * trade.open_rate - trade.open_order_id = None - logger.info('Partial buy order timeout for %s.', trade) - rpc.send_msg('*Timeout:* Remaining buy order for {} cancelled'.format( - trade.pair.replace('_', '/'))) - return False + + # if trade is partially complete, edit the stake details for the trade + # and close the order + trade.amount = order['amount'] - order['remaining'] + trade.stake_amount = trade.amount * trade.open_rate + trade.open_order_id = None + logger.info('Partial buy order timeout for %s.', trade) + rpc.send_msg('*Timeout:* Remaining buy order for {} cancelled'.format( + trade.pair.replace('_', '/'))) + return False # FIX: 20180110, should cancel_order() be cond. or unconditionally called? @@ -189,9 +189,9 @@ def handle_timedout_limit_sell(trade: Trade, order: Dict) -> bool: trade.pair.replace('_', '/'))) logger.info('Sell order timeout for %s.', trade) return True - else: - # TODO: figure out how to handle partially complete sell orders - return False + + # TODO: figure out how to handle partially complete sell orders + return False def check_handle_timedout(timeoutvalue: int) -> None: