From e01c85bb3a765767099432cb113f3ee202b5a584 Mon Sep 17 00:00:00 2001 From: gcarq Date: Wed, 8 Nov 2017 22:43:47 +0100 Subject: [PATCH 01/29] add argparse and implement basic arguments --- freqtrade/analyze.py | 2 -- freqtrade/main.py | 38 ++++++++++++++++++++++++++------------ freqtrade/misc.py | 33 +++++++++++++++++++++++++++++++++ freqtrade/persistence.py | 2 -- 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 32bce45e0..daa2a1436 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -10,8 +10,6 @@ from freqtrade import exchange from freqtrade.exchange import Bittrex, get_ticker_history from freqtrade.vendor.qtpylib.indicators import awesome_oscillator -logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) diff --git a/freqtrade/main.py b/freqtrade/main.py index 2371f90a9..925bf340c 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -5,21 +5,19 @@ import logging import time import traceback from datetime import datetime -from typing import Dict, Optional from signal import signal, SIGINT, SIGABRT, SIGTERM +from typing import Dict, Optional import requests from jsonschema import validate from freqtrade import __version__, exchange, persistence from freqtrade.analyze import get_buy_signal -from freqtrade.misc import CONF_SCHEMA, State, get_state, update_state +from freqtrade.misc import CONF_SCHEMA, State, get_state, update_state, build_arg_parser from freqtrade.persistence import Trade from freqtrade.rpc import telegram -logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) +logger = logging.getLogger('freqtrade') _CONF = {} @@ -42,7 +40,7 @@ def _process() -> bool: Trade.session.add(trade) state_changed = True else: - logging.info('Got no buy signal...') + logger.info('Got no buy signal...') except ValueError: logger.exception('Unable to create trade') @@ -163,7 +161,10 @@ def create_trade(stake_amount: float) -> Optional[Trade]: if one pair triggers the buy_signal a new trade record gets created :param stake_amount: amount of btc to spend """ - logger.info('Creating new trade with stake_amount: %f ...', stake_amount) + logger.info( + 'Checking buy signals to create a new trade with stake_amount: %f ...', + stake_amount + ) whitelist = copy.deepcopy(_CONF['exchange']['pair_whitelist']) # Check if stake_amount is fulfilled if exchange.get_balance(_CONF['stake_currency']) < stake_amount: @@ -255,15 +256,28 @@ def main(): Loads and validates the config and handles the main loop :return: None """ - logger.info('Starting freqtrade %s', __version__) - global _CONF - with open('config.json') as file: - _CONF = json.load(file) + args = build_arg_parser().parse_args() + # Initialize logger + logging.basicConfig( + level=args.loglevel, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + ) + + logger.info( + 'Starting freqtrade %s (loglevel=%s)', + __version__, + logging.getLevelName(args.loglevel) + ) + + # Load and validate configuration + with open(args.config) as file: + _CONF = json.load(file) logger.info('Validating configuration ...') validate(_CONF, CONF_SCHEMA) + # Initialize all modules and start main loop init(_CONF) old_state = get_state() logger.info('Initial State: %s', old_state) @@ -273,7 +287,7 @@ def main(): # Log state transition if new_state != old_state: telegram.send_msg('*Status:* `{}`'.format(new_state.name.lower())) - logging.info('Changing state to: %s', new_state.name) + logger.info('Changing state to: %s', new_state.name) if new_state == State.STOPPED: time.sleep(1) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 585aee3de..4787ef2f9 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -1,7 +1,11 @@ +import argparse import enum +import logging from wrapt import synchronized +from freqtrade import __version__ + class State(enum.Enum): RUNNING = 0 @@ -32,6 +36,35 @@ def get_state() -> State: return _STATE +def build_arg_parser() -> argparse.ArgumentParser: + """ Builds and returns an ArgumentParser instance """ + parser = argparse.ArgumentParser( + description='Simple High Frequency Trading Bot for crypto currencies' + ) + parser.add_argument( + '-c', '--config', + help='specify configuration file (default: config.json)', + dest='config', + default='config.json', + type=str, + metavar='PATH', + ) + parser.add_argument( + '-v', '--verbose', + help='be verbose', + action='store_const', + dest='loglevel', + const=logging.DEBUG, + default=logging.INFO, + ) + parser.add_argument( + '--version', + action='version', + version='%(prog)s {}'.format(__version__), + ) + return parser + + # Required json-schema for user specified config CONF_SCHEMA = { 'type': 'object', diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 724ad7dcb..f90134275 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -9,8 +9,6 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker -logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) _CONF = {} From 8b464033ffcb6053f249ffe4c04d376d38fcd50b Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 10 Nov 2017 17:26:52 +0100 Subject: [PATCH 02/29] add missing commands to README --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 99546632f..0eea256ae 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,11 @@ Persistence is achieved through sqlite. * /status [table]: Lists all open trades * /count: Displays number of open trades * /profit: Lists cumulative profit from all finished trades -* /forcesell : Instantly sells the given trade (Ignoring `minimum_roi`). +* /forcesell |all: Instantly sells the given trade (Ignoring `minimum_roi`). * /performance: Show performance of each finished trade grouped by pair +* /balance: Show account balance per currency +* /help: Show help message +* /version: Show version ### Config `minimal_roi` is a JSON object where the key is a duration From b709ccbf536bd04d2b4ebc4f237c7d433c045c04 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 10 Nov 2017 17:56:03 +0100 Subject: [PATCH 03/29] enhance logging messages --- freqtrade/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 925bf340c..1e31a0ca7 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -40,7 +40,10 @@ def _process() -> bool: Trade.session.add(trade) state_changed = True else: - logger.info('Got no buy signal...') + logger.info( + 'Checked all whitelisted currencies. ' + 'Found no suitable entry positions for buying. Will keep looking ...' + ) except ValueError: logger.exception('Unable to create trade') @@ -83,7 +86,10 @@ def close_trade_if_fulfilled(trade: Trade) -> bool: and trade.close_rate is not None \ and trade.open_order_id is None: trade.is_open = False - logger.info('No open orders found and trade is fulfilled. Marking %s as closed ...', trade) + logger.info( + 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', + trade + ) return True return False From 3126dcfceae8085aa7653a6c1504448a14a37629 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 10 Nov 2017 23:39:49 +0100 Subject: [PATCH 04/29] drop sleep_time and use python-bittrex request delay --- freqtrade/exchange/__init__.py | 4 ---- freqtrade/exchange/bittrex.py | 5 ----- freqtrade/exchange/interface.py | 8 -------- freqtrade/main.py | 2 -- 4 files changed, 19 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index feebd6bf1..e06ab5207 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -161,9 +161,5 @@ def get_name() -> str: return _API.name -def get_sleep_time() -> float: - return _API.sleep_time - - def get_fee() -> float: return _API.fee diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 8e9994354..88b2bce6b 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -21,11 +21,6 @@ class Bittrex(Exchange): TICKER_METHOD: str = BASE_URL + '/Api/v2.0/pub/market/GetTicks' PAIR_DETAIL_METHOD: str = BASE_URL + '/Market/Index' - @property - def sleep_time(self) -> float: - """ Sleep time to avoid rate limits, used in the main loop """ - return 25 - def __init__(self, config: dict) -> None: global _API, _EXCHANGE_CONF diff --git a/freqtrade/exchange/interface.py b/freqtrade/exchange/interface.py index 468af7343..f2bd3b0dc 100644 --- a/freqtrade/exchange/interface.py +++ b/freqtrade/exchange/interface.py @@ -18,14 +18,6 @@ class Exchange(ABC): :return: percentage in float """ - @property - @abstractmethod - def sleep_time(self) -> float: - """ - Sleep time in seconds for the main loop to avoid API rate limits. - :return: float - """ - @abstractmethod def buy(self, pair: str, rate: float, amount: float) -> str: """ diff --git a/freqtrade/main.py b/freqtrade/main.py index 1e31a0ca7..3c7c74355 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -299,8 +299,6 @@ def main(): time.sleep(1) elif new_state == State.RUNNING: _process() - # We need to sleep here because otherwise we would run into bittrex rate limit - time.sleep(exchange.get_sleep_time()) old_state = new_state From 83fd27e0311471b1bd2f33ecfc6a1a9b9eb1f641 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 11 Nov 2017 08:32:35 +0200 Subject: [PATCH 05/29] add sar reversal as trigger --- freqtrade/tests/test_hyperopt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index 50f3b94f4..d7e7cbe3f 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -50,6 +50,7 @@ def buy_strategy_generator(params): 'ao_cross_zero': (crossed_above(dataframe['ao'], 0.0)), 'ema5_cross_ema10': (crossed_above(dataframe['ema5'], dataframe['ema10'])), 'macd_cross_signal': (crossed_above(dataframe['macd'], dataframe['macdsignal'])), + 'sar_reversal': (crossed_above(dataframe['close'], dataframe['sar'])), } conditions.append(triggers.get(params['trigger']['type'])) @@ -125,6 +126,7 @@ def test_hyperopt(backtest_conf, backdata, mocker): {'type': 'ao_cross_zero'}, {'type': 'ema5_cross_ema10'}, {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, ]), } trials = Trials() From 274972f7afc480b4e0adc28f0acf0e3df47d5fcf Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 11 Nov 2017 09:26:05 +0200 Subject: [PATCH 06/29] make faststoch trigger use crossed_above helper --- freqtrade/tests/test_hyperopt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index d7e7cbe3f..e000e6a5b 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -42,11 +42,10 @@ def buy_strategy_generator(params): prevsma = dataframe['sma'].shift(1) conditions.append(dataframe['sma'] > prevsma) - prev_fastd = dataframe['fastd'].shift(1) # TRIGGERS triggers = { 'lower_bb': dataframe['tema'] <= dataframe['blower'], - 'faststoch10': (dataframe['fastd'] >= 10) & (prev_fastd < 10), + 'faststoch10': (crossed_above(dataframe['fastd'], 10.0)), 'ao_cross_zero': (crossed_above(dataframe['ao'], 0.0)), 'ema5_cross_ema10': (crossed_above(dataframe['ema5'], dataframe['ema10'])), 'macd_cross_signal': (crossed_above(dataframe['macd'], dataframe['macdsignal'])), From 3db13fae1390a57fc3434900adc8ff13b31d5760 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 11 Nov 2017 09:35:04 +0200 Subject: [PATCH 07/29] add green_candle guard --- freqtrade/tests/test_hyperopt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index e000e6a5b..fce55fb08 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -38,6 +38,8 @@ def buy_strategy_generator(params): conditions.append(dataframe['rsi'] < params['rsi']['value']) if params['over_sar']['enabled']: conditions.append(dataframe['close'] > dataframe['sar']) + if params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) if params['uptrend_sma']['enabled']: prevsma = dataframe['sma'].shift(1) conditions.append(dataframe['sma'] > prevsma) @@ -115,6 +117,10 @@ def test_hyperopt(backtest_conf, backdata, mocker): {'enabled': False}, {'enabled': True} ]), + 'over_sar': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), 'uptrend_sma': hp.choice('uptrend_sma', [ {'enabled': False}, {'enabled': True} From 906caf329b9b0a9687a6a75c53f42b8090525f64 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 11 Nov 2017 10:05:09 +0200 Subject: [PATCH 08/29] remove two unused or poorly performing indicators --- freqtrade/analyze.py | 2 -- freqtrade/tests/test_hyperopt.py | 8 +------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 5c734b24e..e9ef9c290 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -39,9 +39,7 @@ def populate_indicators(dataframe: DataFrame) -> DataFrame: dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) dataframe['mfi'] = ta.MFI(dataframe) - dataframe['cci'] = ta.CCI(dataframe) dataframe['rsi'] = ta.RSI(dataframe) - dataframe['mom'] = ta.MOM(dataframe) dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index fce55fb08..40dd62607 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -32,8 +32,6 @@ def buy_strategy_generator(params): conditions.append(dataframe['fastd'] < params['fastd']['value']) if params['adx']['enabled']: conditions.append(dataframe['adx'] > params['adx']['value']) - if params['cci']['enabled']: - conditions.append(dataframe['cci'] < params['cci']['value']) if params['rsi']['enabled']: conditions.append(dataframe['rsi'] < params['rsi']['value']) if params['over_sar']['enabled']: @@ -99,11 +97,7 @@ def test_hyperopt(backtest_conf, backdata, mocker): ]), 'adx': hp.choice('adx', [ {'enabled': False}, - {'enabled': True, 'value': hp.uniform('adx-value', 10, 30)} - ]), - 'cci': hp.choice('cci', [ - {'enabled': False}, - {'enabled': True, 'value': hp.uniform('cci-value', -150, -100)} + {'enabled': True, 'value': hp.uniform('adx-value', 10, 50)} ]), 'rsi': hp.choice('rsi', [ {'enabled': False}, From a4284351e3d10e11bc95a79f8c9194ad1a5943fb Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 11 Nov 2017 10:26:04 +0200 Subject: [PATCH 09/29] fix green_candle --- freqtrade/tests/test_hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index 40dd62607..b1c546f34 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -111,7 +111,7 @@ def test_hyperopt(backtest_conf, backdata, mocker): {'enabled': False}, {'enabled': True} ]), - 'over_sar': hp.choice('green_candle', [ + 'green_candle': hp.choice('green_candle', [ {'enabled': False}, {'enabled': True} ]), From cf79b15651b55abab8cd8031aa7d67eb948bdf3d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 11 Nov 2017 11:50:10 +0200 Subject: [PATCH 10/29] use discrete values for filters --- freqtrade/tests/test_hyperopt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index b1c546f34..d41f1a7ef 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -89,19 +89,19 @@ def test_hyperopt(backtest_conf, backdata, mocker): space = { 'mfi': hp.choice('mfi', [ {'enabled': False}, - {'enabled': True, 'value': hp.uniform('mfi-value', 5, 15)} + {'enabled': True, 'value': hp.quniform('mfi-value', 5, 15, 1)} ]), 'fastd': hp.choice('fastd', [ {'enabled': False}, - {'enabled': True, 'value': hp.uniform('fastd-value', 5, 40)} + {'enabled': True, 'value': hp.quniform('fastd-value', 5, 40, 1)} ]), 'adx': hp.choice('adx', [ {'enabled': False}, - {'enabled': True, 'value': hp.uniform('adx-value', 10, 50)} + {'enabled': True, 'value': hp.quniform('adx-value', 10, 50, 1)} ]), 'rsi': hp.choice('rsi', [ {'enabled': False}, - {'enabled': True, 'value': hp.uniform('rsi-value', 20, 30)} + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 30, 1)} ]), 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ {'enabled': False}, From 8f817a3634931cdfa51f9f8fac4a920c88ff094c Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 11 Nov 2017 15:29:31 +0100 Subject: [PATCH 11/29] use TTLCache for get_ticker_history --- freqtrade/exchange/__init__.py | 2 ++ freqtrade/exchange/bittrex.py | 2 +- freqtrade/main.py | 1 + requirements.txt | 1 + setup.py | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index e06ab5207..dc9dfa427 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -4,6 +4,7 @@ from random import randint from typing import List, Dict, Any, Optional import arrow +from cachetools import cached, TTLCache from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.interface import Exchange @@ -127,6 +128,7 @@ def get_ticker(pair: str) -> dict: return _API.get_ticker(pair) +@cached(TTLCache(maxsize=100, ttl=30)) def get_ticker_history(pair: str, tick_interval: Optional[int] = 5) -> List: return _API.get_ticker_history(pair, tick_interval) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 88b2bce6b..d65c49d42 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -28,7 +28,7 @@ class Bittrex(Exchange): _API = _Bittrex( api_key=_EXCHANGE_CONF['key'], api_secret=_EXCHANGE_CONF['secret'], - calls_per_second=5, + calls_per_second=3, ) @property diff --git a/freqtrade/main.py b/freqtrade/main.py index 3c7c74355..ef25f2378 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -299,6 +299,7 @@ def main(): time.sleep(1) elif new_state == State.RUNNING: _process() + time.sleep(5) old_state = new_state diff --git a/requirements.txt b/requirements.txt index 334afb3b9..1ed05807b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ SQLAlchemy==1.1.14 python-telegram-bot==8.1.1 arrow==0.10.0 +cachetools==2.0.1 requests==2.18.4 urllib3==1.22 wrapt==1.10.11 diff --git a/setup.py b/setup.py index 767783f1c..2c4a3f4c9 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ setup(name='freqtrade', 'jsonschema', 'TA-Lib', 'tabulate', + 'cachetools', ], dependency_links=[ "git+https://github.com/ericsomdahl/python-bittrex.git@0.2.0#egg=python-bittrex" From d3b3370f239877cf9aad256d7440db85a8f8f329 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 11 Nov 2017 16:47:19 +0100 Subject: [PATCH 12/29] Add configurable throttle mechanism --- config.json.example | 5 ++++- freqtrade/main.py | 11 +++++------ freqtrade/misc.py | 27 +++++++++++++++++++++++++++ freqtrade/tests/test_misc.py | 20 ++++++++++++++++++++ 4 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 freqtrade/tests/test_misc.py diff --git a/config.json.example b/config.json.example index 685189087..e91b6772f 100644 --- a/config.json.example +++ b/config.json.example @@ -34,5 +34,8 @@ "token": "token", "chat_id": "chat_id" }, - "initial_state": "running" + "initial_state": "running", + "internals": { + "process_throttle_secs": 5 + } } \ No newline at end of file diff --git a/freqtrade/main.py b/freqtrade/main.py index ef25f2378..1a9990abd 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -13,7 +13,7 @@ from jsonschema import validate from freqtrade import __version__, exchange, persistence from freqtrade.analyze import get_buy_signal -from freqtrade.misc import CONF_SCHEMA, State, get_state, update_state, build_arg_parser +from freqtrade.misc import CONF_SCHEMA, State, get_state, update_state, build_arg_parser, throttle from freqtrade.persistence import Trade from freqtrade.rpc import telegram @@ -280,14 +280,14 @@ def main(): # Load and validate configuration with open(args.config) as file: _CONF = json.load(file) + if 'internals' not in _CONF: + _CONF['internals'] = {} logger.info('Validating configuration ...') validate(_CONF, CONF_SCHEMA) # Initialize all modules and start main loop init(_CONF) - old_state = get_state() - logger.info('Initial State: %s', old_state) - telegram.send_msg('*Status:* `{}`'.format(old_state.name.lower())) + old_state = None while True: new_state = get_state() # Log state transition @@ -298,8 +298,7 @@ def main(): if new_state == State.STOPPED: time.sleep(1) elif new_state == State.RUNNING: - _process() - time.sleep(5) + throttle(_process, min_secs=_CONF['internals'].get('process_throttle_secs', 5)) old_state = new_state diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 4787ef2f9..f7f6a8e42 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -1,11 +1,15 @@ import argparse import enum import logging +from typing import Any, Callable +import time from wrapt import synchronized from freqtrade import __version__ +logger = logging.getLogger(__name__) + class State(enum.Enum): RUNNING = 0 @@ -36,6 +40,23 @@ def get_state() -> State: return _STATE +def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: + """ + Throttles the given callable that it + takes at least `min_secs` to finish execution. + :param func: Any callable + :param min_secs: minimum execution time in seconds + :return: Any + """ + start = time.time() + result = func(*args, **kwargs) + end = time.time() + duration = max(min_secs - (end - start), 0.0) + logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) + time.sleep(duration) + return result + + def build_arg_parser() -> argparse.ArgumentParser: """ Builds and returns an ArgumentParser instance """ parser = argparse.ArgumentParser( @@ -104,6 +125,12 @@ CONF_SCHEMA = { 'required': ['enabled', 'token', 'chat_id'] }, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, + 'internals': { + 'type': 'object', + 'properties': { + 'process_throttle_secs': {'type': 'number'} + } + } }, 'definitions': { 'exchange': { diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py new file mode 100644 index 000000000..2dd8bbb27 --- /dev/null +++ b/freqtrade/tests/test_misc.py @@ -0,0 +1,20 @@ +# pragma pylint: disable=missing-docstring +import time + +from freqtrade.misc import throttle + + +def test_throttle(): + + def func(): + return 42 + + start = time.time() + result = throttle(func, 0.1) + end = time.time() + + assert result == 42 + assert end - start > 0.1 + + result = throttle(func, -1) + assert result == 42 From 12ae1e111ea9dea9334adc21d490b563ecc5aaa4 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 11 Nov 2017 17:14:55 +0100 Subject: [PATCH 13/29] use get_candles from python-bittrex --- freqtrade/exchange/bittrex.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index d65c49d42..2f1d000a9 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -1,14 +1,14 @@ import logging from typing import List, Dict -import requests -from bittrex.bittrex import Bittrex as _Bittrex +from bittrex.bittrex import Bittrex as _Bittrex, API_V2_0, API_V1_1 from freqtrade.exchange.interface import Exchange logger = logging.getLogger(__name__) _API: _Bittrex = None +_API_V2: _Bittrex = None _EXCHANGE_CONF: dict = {} @@ -18,17 +18,23 @@ class Bittrex(Exchange): """ # Base URL and API endpoints BASE_URL: str = 'https://www.bittrex.com' - TICKER_METHOD: str = BASE_URL + '/Api/v2.0/pub/market/GetTicks' PAIR_DETAIL_METHOD: str = BASE_URL + '/Market/Index' def __init__(self, config: dict) -> None: - global _API, _EXCHANGE_CONF + global _API, _API_V2, _EXCHANGE_CONF _EXCHANGE_CONF.update(config) _API = _Bittrex( api_key=_EXCHANGE_CONF['key'], api_secret=_EXCHANGE_CONF['secret'], calls_per_second=3, + api_version=API_V1_1, + ) + _API_V2 = _Bittrex( + api_key=_EXCHANGE_CONF['key'], + api_secret=_EXCHANGE_CONF['secret'], + calls_per_second=3, + api_version=API_V2_0, ) @property @@ -90,10 +96,7 @@ class Bittrex(Exchange): else: raise ValueError('Cannot parse tick_interval: {}'.format(tick_interval)) - data = requests.get(self.TICKER_METHOD, params={ - 'marketName': pair.replace('_', '-'), - 'tickInterval': interval, - }).json() + data = _API_V2.get_candles(pair.replace('_', '-'), interval) if not data['success']: raise RuntimeError('{message} params=({pair})'.format( message=data['message'], From bcd3340a80a844388c40d843e6aa242ec9453e19 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 11 Nov 2017 19:20:16 +0100 Subject: [PATCH 14/29] implement get_market_summaries --- freqtrade/exchange/__init__.py | 4 ++++ freqtrade/exchange/bittrex.py | 6 ++++++ freqtrade/exchange/interface.py | 24 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index dc9dfa427..523274079 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -159,6 +159,10 @@ def get_markets() -> List[str]: return _API.get_markets() +def get_market_summaries() -> List[Dict]: + return _API.get_market_summaries() + + def get_name() -> str: return _API.name diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 2f1d000a9..0294f3d00 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -137,3 +137,9 @@ class Bittrex(Exchange): if not data['success']: raise RuntimeError('{message}'.format(message=data['message'])) return [m['MarketName'].replace('-', '_') for m in data['result']] + + def get_market_summaries(self) -> List[Dict]: + data = _API.get_market_summaries() + if not data['success']: + raise RuntimeError('{message}'.format(message=data['message'])) + return data['result'] diff --git a/freqtrade/exchange/interface.py b/freqtrade/exchange/interface.py index f2bd3b0dc..0000461d2 100644 --- a/freqtrade/exchange/interface.py +++ b/freqtrade/exchange/interface.py @@ -131,3 +131,27 @@ class Exchange(ABC): Returns all available markets. :return: List of all available pairs """ + + @abstractmethod + def get_market_summaries(self) -> List[Dict]: + """ + Returns a 24h market summary for all available markets + :return: list, format: [ + { + 'MarketName': str, + 'High': float, + 'Low': float, + 'Volume': float, + 'Last': float, + 'TimeStamp': datetime, + 'BaseVolume': float, + 'Bid': float, + 'Ask': float, + 'OpenBuyOrders': int, + 'OpenSellOrders': int, + 'PrevDay': float, + 'Created': datetime + }, + ... + ] + """ From 517879382bc82545e3755fb30a4f8782e683fd0e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 11 Nov 2017 19:20:53 +0100 Subject: [PATCH 15/29] Add argument for dynamic-whitelist handling If --dynamic-whitelist is passed the whitelist in the config file is ignored. It gets automatically refreshed every 30 minutes and currently selects the 20 topmost BaseVolume markets --- freqtrade/main.py | 26 +++++++++++++++++++++++++- freqtrade/misc.py | 5 +++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 1a9990abd..557684a7b 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -6,9 +6,10 @@ import time import traceback from datetime import datetime from signal import signal, SIGINT, SIGABRT, SIGTERM -from typing import Dict, Optional +from typing import Dict, Optional, List import requests +from cachetools import cached, TTLCache from jsonschema import validate from freqtrade import __version__, exchange, persistence @@ -244,6 +245,25 @@ def init(config: dict, db_url: Optional[str] = None) -> None: signal(sig, cleanup) +@cached(TTLCache(maxsize=1, ttl=1800)) +def gen_pair_whitelist(base_currency: str, topn: int = 20, key: str = 'BaseVolume') -> List[str]: + """ + Updates the whitelist with with a dynamically generated list + :param base_currency: base currency as str + :param topn: maximum number of returned results + :param key: sort key (defaults to 'BaseVolume') + :return: List of pairs + """ + summaries = sorted( + (s for s in exchange.get_market_summaries() if s['MarketName'].startswith(base_currency)), + key=lambda s: s[key], + reverse=True + ) + pairs = [s['MarketName'].replace('-', '_') for s in summaries[:topn]] + logger.debug('Generated pair whitelist: %s ...', pairs) + return pairs + + def cleanup(*args, **kwargs) -> None: """ Cleanup the application state und finish all pending tasks @@ -286,6 +306,8 @@ def main(): validate(_CONF, CONF_SCHEMA) # Initialize all modules and start main loop + if args.dynamic_whitelist: + logger.info('Using dynamically generated whitelist. (--dynamic-whitelist detected)') init(_CONF) old_state = None while True: @@ -298,6 +320,8 @@ def main(): if new_state == State.STOPPED: time.sleep(1) elif new_state == State.RUNNING: + if args.dynamic_whitelist: + _CONF['exchange']['pair_whitelist'] = gen_pair_whitelist(_CONF['stake_currency']) throttle(_process, min_secs=_CONF['internals'].get('process_throttle_secs', 5)) old_state = new_state diff --git a/freqtrade/misc.py b/freqtrade/misc.py index f7f6a8e42..35e67a976 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -83,6 +83,11 @@ def build_arg_parser() -> argparse.ArgumentParser: action='version', version='%(prog)s {}'.format(__version__), ) + parser.add_argument( + '--dynamic-whitelist', + help='dynamically generate and update whitelist based on 24h BaseVolume', + action='store_true', + ) return parser From 1c3c316e4582f16d6480e01248fede0a770ad239 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 11 Nov 2017 21:29:35 +0100 Subject: [PATCH 16/29] reduce calls_per_second --- freqtrade/exchange/bittrex.py | 4 ++-- freqtrade/main.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 0294f3d00..51ddd17e0 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -27,13 +27,13 @@ class Bittrex(Exchange): _API = _Bittrex( api_key=_EXCHANGE_CONF['key'], api_secret=_EXCHANGE_CONF['secret'], - calls_per_second=3, + calls_per_second=2, api_version=API_V1_1, ) _API_V2 = _Bittrex( api_key=_EXCHANGE_CONF['key'], api_secret=_EXCHANGE_CONF['secret'], - calls_per_second=3, + calls_per_second=2, api_version=API_V2_0, ) diff --git a/freqtrade/main.py b/freqtrade/main.py index 557684a7b..932b1532e 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -322,7 +322,7 @@ def main(): elif new_state == State.RUNNING: if args.dynamic_whitelist: _CONF['exchange']['pair_whitelist'] = gen_pair_whitelist(_CONF['stake_currency']) - throttle(_process, min_secs=_CONF['internals'].get('process_throttle_secs', 5)) + throttle(_process, min_secs=_CONF['internals'].get('process_throttle_secs', 10)) old_state = new_state From 15b20b83faed9087089024aafcde267f5e5e3bbd Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 12 Nov 2017 08:30:58 +0200 Subject: [PATCH 17/29] optimize hyperopt objective function --- freqtrade/tests/test_hyperopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index d41f1a7ef..b501f35cc 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -15,7 +15,7 @@ from freqtrade.vendor.qtpylib.indicators import crossed_above logging.disable(logging.DEBUG) # disable debug logs that slow backtesting a lot # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data -TARGET_TRADES = 1200 +TARGET_TRADES = 1300 def buy_strategy_generator(params): @@ -77,8 +77,8 @@ def test_hyperopt(backtest_conf, backdata, mocker): total_profit = results.profit.sum() * 1000 trade_count = len(results.index) - trade_loss = 1 - 0.8 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5) - profit_loss = exp(-total_profit**3 / 10**11) + trade_loss = 1 - 0.4 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2) + profit_loss = max(0, 1 - total_profit / 15000) # max profit 15000 return { 'loss': trade_loss + profit_loss, From 2963a90008723f3c2797915686e0f04663dbf9f0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 12 Nov 2017 08:38:52 +0200 Subject: [PATCH 18/29] add stochastics trigger --- freqtrade/tests/test_hyperopt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index b501f35cc..33fd1e69b 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -50,6 +50,7 @@ def buy_strategy_generator(params): 'ema5_cross_ema10': (crossed_above(dataframe['ema5'], dataframe['ema10'])), 'macd_cross_signal': (crossed_above(dataframe['macd'], dataframe['macdsignal'])), 'sar_reversal': (crossed_above(dataframe['close'], dataframe['sar'])), + 'stochf_cross': (crossed_above(dataframe['fastk'], dataframe['fastd'])), } conditions.append(triggers.get(params['trigger']['type'])) @@ -126,6 +127,7 @@ def test_hyperopt(backtest_conf, backdata, mocker): {'type': 'ema5_cross_ema10'}, {'type': 'macd_cross_signal'}, {'type': 'sar_reversal'}, + {'type': 'stochf_cross'}, ]), } trials = Trials() From 13537e3ce4a477acf4d73276d5a4a6ceae16f48b Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 12 Nov 2017 08:45:32 +0200 Subject: [PATCH 19/29] add short ema guard to hyperopt --- freqtrade/tests/test_hyperopt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index 33fd1e69b..d8d1a712a 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -26,6 +26,8 @@ def buy_strategy_generator(params): # GUARDS AND TRENDS if params['uptrend_long_ema']['enabled']: conditions.append(dataframe['ema50'] > dataframe['ema100']) + if params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) if params['mfi']['enabled']: conditions.append(dataframe['mfi'] < params['mfi']['value']) if params['fastd']['enabled']: @@ -108,6 +110,10 @@ def test_hyperopt(backtest_conf, backdata, mocker): {'enabled': False}, {'enabled': True} ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), 'over_sar': hp.choice('over_sar', [ {'enabled': False}, {'enabled': True} From 660f01b51432b4d844cd45cdcdabfd16abc741c9 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 12 Nov 2017 09:13:54 +0200 Subject: [PATCH 20/29] add hilbert transform leadsine trigger --- freqtrade/analyze.py | 3 +++ freqtrade/tests/test_hyperopt.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index e9ef9c290..3143abc1f 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -49,6 +49,9 @@ def populate_indicators(dataframe: DataFrame) -> DataFrame: dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] dataframe['macdhist'] = macd['macdhist'] + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] return dataframe diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index d8d1a712a..9e33829b9 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -53,6 +53,7 @@ def buy_strategy_generator(params): 'macd_cross_signal': (crossed_above(dataframe['macd'], dataframe['macdsignal'])), 'sar_reversal': (crossed_above(dataframe['close'], dataframe['sar'])), 'stochf_cross': (crossed_above(dataframe['fastk'], dataframe['fastd'])), + 'ht_sine': (crossed_above(dataframe['htleadsine'], dataframe['htsine'])), } conditions.append(triggers.get(params['trigger']['type'])) @@ -134,6 +135,7 @@ def test_hyperopt(backtest_conf, backdata, mocker): {'type': 'macd_cross_signal'}, {'type': 'sar_reversal'}, {'type': 'stochf_cross'}, + {'type': 'ht_sine'}, ]), } trials = Trials() From 8e68c5358e52bcce91966f118d1ae4f4157ea284 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 12 Nov 2017 09:44:31 +0200 Subject: [PATCH 21/29] clean up prints during hyperopt --- freqtrade/tests/test_hyperopt.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index 9e33829b9..d90d90689 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -16,11 +16,10 @@ logging.disable(logging.DEBUG) # disable debug logs that slow backtesting a lot # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data TARGET_TRADES = 1300 - +TOTAL_TRIES = 4 +current_tries = 0 def buy_strategy_generator(params): - print(params) - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: conditions = [] # GUARDS AND TRENDS @@ -76,7 +75,6 @@ def test_hyperopt(backtest_conf, backdata, mocker): results = backtest(backtest_conf, backdata, mocker) result = format_results(results) - print(result) total_profit = results.profit.sum() * 1000 trade_count = len(results.index) @@ -84,6 +82,10 @@ def test_hyperopt(backtest_conf, backdata, mocker): trade_loss = 1 - 0.4 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2) profit_loss = max(0, 1 - total_profit / 15000) # max profit 15000 + global current_tries + current_tries += 1 + print('{}/{}: {}'.format(current_tries, TOTAL_TRIES, result)) + return { 'loss': trade_loss + profit_loss, 'status': STATUS_OK, @@ -139,7 +141,7 @@ def test_hyperopt(backtest_conf, backdata, mocker): ]), } trials = Trials() - best = fmin(fn=optimizer, space=space, algo=tpe.suggest, max_evals=4, trials=trials) + best = fmin(fn=optimizer, space=space, algo=tpe.suggest, max_evals=TOTAL_TRIES, trials=trials) print('\n\n\n\n==================== HYPEROPT BACKTESTING REPORT ==============================') print('Best parameters {}'.format(best)) newlist = sorted(trials.results, key=itemgetter('loss')) From 0f0b10b6cc7f4f354021fbc4d26afab6beb4e908 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 13 Nov 2017 07:28:56 +0200 Subject: [PATCH 22/29] adjust search spaces --- freqtrade/tests/test_hyperopt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index d90d90689..bece2edae 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -95,19 +95,19 @@ def test_hyperopt(backtest_conf, backdata, mocker): space = { 'mfi': hp.choice('mfi', [ {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 5, 15, 1)} + {'enabled': True, 'value': hp.quniform('mfi-value', 5, 25, 1)} ]), 'fastd': hp.choice('fastd', [ {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 5, 40, 1)} + {'enabled': True, 'value': hp.quniform('fastd-value', 10, 50, 1)} ]), 'adx': hp.choice('adx', [ {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 10, 50, 1)} + {'enabled': True, 'value': hp.quniform('adx-value', 15, 50, 1)} ]), 'rsi': hp.choice('rsi', [ {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 30, 1)} + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)} ]), 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ {'enabled': False}, From 81f7172c4a0a2a1b7fb4a5f3654c7cbc1011baeb Mon Sep 17 00:00:00 2001 From: gcarq Date: Mon, 13 Nov 2017 19:54:09 +0100 Subject: [PATCH 23/29] sanitize get_ticker_history (fixes #100) --- freqtrade/analyze.py | 12 +++++------- freqtrade/exchange/__init__.py | 2 +- freqtrade/exchange/bittrex.py | 9 ++++++++- freqtrade/exchange/interface.py | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 3143abc1f..e13d2b39d 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -79,13 +79,12 @@ def analyze_ticker(pair: str) -> DataFrame: add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ - data = get_ticker_history(pair) - dataframe = parse_ticker_dataframe(data) - - if dataframe.empty: - logger.warning('Empty dataframe for pair %s', pair) - return dataframe + ticker_hist = get_ticker_history(pair) + if not ticker_hist: + logger.warning('Empty ticker history for pair %s', pair) + return DataFrame() + dataframe = parse_ticker_dataframe(ticker_hist) dataframe = populate_indicators(dataframe) dataframe = populate_buy_trend(dataframe) return dataframe @@ -98,7 +97,6 @@ def get_buy_signal(pair: str) -> bool: :return: True if pair is good for buying, False otherwise """ dataframe = analyze_ticker(pair) - if dataframe.empty: return False diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 523274079..10de18b14 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -129,7 +129,7 @@ def get_ticker(pair: str) -> dict: @cached(TTLCache(maxsize=100, ttl=30)) -def get_ticker_history(pair: str, tick_interval: Optional[int] = 5) -> List: +def get_ticker_history(pair: str, tick_interval: Optional[int] = 5) -> List[Dict]: return _API.get_ticker_history(pair, tick_interval) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 51ddd17e0..b1d8e68fb 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -88,7 +88,7 @@ class Bittrex(Exchange): 'last': float(data['result']['Last']), } - def get_ticker_history(self, pair: str, tick_interval: int): + def get_ticker_history(self, pair: str, tick_interval: int) -> List[Dict]: if tick_interval == 1: interval = 'oneMin' elif tick_interval == 5: @@ -97,6 +97,13 @@ class Bittrex(Exchange): raise ValueError('Cannot parse tick_interval: {}'.format(tick_interval)) data = _API_V2.get_candles(pair.replace('_', '-'), interval) + # This sanity check is necessary because bittrex returns nonsense sometimes + for prop in ['C', 'V', 'O', 'H', 'L', 'T']: + for tick in data['result']: + if prop not in tick.keys(): + logger.warning('Required property {} not present in response'.format(prop)) + return [] + if not data['success']: raise RuntimeError('{message} params=({pair})'.format( message=data['message'], diff --git a/freqtrade/exchange/interface.py b/freqtrade/exchange/interface.py index 0000461d2..60e6b03cd 100644 --- a/freqtrade/exchange/interface.py +++ b/freqtrade/exchange/interface.py @@ -74,7 +74,7 @@ class Exchange(ABC): """ @abstractmethod - def get_ticker_history(self, pair: str, tick_interval: int) -> List: + def get_ticker_history(self, pair: str, tick_interval: int) -> List[Dict]: """ Gets ticker history for given pair. :param pair: Pair as str, format: BTC_ETC From dd9cb008fbc0a437576811f8d5e5021786f97bcc Mon Sep 17 00:00:00 2001 From: gcarq Date: Mon, 13 Nov 2017 21:34:47 +0100 Subject: [PATCH 24/29] refresh whitelist based on wallet health (fixes #60) Refreshs the whitelist in each iteration based on the wallet health, disabled wallets will be removed from the whitelist automatically. --- freqtrade/exchange/__init__.py | 4 +++ freqtrade/exchange/bittrex.py | 17 +++++++++++- freqtrade/exchange/interface.py | 14 ++++++++++ freqtrade/main.py | 45 +++++++++++++++++++++++++++----- freqtrade/tests/conftest.py | 33 ++++++++++++++++++++++- freqtrade/tests/test_exchange.py | 4 ++- freqtrade/tests/test_main.py | 12 ++++++--- 7 files changed, 115 insertions(+), 14 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 10de18b14..a6e986f7b 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -169,3 +169,7 @@ def get_name() -> str: def get_fee() -> float: return _API.fee + + +def get_wallet_health() -> List[Dict]: + return _API.get_wallet_health() diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index b1d8e68fb..6f965a9b4 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -82,6 +82,10 @@ class Bittrex(Exchange): raise RuntimeError('{message} params=({pair})'.format( message=data['message'], pair=pair)) + if not data['result']['Bid'] or not data['result']['Ask'] or not data['result']['Last']: + raise RuntimeError('{message} params=({pair})'.format( + message=data['message'], + pair=pair)) return { 'bid': float(data['result']['Bid']), 'ask': float(data['result']['Ask']), @@ -101,7 +105,7 @@ class Bittrex(Exchange): for prop in ['C', 'V', 'O', 'H', 'L', 'T']: for tick in data['result']: if prop not in tick.keys(): - logger.warning('Required property {} not present in response'.format(prop)) + logger.warning('Required property %s not present in response', prop) return [] if not data['success']: @@ -150,3 +154,14 @@ class Bittrex(Exchange): if not data['success']: raise RuntimeError('{message}'.format(message=data['message'])) return data['result'] + + def get_wallet_health(self) -> List[Dict]: + data = _API_V2.get_wallet_health() + if not data['success']: + raise RuntimeError('{message}'.format(message=data['message'])) + return [{ + 'Currency': entry['Health']['Currency'], + 'IsActive': entry['Health']['IsActive'], + 'LastChecked': entry['Health']['LastChecked'], + 'Notice': entry['Currency'].get('Notice'), + } for entry in data['result']] diff --git a/freqtrade/exchange/interface.py b/freqtrade/exchange/interface.py index 60e6b03cd..a46b3c054 100644 --- a/freqtrade/exchange/interface.py +++ b/freqtrade/exchange/interface.py @@ -155,3 +155,17 @@ class Exchange(ABC): ... ] """ + + @abstractmethod + def get_wallet_health(self) -> List[Dict]: + """ + Returns a list of all wallet health information + :return: list, format: [ + { + 'Currency': str, + 'IsActive': bool, + 'LastChecked': str, + 'Notice': str + }, + ... + """ diff --git a/freqtrade/main.py b/freqtrade/main.py index 932b1532e..cee797be6 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -23,14 +23,45 @@ logger = logging.getLogger('freqtrade') _CONF = {} -def _process() -> bool: +def refresh_whitelist(whitelist: Optional[List[str]] = None) -> None: + """ + Check wallet health and remove pair from whitelist if necessary + :param whitelist: a new whitelist (optional) + :return: None + """ + whitelist = whitelist or _CONF['exchange']['pair_whitelist'] + + sanitized_whitelist = [] + health = exchange.get_wallet_health() + for status in health: + pair = '{}_{}'.format(_CONF['stake_currency'], status['Currency']) + if pair not in whitelist: + continue + if status['IsActive']: + sanitized_whitelist.append(pair) + else: + logger.info( + 'Ignoring %s from whitelist (reason: %s).', + pair, status.get('Notice') or 'wallet is not active' + ) + if _CONF['exchange']['pair_whitelist'] != sanitized_whitelist: + logger.debug('Using refreshed pair whitelist: %s ...', sanitized_whitelist) + _CONF['exchange']['pair_whitelist'] = sanitized_whitelist + + +def _process(dynamic_whitelist: Optional[bool] = False) -> bool: """ Queries the persistence layer for open trades and handles them, otherwise a new trade is created. + :param: dynamic_whitelist: True is a dynamic whitelist should be generated (optional) :return: True if a trade has been created or closed, False otherwise """ state_changed = False try: + # Refresh whitelist based on wallet maintenance + refresh_whitelist( + gen_pair_whitelist(_CONF['stake_currency']) if dynamic_whitelist else None + ) # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() if len(trades) < _CONF['max_open_trades']: @@ -259,9 +290,7 @@ def gen_pair_whitelist(base_currency: str, topn: int = 20, key: str = 'BaseVolum key=lambda s: s[key], reverse=True ) - pairs = [s['MarketName'].replace('-', '_') for s in summaries[:topn]] - logger.debug('Generated pair whitelist: %s ...', pairs) - return pairs + return [s['MarketName'].replace('-', '_') for s in summaries[:topn]] def cleanup(*args, **kwargs) -> None: @@ -320,9 +349,11 @@ def main(): if new_state == State.STOPPED: time.sleep(1) elif new_state == State.RUNNING: - if args.dynamic_whitelist: - _CONF['exchange']['pair_whitelist'] = gen_pair_whitelist(_CONF['stake_currency']) - throttle(_process, min_secs=_CONF['internals'].get('process_throttle_secs', 10)) + throttle( + _process, + min_secs=_CONF['internals'].get('process_throttle_secs', 10), + dynamic_whitelist=args.dynamic_whitelist, + ) old_state = new_state diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 25d77d688..25273c546 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -37,7 +37,8 @@ def default_conf(): "BTC_ETH", "BTC_TKN", "BTC_TRST", - "BTC_SWT" + "BTC_SWT", + "BTC_BCC" ] }, "telegram": { @@ -90,6 +91,36 @@ def ticker(): }) +@pytest.fixture +def health(): + return MagicMock(return_value=[{ + 'Currency': 'BTC', + 'IsActive': True, + 'LastChecked': '2017-11-13T20:15:00.00', + 'Notice': None + }, { + 'Currency': 'ETH', + 'IsActive': True, + 'LastChecked': '2017-11-13T20:15:00.00', + 'Notice': None + }, { + 'Currency': 'TRST', + 'IsActive': True, + 'LastChecked': '2017-11-13T20:15:00.00', + 'Notice': None + }, { + 'Currency': 'SWT', + 'IsActive': True, + 'LastChecked': '2017-11-13T20:15:00.00', + 'Notice': None + }, { + 'Currency': 'BCC', + 'IsActive': False, + 'LastChecked': '2017-11-13T20:15:00.00', + 'Notice': None + }]) + + @pytest.fixture def limit_buy_order(): return { diff --git a/freqtrade/tests/test_exchange.py b/freqtrade/tests/test_exchange.py index d4d4e2588..309ff4a5e 100644 --- a/freqtrade/tests/test_exchange.py +++ b/freqtrade/tests/test_exchange.py @@ -8,7 +8,9 @@ from freqtrade.exchange import validate_pairs def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() - api_mock.get_markets = MagicMock(return_value=['BTC_ETH', 'BTC_TKN', 'BTC_TRST', 'BTC_SWT']) + api_mock.get_markets = MagicMock(return_value=[ + 'BTC_ETH', 'BTC_TKN', 'BTC_TRST', 'BTC_SWT', 'BTC_BCC', + ]) mocker.patch('freqtrade.exchange._API', api_mock) mocker.patch.dict('freqtrade.exchange._CONF', default_conf) validate_pairs(default_conf['exchange']['pair_whitelist']) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index f114b4dde..6314a919c 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -13,13 +13,14 @@ from freqtrade.misc import get_state, State from freqtrade.persistence import Trade -def test_process_trade_creation(default_conf, ticker, mocker): +def test_process_trade_creation(default_conf, ticker, health, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch.multiple('freqtrade.main.telegram', init=MagicMock(), send_msg=MagicMock()) mocker.patch('freqtrade.main.get_buy_signal', side_effect=lambda _: True) mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=ticker, + get_wallet_health=health, buy=MagicMock(return_value='mocked_limit_buy')) init(default_conf, create_engine('sqlite://')) @@ -41,7 +42,7 @@ def test_process_trade_creation(default_conf, ticker, mocker): assert trade.amount == 0.6864067381401302 -def test_process_exchange_failures(default_conf, ticker, mocker): +def test_process_exchange_failures(default_conf, ticker, health, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch.multiple('freqtrade.main.telegram', init=MagicMock(), send_msg=MagicMock()) mocker.patch('freqtrade.main.get_buy_signal', side_effect=lambda _: True) @@ -49,6 +50,7 @@ def test_process_exchange_failures(default_conf, ticker, mocker): mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=ticker, + get_wallet_health=health, buy=MagicMock(side_effect=requests.exceptions.RequestException)) init(default_conf, create_engine('sqlite://')) result = _process() @@ -56,7 +58,7 @@ def test_process_exchange_failures(default_conf, ticker, mocker): assert sleep_mock.has_calls() -def test_process_runtime_error(default_conf, ticker, mocker): +def test_process_runtime_error(default_conf, ticker, health, mocker): msg_mock = MagicMock() mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch.multiple('freqtrade.main.telegram', init=MagicMock(), send_msg=msg_mock) @@ -64,6 +66,7 @@ def test_process_runtime_error(default_conf, ticker, mocker): mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=ticker, + get_wallet_health=health, buy=MagicMock(side_effect=RuntimeError)) init(default_conf, create_engine('sqlite://')) assert get_state() == State.RUNNING @@ -74,13 +77,14 @@ def test_process_runtime_error(default_conf, ticker, mocker): assert 'RuntimeError' in msg_mock.call_args_list[-1][0][0] -def test_process_trade_handling(default_conf, ticker, limit_buy_order, mocker): +def test_process_trade_handling(default_conf, ticker, limit_buy_order, health, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch.multiple('freqtrade.main.telegram', init=MagicMock(), send_msg=MagicMock()) mocker.patch('freqtrade.main.get_buy_signal', side_effect=lambda _: True) mocker.patch.multiple('freqtrade.main.exchange', validate_pairs=MagicMock(), get_ticker=ticker, + get_wallet_health=health, buy=MagicMock(return_value='mocked_limit_buy'), get_order=MagicMock(return_value=limit_buy_order)) init(default_conf, create_engine('sqlite://')) From e8101a6da59c0b6ed6125d2504b9d45d72016da7 Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 14 Nov 2017 17:48:19 +0100 Subject: [PATCH 25/29] default BaseVolume to 0.0 if null --- freqtrade/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index cee797be6..d44478d7d 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -287,7 +287,7 @@ def gen_pair_whitelist(base_currency: str, topn: int = 20, key: str = 'BaseVolum """ summaries = sorted( (s for s in exchange.get_market_summaries() if s['MarketName'].startswith(base_currency)), - key=lambda s: s[key], + key=lambda s: s.get(key) or 0.0, reverse=True ) return [s['MarketName'].replace('-', '_') for s in summaries[:topn]] From b83309b55d32849500e99e8c2d1d27134d5df605 Mon Sep 17 00:00:00 2001 From: gcarq Date: Wed, 15 Nov 2017 23:16:39 +0100 Subject: [PATCH 26/29] reduce calls_per_second to 1 --- freqtrade/exchange/bittrex.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 6f965a9b4..2a9738cda 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -27,13 +27,13 @@ class Bittrex(Exchange): _API = _Bittrex( api_key=_EXCHANGE_CONF['key'], api_secret=_EXCHANGE_CONF['secret'], - calls_per_second=2, + calls_per_second=1, api_version=API_V1_1, ) _API_V2 = _Bittrex( api_key=_EXCHANGE_CONF['key'], api_secret=_EXCHANGE_CONF['secret'], - calls_per_second=2, + calls_per_second=1, api_version=API_V2_0, ) From b5f58724a0e7e660df49736ccd2e69b3d8cf7be8 Mon Sep 17 00:00:00 2001 From: gcarq Date: Wed, 15 Nov 2017 23:16:54 +0100 Subject: [PATCH 27/29] get_ticker_history: check if result is set (fixes #103) --- freqtrade/exchange/bittrex.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 2a9738cda..718500753 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -101,7 +101,11 @@ class Bittrex(Exchange): raise ValueError('Cannot parse tick_interval: {}'.format(tick_interval)) data = _API_V2.get_candles(pair.replace('_', '-'), interval) - # This sanity check is necessary because bittrex returns nonsense sometimes + + # These sanity check are necessary because bittrex cannot keep their API stable. + if not data.get('result'): + return [] + for prop in ['C', 'V', 'O', 'H', 'L', 'T']: for tick in data['result']: if prop not in tick.keys(): From 4e05691cabc93637282e25074a2f89366e9c58f8 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 16 Nov 2017 00:01:47 +0100 Subject: [PATCH 28/29] check if balance list is empty (fixes #105) --- freqtrade/rpc/telegram.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a3d700b7e..0e0487df6 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -273,18 +273,21 @@ def _balance(bot: Bot, update: Update) -> None: Handler for /balance Returns current account balance per crypto """ - output = "" - balances = exchange.get_balances() + output = '' + balances = [ + c for c in exchange.get_balances() + if c['Balance'] or c['Available'] or c['Pending'] + ] + if not balances: + output = '`All balances are zero.`' + for currency in balances: - if not currency['Balance'] and not currency['Available'] and not currency['Pending']: - continue output += """*Currency*: {Currency} *Available*: {Available} *Balance*: {Balance} *Pending*: {Pending} """.format(**currency) - send_msg(output) From 2e953a937df3b492335710d9c961f011c57c2c70 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 16 Nov 2017 00:40:36 +0100 Subject: [PATCH 29/29] version bump --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index b3d4bd950..75254ebcc 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,3 +1,3 @@ -__version__ = '0.14.1' +__version__ = '0.14.2' from . import main