stable/freqtrade/main.py

568 lines
20 KiB
Python
Raw Normal View History

2017-11-07 16:54:44 +00:00
#!/usr/bin/env python3
2017-10-06 10:22:04 +00:00
import copy
import json
2017-05-12 17:11:56 +00:00
import logging
2017-11-17 16:18:31 +00:00
import sys
2017-05-12 17:11:56 +00:00
import time
import traceback
from datetime import datetime
2018-01-26 16:49:14 +00:00
from typing import Dict, List, Optional, Any
2017-09-01 19:11:46 +00:00
2018-01-10 07:51:36 +00:00
import arrow
import requests
from cachetools import cached, TTLCache
2018-01-10 07:51:36 +00:00
from freqtrade import (DependencyException, OperationalException, __version__,
exchange, persistence, rpc)
from freqtrade.analyze import get_signal
2017-12-25 07:51:41 +00:00
from freqtrade.fiat_convert import CryptoToFiatConverter
2018-01-10 07:51:36 +00:00
from freqtrade.misc import (State, get_state, load_config, parse_args,
throttle, update_state)
from freqtrade.persistence import Trade
2018-01-15 08:35:11 +00:00
from freqtrade.strategy.strategy import Strategy
2017-05-12 17:11:56 +00:00
logger = logging.getLogger('freqtrade')
2017-05-17 23:46:08 +00:00
2018-01-26 16:49:14 +00:00
_CONF: Dict[str, Any] = {}
2017-09-08 21:10:22 +00:00
def refresh_whitelist(whitelist: List[str]) -> List[str]:
"""
Check wallet health and remove pair from whitelist if necessary
2018-01-02 11:04:47 +00:00
:param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to trade
:return: the list of pairs the user wants to trade without the one unavailable or black_listed
"""
2018-01-02 11:04:47 +00:00
sanitized_whitelist = whitelist
health = exchange.get_wallet_health()
2018-01-02 11:04:47 +00:00
known_pairs = set()
for status in health:
pair = '{}_{}'.format(_CONF['stake_currency'], status['Currency'])
# pair is not int the generated dynamic market, or in the blacklist ... ignore it
if pair not in whitelist or pair in _CONF['exchange'].get('pair_blacklist', []):
continue
# else the pair is valid
known_pairs.add(pair)
# Market is not active
2018-01-02 11:04:47 +00:00
if not status['IsActive']:
sanitized_whitelist.remove(pair)
logger.info(
'Ignoring %s from whitelist (reason: %s).',
pair, status.get('Notice') or 'wallet is not active'
)
2018-01-02 11:04:47 +00:00
# We need to remove pairs that are unknown
final_list = [x for x in sanitized_whitelist if x in known_pairs]
return final_list
2018-01-30 08:38:24 +00:00
def process_maybe_execute_buy(interval: int) -> bool:
"""
Tries to execute a buy trade in a safe way
:return: True if executed
"""
try:
# Create entity and execute trade
if create_trade(float(_CONF['stake_amount']), interval):
return True
2018-01-26 17:01:54 +00:00
2018-02-24 16:33:08 +00:00
logger.info('Found no buy signals for whitelisted currencies. Trying again..')
2018-01-26 17:01:54 +00:00
return False
except DependencyException as exception:
logger.warning('Unable to create trade: %s', exception)
return False
2018-01-30 08:38:24 +00:00
def process_maybe_execute_sell(trade: Trade, interval: int) -> bool:
"""
Tries to execute a sell trade
:return: True if executed
"""
# Get order details for actual price per unit
if trade.open_order_id:
# Update trade with order values
2018-02-24 16:33:08 +00:00
logger.info('Found open order for %s', trade)
trade.update(exchange.get_order(trade.open_order_id))
if trade.is_open and trade.open_order_id is None:
# Check if we can sell our current pair
return handle_trade(trade, interval)
return False
def _process(interval: int, nb_assets: Optional[int] = 0) -> bool:
"""
Queries the persistence layer for open trades and handles them,
otherwise a new trade is created.
:param: nb_assets: the maximum number of pairs to be traded at the same time
:return: True if one or more trades has been created or closed, False otherwise
"""
2017-11-07 21:26:08 +00:00
state_changed = False
try:
# Refresh whitelist based on wallet maintenance
sanitized_list = refresh_whitelist(
2017-12-16 02:39:47 +00:00
gen_pair_whitelist(
_CONF['stake_currency']
) if nb_assets else _CONF['exchange']['pair_whitelist']
)
# Keep only the subsets of pairs wanted (up to nb_assets)
final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list
_CONF['exchange']['pair_whitelist'] = final_list
# Query trades from persistence layer
trades = Trade.query.filter(Trade.is_open.is_(True)).all()
2018-01-30 08:38:24 +00:00
# First process current opened trades
for trade in trades:
state_changed |= process_maybe_execute_sell(trade, interval)
2018-01-30 08:38:24 +00:00
# Then looking for buy opportunities
if len(trades) < _CONF['max_open_trades']:
state_changed = process_maybe_execute_buy(interval)
if 'unfilledtimeout' in _CONF:
# Check and handle any timed out open orders
check_handle_timedout(_CONF['unfilledtimeout'])
Trade.session.flush()
except (requests.exceptions.RequestException, json.JSONDecodeError) as error:
logger.warning(
2018-02-24 16:33:08 +00:00
'%s in _process(), retrying in 30 seconds...',
error
)
time.sleep(30)
except OperationalException:
2018-02-24 16:33:08 +00:00
rpc.send_msg('*Status:* OperationalException:\n```\n{traceback}```{hint}'.format(
traceback=traceback.format_exc(),
2017-11-06 17:15:33 +00:00
hint='Issue `/start` if you think it is safe to restart.'
))
2018-02-24 16:33:08 +00:00
logger.exception('OperationalException. Stopping trader ...')
update_state(State.STOPPED)
2017-11-07 21:26:08 +00:00
return state_changed
2017-05-14 12:14:16 +00:00
2017-05-12 17:11:56 +00:00
# FIX: 20180110, why is cancel.order unconditionally here, whereas
# it is conditionally called in the
# handle_timedout_limit_sell()?
def handle_timedout_limit_buy(trade: Trade, order: Dict) -> bool:
"""Buy timeout - cancel order
:return: True if order was fully cancelled
"""
exchange.cancel_order(trade.open_order_id)
if order['remaining'] == order['amount']:
# if trade is not partially completed, just delete the trade
Trade.session.delete(trade)
# FIX? do we really need to flush, caller of
# check_handle_timedout will flush afterwards
Trade.session.flush()
logger.info('Buy order timeout for %s.', trade)
rpc.send_msg('*Timeout:* Unfilled buy order for {} cancelled'.format(
trade.pair.replace('_', '/')))
return True
2018-01-26 17:01:54 +00:00
# if trade is partially complete, edit the stake details for the trade
# and close the order
trade.amount = order['amount'] - order['remaining']
trade.stake_amount = trade.amount * trade.open_rate
trade.open_order_id = None
logger.info('Partial buy order timeout for %s.', trade)
rpc.send_msg('*Timeout:* Remaining buy order for {} cancelled'.format(
trade.pair.replace('_', '/')))
return False
# FIX: 20180110, should cancel_order() be cond. or unconditionally called?
def handle_timedout_limit_sell(trade: Trade, order: Dict) -> bool:
"""
Sell timeout - cancel order and update trade
:return: True if order was fully cancelled
"""
if order['remaining'] == order['amount']:
# if trade is not partially completed, just cancel the trade
exchange.cancel_order(trade.open_order_id)
trade.close_rate = None
trade.close_profit = None
trade.close_date = None
trade.is_open = True
trade.open_order_id = None
rpc.send_msg('*Timeout:* Unfilled sell order for {} cancelled'.format(
trade.pair.replace('_', '/')))
logger.info('Sell order timeout for %s.', trade)
return True
2018-01-26 17:01:54 +00:00
# TODO: figure out how to handle partially complete sell orders
return False
def check_handle_timedout(timeoutvalue: int) -> None:
2018-01-03 03:30:10 +00:00
"""
Check if any orders are timed out and cancel if neccessary
:param timeoutvalue: Number of minutes until order is considered timed out
:return: None
2018-01-03 03:30:10 +00:00
"""
timeoutthreashold = arrow.utcnow().shift(minutes=-timeoutvalue).datetime
for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all():
try:
order = exchange.get_order(trade.open_order_id)
except requests.exceptions.RequestException:
logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc())
continue
ordertime = arrow.get(order['opened'])
# Check if trade is still actually open
if int(order['remaining']) == 0:
continue
if order['type'] == "LIMIT_BUY" and ordertime < timeoutthreashold:
handle_timedout_limit_buy(trade, order)
elif order['type'] == "LIMIT_SELL" and ordertime < timeoutthreashold:
handle_timedout_limit_sell(trade, order)
2018-01-03 03:30:10 +00:00
def execute_sell(trade: Trade, limit: float) -> None:
"""
Executes a limit sell for the given trade and limit
:param trade: Trade instance
:param limit: limit rate for the sell order
:return: None
"""
# Execute sell and update trade record
order_id = exchange.sell(str(trade.pair), limit, trade.amount)
trade.open_order_id = order_id
2017-12-17 21:07:56 +00:00
fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2)
2017-12-25 07:51:41 +00:00
profit_trade = trade.calc_profit(rate=limit)
current_rate = exchange.get_ticker(trade.pair, False)['bid']
profit = trade.calc_profit_percent(current_rate)
message = """*{exchange}:* Selling
*Current Pair:* [{pair}]({pair_url})
*Limit:* `{limit}`
*Amount:* `{amount}`
*Open Rate:* `{open_rate:.8f}`
*Current Rate:* `{current_rate:.8f}`
*Profit:* `{profit:.2f}%`
""".format(
2018-01-09 10:59:06 +00:00
exchange=trade.exchange,
pair=trade.pair,
2018-01-09 10:59:06 +00:00
pair_url=exchange.get_pair_detail_url(trade.pair),
limit=limit,
open_rate=trade.open_rate,
current_rate=current_rate,
amount=round(trade.amount, 8),
profit=round(profit * 100, 2),
2018-01-09 10:59:06 +00:00
)
# For regular case, when the configuration exists
if 'stake_currency' in _CONF and 'fiat_display_currency' in _CONF:
fiat_converter = CryptoToFiatConverter()
profit_fiat = fiat_converter.convert_amount(
profit_trade,
_CONF['stake_currency'],
_CONF['fiat_display_currency']
)
message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \
'` / {profit_fiat:.3f} {fiat})`'.format(
2018-01-09 10:59:06 +00:00
gain="profit" if fmt_exp_profit > 0 else "loss",
profit_percent=fmt_exp_profit,
profit_coin=profit_trade,
coin=_CONF['stake_currency'],
profit_fiat=profit_fiat,
fiat=_CONF['fiat_display_currency'],
)
# Because telegram._forcesell does not have the configuration
# Ignore the FIAT value and does not show the stake_currency as well
else:
message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f})`'.format(
gain="profit" if fmt_exp_profit > 0 else "loss",
profit_percent=fmt_exp_profit,
profit_coin=profit_trade
)
# Send the message
rpc.send_msg(message)
Trade.session.flush()
2017-09-07 14:33:04 +00:00
2017-05-12 17:11:56 +00:00
2017-11-16 05:49:06 +00:00
def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) -> bool:
"""
2017-11-16 05:49:06 +00:00
Based an earlier trade and current price and ROI configuration, decides whether bot should sell
:return True if bot should sell at current rate
"""
2018-01-15 08:35:11 +00:00
strategy = Strategy()
2017-12-17 21:07:56 +00:00
current_profit = trade.calc_profit_percent(current_rate)
if strategy.stoploss is not None and current_profit < strategy.stoploss:
logger.debug('Stop loss hit.')
return True
# Check if time matches and current rate is above threshold
2018-02-06 17:44:33 +00:00
time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60
for duration, threshold in strategy.minimal_roi.items():
2018-02-11 13:03:23 +00:00
if time_diff <= duration:
2018-02-06 19:16:44 +00:00
return False
2018-02-11 13:03:23 +00:00
if current_profit > threshold:
2018-02-11 12:37:12 +00:00
return True
return False
def should_sell(trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool:
"""
This function evaluate if on the condition required to trigger a sell has been reached
if the threshold is reached and updates the trade record.
:return: True if trade should be sold, False otherwise
"""
# Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee)
if min_roi_reached(trade, rate, date):
logger.debug('Executing sell due to ROI ...')
return True
# Experimental: Check if the trade is profitable before selling it (avoid selling at loss)
if _CONF.get('experimental', {}).get('sell_profit_only', False):
logger.debug('Checking if trade is profitable ...')
if trade.calc_profit(rate=rate) <= 0:
return False
if sell and not buy and _CONF.get('experimental', {}).get('use_sell_signal', False):
logger.debug('Executing sell due to sell signal ...')
return True
return False
def handle_trade(trade: Trade, interval: int) -> bool:
2017-05-12 17:11:56 +00:00
"""
2017-05-14 12:14:16 +00:00
Sells the current pair if the threshold is reached and updates the trade record.
2017-11-07 21:26:08 +00:00
:return: True if trade has been sold, False otherwise
2017-05-12 17:11:56 +00:00
"""
2017-11-07 21:26:08 +00:00
if not trade.is_open:
raise ValueError('attempt to handle closed trade: {}'.format(trade))
2017-09-25 12:17:29 +00:00
2017-11-07 21:26:08 +00:00
logger.debug('Handling %s ...', trade)
current_rate = exchange.get_ticker(trade.pair)['bid']
(buy, sell) = (False, False)
if _CONF.get('experimental', {}).get('use_sell_signal'):
(buy, sell) = get_signal(trade.pair, interval)
if should_sell(trade, current_rate, datetime.utcnow(), buy, sell):
execute_sell(trade, current_rate)
return True
return False
2017-05-12 17:11:56 +00:00
2017-10-06 10:22:04 +00:00
2017-09-20 14:34:47 +00:00
def get_target_bid(ticker: Dict[str, float]) -> float:
2017-09-17 20:21:46 +00:00
""" Calculates bid target between current ask price and last price """
if ticker['ask'] < ticker['last']:
return ticker['ask']
balance = _CONF['bid_strategy']['ask_last_balance']
return ticker['ask'] + balance * (ticker['last'] - ticker['ask'])
2017-05-12 17:11:56 +00:00
def create_trade(stake_amount: float, interval: int) -> bool:
2017-05-12 17:11:56 +00:00
"""
2017-09-01 18:46:01 +00:00
Checks the implemented trading indicator(s) for a randomly picked pair,
if one pair triggers the buy_signal a new trade record gets created
2017-05-12 17:11:56 +00:00
:param stake_amount: amount of btc to spend
:return: True if a trade object has been created and persisted, False otherwise
2017-05-12 17:11:56 +00:00
"""
logger.info(
'Checking buy signals to create a new trade with stake_amount: %f ...',
stake_amount
)
2017-10-06 10:22:04 +00:00
whitelist = copy.deepcopy(_CONF['exchange']['pair_whitelist'])
2017-09-11 11:59:11 +00:00
# Check if stake_amount is fulfilled
2017-09-08 21:10:22 +00:00
if exchange.get_balance(_CONF['stake_currency']) < stake_amount:
raise DependencyException(
2017-11-03 20:37:20 +00:00
'stake amount is not fulfilled (currency={})'.format(_CONF['stake_currency'])
2017-09-08 21:10:22 +00:00
)
# Remove currently opened and latest pairs from whitelist
2017-09-30 15:02:07 +00:00
for trade in Trade.query.filter(Trade.is_open.is_(True)).all():
2017-05-21 14:52:36 +00:00
if trade.pair in whitelist:
whitelist.remove(trade.pair)
logger.debug('Ignoring %s in pair whitelist', trade.pair)
if not whitelist:
raise DependencyException('No pair in whitelist')
2017-05-24 19:52:41 +00:00
# Pick pair based on StochRSI buy signals
for _pair in whitelist:
(buy, sell) = get_signal(_pair, interval)
if buy and not sell:
2017-09-08 21:10:22 +00:00
pair = _pair
2017-05-24 19:52:41 +00:00
break
else:
return False
2017-05-24 19:52:41 +00:00
2017-11-22 20:01:44 +00:00
# Calculate amount
buy_limit = get_target_bid(exchange.get_ticker(pair))
2017-11-22 20:01:44 +00:00
amount = stake_amount / buy_limit
2017-05-14 12:14:16 +00:00
order_id = exchange.buy(pair, buy_limit, amount)
fiat_converter = CryptoToFiatConverter()
stake_amount_fiat = fiat_converter.convert_amount(
stake_amount,
_CONF['stake_currency'],
_CONF['fiat_display_currency']
)
2017-05-14 12:14:16 +00:00
# Create trade entity and return
rpc.send_msg('*{}:* Buying [{}]({}) with limit `{:.8f} ({:.6f} {}, {:.3f} {})` '.format(
exchange.get_name().upper(),
2017-06-05 19:17:10 +00:00
pair.replace('_', '/'),
exchange.get_pair_detail_url(pair),
buy_limit, stake_amount, _CONF['stake_currency'],
stake_amount_fiat, _CONF['fiat_display_currency']
2017-11-18 20:43:21 +00:00
))
# Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL
trade = Trade(
pair=pair,
stake_amount=stake_amount,
amount=amount,
2017-12-17 21:07:56 +00:00
fee=exchange.get_fee(),
open_rate=buy_limit,
open_date=datetime.utcnow(),
exchange=exchange.get_name().upper(),
open_order_id=order_id
)
Trade.session.add(trade)
Trade.session.flush()
return True
2017-05-12 17:11:56 +00:00
2017-09-08 21:10:22 +00:00
def init(config: dict, db_url: Optional[str] = None) -> None:
"""
Initializes all modules and updates the config
:param config: config as dict
2017-09-08 19:39:31 +00:00
:param db_url: database connector string for sqlalchemy (Optional)
:return: None
"""
# Initialize all modules
rpc.init(config)
2017-09-08 19:39:31 +00:00
persistence.init(config, db_url)
exchange.init(config)
2018-01-15 08:35:11 +00:00
strategy = Strategy()
strategy.init(config)
# Set initial application state
initial_state = config.get('initial_state')
if initial_state:
update_state(State[initial_state.upper()])
else:
update_state(State.STOPPED)
@cached(TTLCache(maxsize=1, ttl=1800))
def gen_pair_whitelist(base_currency: str, key: str = 'BaseVolume') -> List[str]:
"""
Updates the whitelist with with a dynamically generated list
:param base_currency: base currency as str
:param key: sort key (defaults to 'BaseVolume')
:return: List of pairs
"""
summaries = sorted(
(s for s in exchange.get_market_summaries() if s['MarketName'].startswith(base_currency)),
2017-11-14 16:48:19 +00:00
key=lambda s: s.get(key) or 0.0,
reverse=True
)
return [s['MarketName'].replace('-', '_') for s in summaries]
def cleanup() -> None:
"""
Cleanup the application state und finish all pending tasks
:return: None
"""
rpc.send_msg('*Status:* `Stopping trader...`')
logger.info('Stopping trader and cleaning up modules...')
update_state(State.STOPPED)
persistence.cleanup()
rpc.cleanup()
exit(0)
def main(sysargv=sys.argv[1:]) -> int:
"""
2017-11-08 20:17:51 +00:00
Loads and validates the config and handles the main loop
:return: None
"""
2017-11-08 20:17:51 +00:00
global _CONF
args = parse_args(sysargv,
'Simple High Frequency Trading Bot for crypto currencies')
# A subcommand has been issued
if hasattr(args, 'func'):
args.func(args)
return 0
2017-11-08 20:17:51 +00:00
# Initialize logger
logging.basicConfig(
level=args.loglevel,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
)
logger.info(
'Starting freqtrade %s (loglevel=%s)',
__version__,
logging.getLevelName(args.loglevel)
)
# Load and validate configuration
2017-11-17 16:54:31 +00:00
_CONF = load_config(args.config)
2017-11-08 20:17:51 +00:00
2018-01-15 08:35:11 +00:00
# Add the strategy file to use
_CONF.update({'strategy': args.strategy})
# Initialize all modules and start main loop
if args.dynamic_whitelist:
logger.info('Using dynamically generated whitelist. (--dynamic-whitelist detected)')
# If the user ask for Dry run with a local DB instead of memory
if args.dry_run_db:
if _CONF.get('dry_run', False):
_CONF.update({'dry_run_db': True})
2017-12-16 02:39:47 +00:00
logger.info(
'Dry_run will use the DB file: "tradesv3.dry_run.sqlite". (--dry_run_db detected)'
)
else:
logger.info('Dry run is disabled. (--dry_run_db ignored)')
try:
init(_CONF)
old_state = None
2018-01-15 08:35:11 +00:00
while True:
new_state = get_state()
# Log state transition
if new_state != old_state:
rpc.send_msg('*Status:* `{}`'.format(new_state.name.lower()))
logger.info('Changing state to: %s', new_state.name)
if new_state == State.STOPPED:
time.sleep(1)
elif new_state == State.RUNNING:
throttle(
_process,
min_secs=_CONF['internals'].get('process_throttle_secs', 10),
nb_assets=args.dynamic_whitelist,
interval=int(_CONF.get('ticker_interval', 5))
)
old_state = new_state
except KeyboardInterrupt:
2018-02-24 16:33:08 +00:00
logger.info('SIGINT received, aborting ...')
except BaseException:
2018-02-24 16:33:08 +00:00
logger.exception('Fatal exception!')
finally:
cleanup()
return 0
2017-09-28 21:47:51 +00:00
if __name__ == '__main__':
main(sys.argv[1:])