From 3473fd3c904503975f7343627ce693af4d2b8813 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 15:51:00 +0200 Subject: [PATCH 01/28] major refactoring to allow proper testing This commit includes: * Reducing complexity of modules * Remove unneeded wrapper classes * Implement init() for each module which initializes everything based on the config * Implement some basic tests --- exchange.py | 354 ++++++++++++++------------- main.py | 263 +++++++++++--------- utils.py => misc.py | 27 +-- persistence.py | 49 ++-- rpc/__init__.py | 1 + rpc/telegram.py | 503 ++++++++++++++++++++------------------- test/__init__.py | 0 test/test_main.py | 112 +++++++++ test/test_persistence.py | 28 +++ 9 files changed, 747 insertions(+), 590 deletions(-) rename utils.py => misc.py (75%) create mode 100644 test/__init__.py create mode 100644 test/test_main.py create mode 100644 test/test_persistence.py diff --git a/exchange.py b/exchange.py index 4385073a1..f81a9fd7e 100644 --- a/exchange.py +++ b/exchange.py @@ -4,11 +4,13 @@ from typing import List from bittrex.bittrex import Bittrex from poloniex import Poloniex -from wrapt import synchronized logger = logging.getLogger(__name__) -_exchange_api = None + +cur_exchange = None +_api = None +_conf = {} class Exchange(enum.Enum): @@ -16,192 +18,184 @@ class Exchange(enum.Enum): BITTREX = 1 -class ApiWrapper(object): +def init(config: dict) -> None: """ - Wrapper for exchanges. - Currently implemented: - * Bittrex - * Poloniex (partly) + Initializes this module with the given config, + it does basic validation whether the specified + exchange and pairs are valid. + :param config: config to use + :return: None """ - def __init__(self, config: dict): - """ - Initializes the ApiWrapper with the given config, - it does basic validation whether the specified - exchange and pairs are valid. - :param config: dict - """ - self.dry_run = config['dry_run'] - if self.dry_run: - logger.info('Instance is running with dry_run enabled') + global _api, cur_exchange - use_poloniex = config.get('poloniex', {}).get('enabled', False) - use_bittrex = config.get('bittrex', {}).get('enabled', False) + _conf.update(config) - if use_poloniex: - self.exchange = Exchange.POLONIEX - self.api = Poloniex(key=config['poloniex']['key'], secret=config['poloniex']['secret']) - elif use_bittrex: - self.exchange = Exchange.BITTREX - self.api = Bittrex(api_key=config['bittrex']['key'], api_secret=config['bittrex']['secret']) - else: - self.api = None - raise RuntimeError('No exchange specified. Aborting!') + if config['dry_run']: + logger.info('Instance is running with dry_run enabled') - # Check if all pairs are available - markets = self.get_markets() - for pair in config[self.exchange.name.lower()]['pair_whitelist']: - if pair not in markets: - raise RuntimeError('Pair {} is not available at Poloniex'.format(pair)) + use_poloniex = config.get('poloniex', {}).get('enabled', False) + use_bittrex = config.get('bittrex', {}).get('enabled', False) - def buy(self, pair: str, rate: float, amount: float) -> str: - """ - Places a limit buy order. - :param pair: Pair as str, format: BTC_ETH - :param rate: Rate limit for order - :param amount: The amount to purchase - :return: order_id of the placed buy order - """ - if self.dry_run: - pass - elif self.exchange == Exchange.POLONIEX: - self.api.buy(pair, rate, amount) - # TODO: return order id - elif self.exchange == Exchange.BITTREX: - data = self.api.buy_limit(pair.replace('_', '-'), amount, rate) - if not data['success']: - raise RuntimeError('BITTREX: {}'.format(data['message'])) - return data['result']['uuid'] + if use_poloniex: + cur_exchange = Exchange.POLONIEX + _api = Poloniex(key=config['poloniex']['key'], secret=config['poloniex']['secret']) + elif use_bittrex: + cur_exchange = Exchange.BITTREX + _api = Bittrex(api_key=config['bittrex']['key'], api_secret=config['bittrex']['secret']) + else: + raise RuntimeError('No exchange specified. Aborting!') - def sell(self, pair: str, rate: float, amount: float) -> str: - """ - Places a limit sell order. - :param pair: Pair as str, format: BTC_ETH - :param rate: Rate limit for order - :param amount: The amount to sell - :return: None - """ - if self.dry_run: - pass - elif self.exchange == Exchange.POLONIEX: - self.api.sell(pair, rate, amount) - # TODO: return order id - elif self.exchange == Exchange.BITTREX: - data = self.api.sell_limit(pair.replace('_', '-'), amount, rate) - if not data['success']: - raise RuntimeError('BITTREX: {}'.format(data['message'])) - return data['result']['uuid'] - - def get_balance(self, currency: str) -> float: - """ - Get account balance. - :param currency: currency as str, format: BTC - :return: float - """ - if self.dry_run: - return 999.9 - elif self.exchange == Exchange.POLONIEX: - data = self.api.returnBalances() - return float(data[currency]) - elif self.exchange == Exchange.BITTREX: - data = self.api.get_balance(currency) - if not data['success']: - raise RuntimeError('BITTREX: {}'.format(data['message'])) - return float(data['result']['Balance'] or 0.0) - - def get_ticker(self, pair: str) -> dict: - """ - Get Ticker for given pair. - :param pair: Pair as str, format: BTC_ETC - :return: dict - """ - if self.exchange == Exchange.POLONIEX: - data = self.api.returnTicker() - return { - 'bid': float(data[pair]['highestBid']), - 'ask': float(data[pair]['lowestAsk']), - 'last': float(data[pair]['last']) - } - elif self.exchange == Exchange.BITTREX: - data = self.api.get_ticker(pair.replace('_', '-')) - if not data['success']: - raise RuntimeError('BITTREX: {}'.format(data['message'])) - return { - 'bid': float(data['result']['Bid']), - 'ask': float(data['result']['Ask']), - 'last': float(data['result']['Last']), - } - - def cancel_order(self, order_id: str) -> None: - """ - Cancel order for given order_id - :param order_id: id as str - :return: None - """ - if self.dry_run: - pass - elif self.exchange == Exchange.POLONIEX: - raise NotImplemented('Not implemented') - elif self.exchange == Exchange.BITTREX: - data = self.api.cancel(order_id) - if not data['success']: - raise RuntimeError('BITTREX: {}'.format(data['message'])) - - def get_open_orders(self, pair: str) -> List[dict]: - """ - Get all open orders for given pair. - :param pair: Pair as str, format: BTC_ETC - :return: list of dicts - """ - if self.dry_run: - return [] - elif self.exchange == Exchange.POLONIEX: - raise NotImplemented('Not implemented') - elif self.exchange == Exchange.BITTREX: - data = self.api.get_open_orders(pair.replace('_', '-')) - if not data['success']: - raise RuntimeError('BITTREX: {}'.format(data['message'])) - return [{ - 'id': entry['OrderUuid'], - 'type': entry['OrderType'], - 'opened': entry['Opened'], - 'rate': entry['PricePerUnit'], - 'amount': entry['Quantity'], - 'remaining': entry['QuantityRemaining'], - } for entry in data['result']] - - def get_pair_detail_url(self, pair: str) -> str: - """ - Returns the market detail url for the given pair - :param pair: pair as str, format: BTC_ANT - :return: url as str - """ - if self.exchange == Exchange.POLONIEX: - raise NotImplemented('Not implemented') - elif self.exchange == Exchange.BITTREX: - return 'https://bittrex.com/Market/Index?MarketName={}'.format(pair.replace('_', '-')) - - def get_markets(self) -> List[str]: - """ - Returns all available markets - :return: list of all available pairs - """ - if self.exchange == Exchange.POLONIEX: - # TODO: implement - raise NotImplemented('Not implemented') - elif self.exchange == Exchange. BITTREX: - data = self.api.get_markets() - if not data['success']: - raise RuntimeError('BITTREX: {}'.format(data['message'])) - return [m['MarketName'].replace('-', '_') for m in data['result']] + # Check if all pairs are available + markets = get_markets() + for pair in config[cur_exchange.name.lower()]['pair_whitelist']: + if pair not in markets: + raise RuntimeError('Pair {} is not available at Poloniex'.format(pair)) -@synchronized -def get_exchange_api(conf: dict) -> ApiWrapper: +def buy(pair: str, rate: float, amount: float) -> str: """ - Returns the current exchange api or instantiates a new one - :return: exchange.ApiWrapper + Places a limit buy order. + :param pair: Pair as str, format: BTC_ETH + :param rate: Rate limit for order + :param amount: The amount to purchase + :return: order_id of the placed buy order """ - global _exchange_api - if not _exchange_api: - _exchange_api = ApiWrapper(conf) - return _exchange_api + if _conf['dry_run']: + return 'dry_run' + elif cur_exchange == Exchange.POLONIEX: + _api.buy(pair, rate, amount) + # TODO: return order id + elif cur_exchange == Exchange.BITTREX: + data = _api.buy_limit(pair.replace('_', '-'), amount, rate) + if not data['success']: + raise RuntimeError('BITTREX: {}'.format(data['message'])) + return data['result']['uuid'] + + +def sell(pair: str, rate: float, amount: float) -> str: + """ + Places a limit sell order. + :param pair: Pair as str, format: BTC_ETH + :param rate: Rate limit for order + :param amount: The amount to sell + :return: None + """ + if _conf['dry_run']: + return 'dry_run' + elif cur_exchange == Exchange.POLONIEX: + _api.sell(pair, rate, amount) + # TODO: return order id + elif cur_exchange == Exchange.BITTREX: + data = _api.sell_limit(pair.replace('_', '-'), amount, rate) + if not data['success']: + raise RuntimeError('BITTREX: {}'.format(data['message'])) + return data['result']['uuid'] + + +def get_balance(currency: str) -> float: + """ + Get account balance. + :param currency: currency as str, format: BTC + :return: float + """ + if _conf['dry_run']: + return 999.9 + elif cur_exchange == Exchange.POLONIEX: + data = _api.returnBalances() + return float(data[currency]) + elif cur_exchange == Exchange.BITTREX: + data = _api.get_balance(currency) + if not data['success']: + raise RuntimeError('BITTREX: {}'.format(data['message'])) + return float(data['result']['Balance'] or 0.0) + + +def get_ticker(pair: str) -> dict: + """ + Get Ticker for given pair. + :param pair: Pair as str, format: BTC_ETC + :return: dict + """ + if cur_exchange == Exchange.POLONIEX: + data = _api.returnTicker() + return { + 'bid': float(data[pair]['highestBid']), + 'ask': float(data[pair]['lowestAsk']), + 'last': float(data[pair]['last']) + } + elif cur_exchange == Exchange.BITTREX: + data = _api.get_ticker(pair.replace('_', '-')) + if not data['success']: + raise RuntimeError('BITTREX: {}'.format(data['message'])) + return { + 'bid': float(data['result']['Bid']), + 'ask': float(data['result']['Ask']), + 'last': float(data['result']['Last']), + } + + +def cancel_order(order_id: str) -> None: + """ + Cancel order for given order_id + :param order_id: id as str + :return: None + """ + if _conf['dry_run']: + pass + elif cur_exchange == Exchange.POLONIEX: + raise NotImplemented('Not implemented') + elif cur_exchange == Exchange.BITTREX: + data = _api.cancel(order_id) + if not data['success']: + raise RuntimeError('BITTREX: {}'.format(data['message'])) + + +def get_open_orders(pair: str) -> List[dict]: + """ + Get all open orders for given pair. + :param pair: Pair as str, format: BTC_ETC + :return: list of dicts + """ + if _conf['dry_run']: + return [] + elif cur_exchange == Exchange.POLONIEX: + raise NotImplemented('Not implemented') + elif cur_exchange == Exchange.BITTREX: + data = _api.get_open_orders(pair.replace('_', '-')) + if not data['success']: + raise RuntimeError('BITTREX: {}'.format(data['message'])) + return [{ + 'id': entry['OrderUuid'], + 'type': entry['OrderType'], + 'opened': entry['Opened'], + 'rate': entry['PricePerUnit'], + 'amount': entry['Quantity'], + 'remaining': entry['QuantityRemaining'], + } for entry in data['result']] + + +def get_pair_detail_url(pair: str) -> str: + """ + Returns the market detail url for the given pair + :param pair: pair as str, format: BTC_ANT + :return: url as str + """ + if cur_exchange == Exchange.POLONIEX: + raise NotImplemented('Not implemented') + elif cur_exchange == Exchange.BITTREX: + return 'https://bittrex.com/Market/Index?MarketName={}'.format(pair.replace('_', '-')) + + +def get_markets() -> List[str]: + """ + Returns all available markets + :return: list of all available pairs + """ + if cur_exchange == Exchange.POLONIEX: + # TODO: implement + raise NotImplemented('Not implemented') + elif cur_exchange == Exchange. BITTREX: + data = _api.get_markets() + if not data['success']: + raise RuntimeError('BITTREX: {}'.format(data['message'])) + return [m['MarketName'].replace('-', '_') for m in data['result']] diff --git a/main.py b/main.py index a97419c94..d0c9287d6 100755 --- a/main.py +++ b/main.py @@ -1,19 +1,23 @@ #!/usr/bin/env python +import enum +import json import logging -import threading import time import traceback from datetime import datetime from json import JSONDecodeError from typing import Optional +from jsonschema import validate from requests import ConnectionError from wrapt import synchronized + +import exchange +import persistence +from rpc import telegram from analyze import get_buy_signal -from persistence import Trade, Session -from exchange import get_exchange_api, Exchange -from rpc.telegram import TelegramHandler -from utils import get_conf +from persistence import Trade +from misc import conf_schema logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') @@ -25,103 +29,79 @@ __license__ = "GPLv3" __version__ = "0.8.0" -CONFIG = get_conf() -api_wrapper = get_exchange_api(CONFIG) +class State(enum.Enum): + RUNNING = 0 + PAUSED = 1 + TERMINATE = 2 -class TradeThread(threading.Thread): - def __init__(self): - super().__init__() - self._should_stop = False - - def stop(self) -> None: - """ stops the trader thread """ - self._should_stop = True - - def run(self) -> None: - """ - Threaded main function - :return: None - """ - try: - TelegramHandler.send_msg('*Status:* `trader started`') - logger.info('Trader started') - while not self._should_stop: - try: - self._process() - except (ConnectionError, JSONDecodeError, ValueError) as error: - msg = 'Got {} during _process()'.format(error.__class__.__name__) - logger.exception(msg) - finally: - Session.flush() - time.sleep(25) - except (RuntimeError, JSONDecodeError): - TelegramHandler.send_msg('*Status:* Got RuntimeError: ```\n{}\n```'.format(traceback.format_exc())) - logger.exception('RuntimeError. Stopping trader ...') - finally: - TelegramHandler.send_msg('*Status:* `Trader has stopped`') - - @staticmethod - def _process() -> None: - """ - Queries the persistence layer for open trades and handles them, - otherwise a new trade is created. - :return: None - """ - # Query trades from persistence layer - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if len(trades) < CONFIG['max_open_trades']: - try: - # Create entity and execute trade - trade = create_trade(float(CONFIG['stake_amount']), api_wrapper.exchange) - if trade: - Session.add(trade) - else: - logging.info('Got no buy signal...') - except ValueError: - logger.exception('Unable to create trade') - - for trade in trades: - # Check if there is already an open order for this trade - orders = api_wrapper.get_open_orders(trade.pair) - orders = [o for o in orders if o['id'] == trade.open_order_id] - if orders: - msg = 'There exists an open order for {}: Order(total={}, remaining={}, type={}, id={})' \ - .format( - trade, - round(orders[0]['amount'], 8), - round(orders[0]['remaining'], 8), - orders[0]['type'], - orders[0]['id']) - logger.info(msg) - continue - - # Update state - trade.open_order_id = None - # Check if this trade can be marked as closed - if close_trade_if_fulfilled(trade): - logger.info('No open orders found and trade is fulfilled. Marking %s as closed ...', trade) - continue - - # Check if we can sell our current pair - handle_trade(trade) - -# Initial stopped TradeThread instance -_instance = TradeThread() +_conf = {} +_cur_state = State.RUNNING @synchronized -def get_instance(recreate: bool=False) -> TradeThread: +def update_state(state: State) -> None: """ - Get the current instance of this thread. This is a singleton. - :param recreate: Must be True if you want to start the instance - :return: TradeThread instance + Updates the application state + :param state: new state + :return: None """ - global _instance - if recreate and not _instance.is_alive(): - logger.debug('Creating thread instance...') - _instance = TradeThread() - return _instance + global _cur_state + _cur_state = state + + +@synchronized +def get_state() -> State: + """ + Gets the current application state + :return: + """ + return _cur_state + + +def _process() -> None: + """ + Queries the persistence layer for open trades and handles them, + otherwise a new trade is created. + :return: None + """ + # Query trades from persistence layer + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + if len(trades) < _conf['max_open_trades']: + try: + # Create entity and execute trade + trade = create_trade(float(_conf['stake_amount']), exchange.cur_exchange) + if trade: + Trade.session.add(trade) + else: + logging.info('Got no buy signal...') + except ValueError: + logger.exception('Unable to create trade') + + for trade in trades: + # Check if there is already an open order for this trade + orders = exchange.get_open_orders(trade.pair) + orders = [o for o in orders if o['id'] == trade.open_order_id] + if orders: + msg = 'There exists an open order for {}: Order(total={}, remaining={}, type={}, id={})' \ + .format( + trade, + round(orders[0]['amount'], 8), + round(orders[0]['remaining'], 8), + orders[0]['type'], + orders[0]['id']) + logger.info(msg) + continue + + # Update state + trade.open_order_id = None + # Check if this trade can be marked as closed + if close_trade_if_fulfilled(trade): + logger.info('No open orders found and trade is fulfilled. Marking %s as closed ...', trade) + continue + + # Check if we can sell our current pair + handle_trade(trade) def close_trade_if_fulfilled(trade: Trade) -> bool: @@ -134,7 +114,6 @@ def close_trade_if_fulfilled(trade: Trade) -> bool: # we can close this trade. if trade.close_profit and trade.close_date and trade.close_rate and not trade.open_order_id: trade.is_open = False - Session.flush() return True return False @@ -150,14 +129,14 @@ def handle_trade(trade: Trade) -> None: logger.debug('Handling open trade %s ...', trade) # Get current rate - current_rate = api_wrapper.get_ticker(trade.pair)['bid'] + current_rate = exchange.get_ticker(trade.pair)['bid'] current_profit = 100 * ((current_rate - trade.open_rate) / trade.open_rate) # Get available balance currency = trade.pair.split('_')[1] - balance = api_wrapper.get_balance(currency) + balance = exchange.get_balance(currency) - for duration, threshold in sorted(CONFIG['minimal_roi'].items()): + for duration, threshold in sorted(_conf['minimal_roi'].items()): duration, threshold = float(duration), float(threshold) # Check if time matches and current rate is above threshold time_diff = (datetime.utcnow() - trade.open_date).total_seconds() / 60 @@ -167,12 +146,12 @@ def handle_trade(trade: Trade) -> None: message = '*{}:* Selling [{}]({}) at rate `{:f} (profit: {}%)`'.format( trade.exchange.name, trade.pair.replace('_', '/'), - api_wrapper.get_pair_detail_url(trade.pair), + exchange.get_pair_detail_url(trade.pair), trade.close_rate, round(profit, 2) ) logger.info(message) - TelegramHandler.send_msg(message) + telegram.send_msg(message) return else: logger.debug('Threshold not reached. (cur_profit: %1.2f%%)', current_profit) @@ -180,18 +159,18 @@ def handle_trade(trade: Trade) -> None: logger.exception('Unable to handle open order') -def create_trade(stake_amount: float, exchange: Exchange) -> Optional[Trade]: +def create_trade(stake_amount: float, _exchange: exchange.Exchange) -> Optional[Trade]: """ Checks the implemented trading indicator(s) for a randomly picked pair, if one pair triggers the buy_signal a new trade record gets created :param stake_amount: amount of btc to spend - :param exchange: exchange to use + :param _exchange: exchange to use """ logger.info('Creating new trade with stake_amount: %f ...', stake_amount) - whitelist = CONFIG[exchange.name.lower()]['pair_whitelist'] + whitelist = _conf[_exchange.name.lower()]['pair_whitelist'] # Check if btc_amount is fulfilled - if api_wrapper.get_balance(CONFIG['stake_currency']) < stake_amount: - raise ValueError('stake amount is not fulfilled (currency={}'.format(CONFIG['stake_currency'])) + if exchange.get_balance(_conf['stake_currency']) < stake_amount: + raise ValueError('stake amount is not fulfilled (currency={}'.format(_conf['stake_currency'])) # Remove currently opened and latest pairs from whitelist trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -213,30 +192,78 @@ def create_trade(stake_amount: float, exchange: Exchange) -> Optional[Trade]: else: return None - open_rate = api_wrapper.get_ticker(pair)['ask'] + open_rate = exchange.get_ticker(pair)['ask'] amount = stake_amount / open_rate - exchange = exchange - order_id = api_wrapper.buy(pair, open_rate, amount) + order_id = exchange.buy(pair, open_rate, amount) # Create trade entity and return message = '*{}:* Buying [{}]({}) at rate `{:f}`'.format( - exchange.name, + _exchange.name, pair.replace('_', '/'), - api_wrapper.get_pair_detail_url(pair), + exchange.get_pair_detail_url(pair), open_rate ) logger.info(message) - TelegramHandler.send_msg(message) + telegram.send_msg(message) return Trade(pair=pair, btc_amount=stake_amount, open_rate=open_rate, + open_date=datetime.utcnow(), amount=amount, - exchange=exchange, - open_order_id=order_id) + exchange=_exchange, + open_order_id=order_id, + is_open=True) + + +def init(config: dict) -> None: + """ + Initializes all modules and updates the config + :param config: config as dict + :return: None + """ + global _conf + + # Initialize all modules + telegram.init(config) + persistence.init(config) + exchange.init(config) + _conf.update(config) + + +def app(config: dict) -> None: + + logger.info('Starting freqtrade %s', __version__) + init(config) + + try: + telegram.send_msg('*Status:* `trader started`') + logger.info('Trader started') + while True: + state = get_state() + if state == State.TERMINATE: + return + elif state == State.PAUSED: + time.sleep(1) + elif state == State.RUNNING: + try: + _process() + except (ConnectionError, JSONDecodeError, ValueError) as error: + msg = 'Got {} during _process()'.format(error.__class__.__name__) + logger.exception(msg) + finally: + time.sleep(25) + except (RuntimeError, JSONDecodeError): + telegram.send_msg( + '*Status:* Got RuntimeError: ```\n{}\n```'.format(traceback.format_exc()) + ) + logger.exception('RuntimeError. Stopping trader ...') + finally: + telegram.send_msg('*Status:* `Trader has stopped`') if __name__ == '__main__': - logger.info('Starting freqtrade %s', __version__) - TelegramHandler.listen() - while True: - time.sleep(0.5) + with open('config.json') as file: + conf = json.load(file) + validate(conf, conf_schema) + app(conf) + diff --git a/utils.py b/misc.py similarity index 75% rename from utils.py rename to misc.py index e7bb006fd..6b67d0a64 100644 --- a/utils.py +++ b/misc.py @@ -1,16 +1,6 @@ -import json -import logging - -from jsonschema import validate -from wrapt import synchronized - -logger = logging.getLogger(__name__) - -_cur_conf = None - # Required json-schema for user specified config -_conf_schema = { +conf_schema = { 'type': 'object', 'properties': { 'max_open_trades': {'type': 'integer'}, @@ -65,18 +55,3 @@ _conf_schema = { 'telegram' ] } - - -@synchronized -def get_conf(filename: str='config.json') -> dict: - """ - Loads the config into memory validates it - and returns the singleton instance - :return: dict - """ - global _cur_conf - if not _cur_conf: - with open(filename) as file: - _cur_conf = json.load(file) - validate(_cur_conf, _conf_schema) - return _cur_conf diff --git a/persistence.py b/persistence.py index f8c954983..a0a85ed3d 100644 --- a/persistence.py +++ b/persistence.py @@ -5,27 +5,48 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.types import Enum -from exchange import Exchange, get_exchange_api -from utils import get_conf +import exchange + + +_db_handle = None +_session = None +_conf = {} -conf = get_conf() -if conf.get('dry_run', False): - db_handle = 'sqlite:///tradesv2.dry_run.sqlite' -else: - db_handle = 'sqlite:///tradesv2.sqlite' -engine = create_engine(db_handle, echo=False) -Session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Base = declarative_base() +def init(config: dict) -> None: + """ + Initializes this module with the given config, + registers all known command handlers + and starts polling for message updates + :param config: config to use + :return: None + """ + global _db_handle, _session + _conf.update(config) + if _conf.get('dry_run', False): + _db_handle = 'sqlite:///tradesv2.dry_run.sqlite' + else: + _db_handle = 'sqlite:///tradesv2.sqlite' + + engine = create_engine(_db_handle, echo=False) + _session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) + Trade.session = _session + Trade.query = _session.query_property() + Base.metadata.create_all(engine) + + +def get_session(): + return _session + + class Trade(Base): __tablename__ = 'trades' - query = Session.query_property() - id = Column(Integer, primary_key=True) - exchange = Column(Enum(Exchange), nullable=False) + exchange = Column(Enum(exchange.Exchange), nullable=False) pair = Column(String, nullable=False) is_open = Column(Boolean, nullable=False, default=True) open_rate = Column(Float, nullable=False) @@ -56,12 +77,10 @@ class Trade(Base): profit = 100 * ((rate - self.open_rate) / self.open_rate) # Execute sell and update trade record - order_id = get_exchange_api(conf).sell(self.pair, rate, amount) + order_id = exchange.sell(str(self.pair), rate, amount) self.close_rate = rate self.close_profit = profit self.close_date = datetime.utcnow() self.open_order_id = order_id - Session.flush() return profit -Base.metadata.create_all(engine) diff --git a/rpc/__init__.py b/rpc/__init__.py index e69de29bb..35fa001c3 100644 --- a/rpc/__init__.py +++ b/rpc/__init__.py @@ -0,0 +1 @@ +from . import telegram diff --git a/rpc/telegram.py b/rpc/telegram.py index efb8746fa..d6abeba61 100644 --- a/rpc/telegram.py +++ b/rpc/telegram.py @@ -9,9 +9,9 @@ from telegram.ext import CommandHandler, Updater from telegram import ParseMode, Bot, Update from wrapt import synchronized -from persistence import Trade, Session -from exchange import get_exchange_api -from utils import get_conf +from persistence import Trade + +import exchange # Remove noisy log messages logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) @@ -19,9 +19,38 @@ logging.getLogger('telegram').setLevel(logging.INFO) logger = logging.getLogger(__name__) _updater = None +_conf = {} -conf = get_conf() -api_wrapper = get_exchange_api(conf) + +def init(config: dict) -> None: + """ + Initializes this module with the given config, + registers all known command handlers + and starts polling for message updates + :param config: config to use + :return: None + """ + _conf.update(config) + + # Register command handler and start telegram message polling + handles = [ + CommandHandler('status', _status), + CommandHandler('profit', _profit), + CommandHandler('start', _start), + CommandHandler('stop', _stop), + CommandHandler('forcesell', _forcesell), + CommandHandler('performance', _performance), + ] + for handle in handles: + get_updater(_conf).dispatcher.add_handler(handle) + get_updater(_conf).start_polling( + clean=True, + bootstrap_retries=3, + timeout=30, + read_latency=60, + ) + logger.info('rpc.telegram is listening for following commands: {}' + .format([h.command for h in handles])) def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[..., Any]: @@ -35,7 +64,7 @@ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[ if not isinstance(bot, Bot) or not isinstance(update, Update): raise ValueError('Received invalid Arguments: {}'.format(*args)) - chat_id = int(conf['telegram']['chat_id']) + chat_id = int(_conf['telegram']['chat_id']) if int(update.message.chat_id) == chat_id: logger.info('Executing handler: %s for chat_id: %s', command_handler.__name__, chat_id) return command_handler(*args, **kwargs) @@ -44,33 +73,31 @@ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[ return wrapper -class TelegramHandler(object): - @staticmethod - @authorized_only - def _status(bot: Bot, update: Update) -> None: - """ - Handler for /status. - Returns the current TradeThread status - :param bot: telegram bot - :param update: message update - :return: None - """ - # Fetch open trade - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - from main import get_instance - if not get_instance().is_alive(): - TelegramHandler.send_msg('*Status:* `trader is not running`', bot=bot) - elif not trades: - TelegramHandler.send_msg('*Status:* `no active order`', bot=bot) - else: - for trade in trades: - # calculate profit and send message to user - current_rate = api_wrapper.get_ticker(trade.pair)['bid'] - current_profit = 100 * ((current_rate - trade.open_rate) / trade.open_rate) - orders = api_wrapper.get_open_orders(trade.pair) - orders = [o for o in orders if o['id'] == trade.open_order_id] - order = orders[0] if orders else None - message = """ +@authorized_only +def _status(bot: Bot, update: Update) -> None: + """ + Handler for /status. + Returns the current TradeThread status + :param bot: telegram bot + :param update: message update + :return: None + """ + # Fetch open trade + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + from main import get_state, State + if not get_state() == State.RUNNING: + send_msg('*Status:* `trader is not running`', bot=bot) + elif not trades: + send_msg('*Status:* `no active order`', bot=bot) + else: + for trade in trades: + # calculate profit and send message to user + current_rate = exchange.get_ticker(trade.pair)['bid'] + current_profit = 100 * ((current_rate - trade.open_rate) / trade.open_rate) + orders = exchange.get_open_orders(trade.pair) + orders = [o for o in orders if o['id'] == trade.open_order_id] + order = orders[0] if orders else None + message = """ *Trade ID:* `{trade_id}` *Current Pair:* [{pair}]({market_url}) *Open Since:* `{date}` @@ -81,239 +108,213 @@ class TelegramHandler(object): *Close Profit:* `{close_profit}` *Current Profit:* `{current_profit}%` *Open Order:* `{open_order}` - """.format( - trade_id=trade.id, - pair=trade.pair, - market_url=api_wrapper.get_pair_detail_url(trade.pair), - date=arrow.get(trade.open_date).humanize(), - open_rate=trade.open_rate, - close_rate=trade.close_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - close_profit='{}%'.format(round(trade.close_profit, 2)) if trade.close_profit else None, - current_profit=round(current_profit, 2), - open_order='{} ({})'.format(order['remaining'], order['type']) if order else None, - ) - TelegramHandler.send_msg(message, bot=bot) + """.format( + trade_id=trade.id, + pair=trade.pair, + market_url=exchange.get_pair_detail_url(trade.pair), + date=arrow.get(trade.open_date).humanize(), + open_rate=trade.open_rate, + close_rate=trade.close_rate, + current_rate=current_rate, + amount=round(trade.amount, 8), + close_profit='{}%'.format(round(trade.close_profit, 2)) if trade.close_profit else None, + current_profit=round(current_profit, 2), + open_order='{} ({})'.format(order['remaining'], order['type']) if order else None, + ) + send_msg(message, bot=bot) - @staticmethod - @authorized_only - def _profit(bot: Bot, update: Update) -> None: - """ - Handler for /profit. - Returns a cumulative profit statistics. - :param bot: telegram bot - :param update: message update - :return: None - """ - trades = Trade.query.order_by(Trade.id).all() - profit_amounts = [] - profits = [] - durations = [] - for trade in trades: - if trade.close_date: - durations.append((trade.close_date - trade.open_date).total_seconds()) - if trade.close_profit: - profit = trade.close_profit - else: - # Get current rate - current_rate = api_wrapper.get_ticker(trade.pair)['bid'] - profit = 100 * ((current_rate - trade.open_rate) / trade.open_rate) +@authorized_only +def _profit(bot: Bot, update: Update) -> None: + """ + Handler for /profit. + Returns a cumulative profit statistics. + :param bot: telegram bot + :param update: message update + :return: None + """ + trades = Trade.query.order_by(Trade.id).all() - profit_amounts.append((profit / 100) * trade.btc_amount) - profits.append(profit) + profit_amounts = [] + profits = [] + durations = [] + for trade in trades: + if trade.close_date: + durations.append((trade.close_date - trade.open_date).total_seconds()) + if trade.close_profit: + profit = trade.close_profit + else: + # Get current rate + current_rate = exchange.get_ticker(trade.pair)['bid'] + profit = 100 * ((current_rate - trade.open_rate) / trade.open_rate) - bp_pair, bp_rate = Session.query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ - .filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by('profit_sum DESC') \ - .first() + profit_amounts.append((profit / 100) * trade.btc_amount) + profits.append(profit) - markdown_msg = """ + bp_pair, bp_rate = Trade.session.query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ + .filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ + .order_by('profit_sum DESC') \ + .first() + + markdown_msg = """ *ROI:* `{profit_btc} ({profit}%)` *Trade Count:* `{trade_count}` *First Trade opened:* `{first_trade_date}` *Latest Trade opened:* `{latest_trade_date}` *Avg. Duration:* `{avg_duration}` *Best Performing:* `{best_pair}: {best_rate}%` - """.format( - profit_btc=round(sum(profit_amounts), 8), - profit=round(sum(profits), 2), - trade_count=len(trades), - first_trade_date=arrow.get(trades[0].open_date).humanize(), - latest_trade_date=arrow.get(trades[-1].open_date).humanize(), - avg_duration=str(timedelta(seconds=sum(durations) / float(len(durations)))).split('.')[0], - best_pair=bp_pair, - best_rate=round(bp_rate, 2), - ) - TelegramHandler.send_msg(markdown_msg, bot=bot) + """.format( + profit_btc=round(sum(profit_amounts), 8), + profit=round(sum(profits), 2), + trade_count=len(trades), + first_trade_date=arrow.get(trades[0].open_date).humanize(), + latest_trade_date=arrow.get(trades[-1].open_date).humanize(), + avg_duration=str(timedelta(seconds=sum(durations) / float(len(durations)))).split('.')[0], + best_pair=bp_pair, + best_rate=round(bp_rate, 2), + ) + send_msg(markdown_msg, bot=bot) - @staticmethod - @authorized_only - def _start(bot: Bot, update: Update) -> None: - """ - Handler for /start. - Starts TradeThread - :param bot: telegram bot - :param update: message update - :return: None - """ - from main import get_instance - if get_instance().is_alive(): - TelegramHandler.send_msg('*Status:* `already running`', bot=bot) - else: - get_instance(recreate=True).start() - @staticmethod - @authorized_only - def _stop(bot: Bot, update: Update) -> None: - """ - Handler for /stop. - Stops TradeThread - :param bot: telegram bot - :param update: message update - :return: None - """ - from main import get_instance - if get_instance().is_alive(): - TelegramHandler.send_msg('`Stopping trader ...`', bot=bot) - get_instance().stop() - else: - TelegramHandler.send_msg('*Status:* `already stopped`', bot=bot) +@authorized_only +def _start(bot: Bot, update: Update) -> None: + """ + Handler for /start. + Starts TradeThread + :param bot: telegram bot + :param update: message update + :return: None + """ + from main import get_state, State, update_state + if get_state() == State.RUNNING: + send_msg('*Status:* `already running`', bot=bot) + else: + update_state(State.RUNNING) - @staticmethod - @authorized_only - def _forcesell(bot: Bot, update: Update) -> None: - """ - Handler for /forcesell . - Sells the given trade at current price - :param bot: telegram bot - :param update: message update - :return: None - """ - from main import get_instance - if not get_instance().is_alive(): - TelegramHandler.send_msg('`trader is not running`', bot=bot) + +@authorized_only +def _stop(bot: Bot, update: Update) -> None: + """ + Handler for /stop. + Stops TradeThread + :param bot: telegram bot + :param update: message update + :return: None + """ + from main import get_state, State, update_state + if get_state() == State.RUNNING: + send_msg('`Stopping trader ...`', bot=bot) + update_state(State.PAUSED) + else: + send_msg('*Status:* `already stopped`', bot=bot) + + +@authorized_only +def _forcesell(bot: Bot, update: Update) -> None: + """ + Handler for /forcesell . + Sells the given trade at current price + :param bot: telegram bot + :param update: message update + :return: None + """ + from main import get_state, State + if get_state() != State.RUNNING: + send_msg('`trader is not running`', bot=bot) + return + + try: + trade_id = int(update.message.text + .replace('/forcesell', '') + .strip()) + # Query for trade + trade = Trade.query.filter(and_( + Trade.id == trade_id, + Trade.is_open.is_(True) + )).first() + if not trade: + send_msg('There is no open trade with ID: `{}`'.format(trade_id)) return + # Get current rate + current_rate = exchange.get_ticker(trade.pair)['bid'] + # Get available balance + currency = trade.pair.split('_')[1] + balance = exchange.get_balance(currency) + # Execute sell + profit = trade.exec_sell_order(current_rate, balance) + message = '*{}:* Selling [{}]({}) at rate `{:f} (profit: {}%)`'.format( + trade.exchange.name, + trade.pair.replace('_', '/'), + exchange.get_pair_detail_url(trade.pair), + trade.close_rate, + round(profit, 2) + ) + logger.info(message) + send_msg(message) + except ValueError: + send_msg('Invalid argument. Usage: `/forcesell `') + logger.warning('/forcesell: Invalid argument received') + + +@authorized_only +def _performance(bot: Bot, update: Update) -> None: + """ + Handler for /performance. + Shows a performance statistic from finished trades + :param bot: telegram bot + :param update: message update + :return: None + """ + from main import get_state, State + if get_state() != State.RUNNING: + send_msg('`trader is not running`', bot=bot) + return + + pair_rates = Trade.session.query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ + .filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ + .order_by('profit_sum DESC') \ + .all() + + stats = '\n'.join('{}. {}\t{}%'.format(i + 1, pair, round(rate, 2)) for i, (pair, rate) in enumerate(pair_rates)) + + message = 'Performance:\n{}\n'.format(stats) + logger.debug(message) + send_msg(message, parse_mode=ParseMode.HTML) + + +@synchronized +def get_updater(config: dict) -> Updater: + """ + Returns the current telegram updater or instantiates a new one + :param config: dict + :return: telegram.ext.Updater + """ + global _updater + if not _updater: + _updater = Updater(token=config['telegram']['token'], workers=0) + return _updater + + +def send_msg(msg: str, bot: Bot=None, parse_mode: ParseMode=ParseMode.MARKDOWN) -> None: + """ + Send given markdown message + :param msg: message + :param bot: alternative bot + :param parse_mode: telegram parse mode + :return: None + """ + if _conf['telegram'].get('enabled', False): try: - trade_id = int(update.message.text - .replace('/forcesell', '') - .strip()) - # Query for trade - trade = Trade.query.filter(and_( - Trade.id == trade_id, - Trade.is_open.is_(True) - )).first() - if not trade: - TelegramHandler.send_msg('There is no open trade with ID: `{}`'.format(trade_id)) - return - # Get current rate - current_rate = api_wrapper.get_ticker(trade.pair)['bid'] - # Get available balance - currency = trade.pair.split('_')[1] - balance = api_wrapper.get_balance(currency) - # Execute sell - profit = trade.exec_sell_order(current_rate, balance) - message = '*{}:* Selling [{}]({}) at rate `{:f} (profit: {}%)`'.format( - trade.exchange.name, - trade.pair.replace('_', '/'), - api_wrapper.get_pair_detail_url(trade.pair), - trade.close_rate, - round(profit, 2) - ) - logger.info(message) - TelegramHandler.send_msg(message) - - except ValueError: - TelegramHandler.send_msg('Invalid argument. Usage: `/forcesell `') - logger.warning('/forcesell: Invalid argument received') - - @staticmethod - @authorized_only - def _performance(bot: Bot, update: Update) -> None: - """ - Handler for /performance. - Shows a performance statistic from finished trades - :param bot: telegram bot - :param update: message update - :return: None - """ - from main import get_instance - if not get_instance().is_alive(): - TelegramHandler.send_msg('`trader is not running`', bot=bot) - return - - pair_rates = Session.query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ - .filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by('profit_sum DESC') \ - .all() - - stats = '\n'.join('{}. {}\t{}%'.format(i + 1, pair, round(rate, 2)) for i, (pair, rate) in enumerate(pair_rates)) - - message = 'Performance:\n{}\n'.format(stats) - logger.debug(message) - TelegramHandler.send_msg(message, parse_mode=ParseMode.HTML) - - @staticmethod - @synchronized - def get_updater(config: dict) -> Updater: - """ - Returns the current telegram updater or instantiates a new one - :param config: dict - :return: telegram.ext.Updater - """ - global _updater - if not _updater: - _updater = Updater(token=config['telegram']['token'], workers=0) - return _updater - - @staticmethod - def listen() -> None: - """ - Registers all known command handlers and starts polling for message updates - :return: None - """ - # Register command handler and start telegram message polling - handles = [ - CommandHandler('status', TelegramHandler._status), - CommandHandler('profit', TelegramHandler._profit), - CommandHandler('start', TelegramHandler._start), - CommandHandler('stop', TelegramHandler._stop), - CommandHandler('forcesell', TelegramHandler._forcesell), - CommandHandler('performance', TelegramHandler._performance), - ] - for handle in handles: - TelegramHandler.get_updater(conf).dispatcher.add_handler(handle) - TelegramHandler.get_updater(conf).start_polling( - clean=True, - bootstrap_retries=3, - timeout=30, - read_latency=60, - ) - logger.info('TelegramHandler is listening for following commands: {}' - .format([h.command for h in handles])) - - @staticmethod - def send_msg(msg: str, bot: Bot=None, parse_mode: ParseMode=ParseMode.MARKDOWN) -> None: - """ - Send given markdown message - :param msg: message - :param bot: alternative bot - :param parse_mode: telegram parse mode - :return: None - """ - if conf['telegram'].get('enabled', False): + bot = bot or get_updater(_conf).bot try: - bot = bot or TelegramHandler.get_updater(conf).bot - try: - bot.send_message(conf['telegram']['chat_id'], msg, parse_mode=parse_mode) - except NetworkError as error: - # Sometimes the telegram server resets the current connection, - # if this is the case we send the message again. - logger.warning('Got Telegram NetworkError: %s! Trying one more time.', error.message) - bot.send_message(conf['telegram']['chat_id'], msg, parse_mode=parse_mode) - except Exception: - logger.exception('Exception occurred within Telegram API') + bot.send_message(_conf['telegram']['chat_id'], msg, parse_mode=parse_mode) + except NetworkError as error: + # Sometimes the telegram server resets the current connection, + # if this is the case we send the message again. + logger.warning('Got Telegram NetworkError: %s! Trying one more time.', error.message) + bot.send_message(_conf['telegram']['chat_id'], msg, parse_mode=parse_mode) + except Exception: + logger.exception('Exception occurred within Telegram API') diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/test_main.py b/test/test_main.py new file mode 100644 index 000000000..99094f72b --- /dev/null +++ b/test/test_main.py @@ -0,0 +1,112 @@ +import unittest +from unittest.mock import patch, MagicMock + +import os +from jsonschema import validate + +import exchange +from main import create_trade, handle_trade, close_trade_if_fulfilled, init +from misc import conf_schema +from persistence import Trade + + +class TestMain(unittest.TestCase): + conf = { + "max_open_trades": 3, + "stake_currency": "BTC", + "stake_amount": 0.05, + "dry_run": True, + "minimal_roi": { + "2880": 0.005, + "720": 0.01, + "0": 0.02 + }, + "poloniex": { + "enabled": False, + "key": "key", + "secret": "secret", + "pair_whitelist": [] + }, + "bittrex": { + "enabled": True, + "key": "key", + "secret": "secret", + "pair_whitelist": [ + "BTC_ETH" + ] + }, + "telegram": { + "enabled": True, + "token": "token", + "chat_id": "chat_id" + } + } + + def test_1_create_trade(self): + with patch.dict('main._conf', self.conf): + with patch('main.get_buy_signal', side_effect=lambda _: True) as buy_signal: + with patch.multiple('main.telegram', init=MagicMock(), send_msg=MagicMock()): + with patch.multiple('main.exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.07256061, + 'ask': 0.072661, + 'last': 0.07256061 + }), + buy=MagicMock(return_value='mocked_order_id')): + init(self.conf) + trade = create_trade(15.0, exchange.Exchange.BITTREX) + Trade.session.add(trade) + Trade.session.flush() + self.assertIsNotNone(trade) + self.assertEqual(trade.open_rate, 0.072661) + self.assertEqual(trade.pair, 'BTC_ETH') + self.assertEqual(trade.exchange, exchange.Exchange.BITTREX) + self.assertEqual(trade.amount, 206.43811673387373) + self.assertEqual(trade.btc_amount, 15.0) + self.assertEqual(trade.is_open, True) + self.assertIsNotNone(trade.open_date) + buy_signal.assert_called_once_with('BTC_ETH') + + def test_2_handle_trade(self): + with patch.dict('main._conf', self.conf): + with patch.multiple('main.telegram', init=MagicMock(), send_msg=MagicMock()): + with patch.multiple('main.exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.17256061, + 'ask': 0.172661, + 'last': 0.17256061 + }), + buy=MagicMock(return_value='mocked_order_id')): + trade = Trade.query.filter(Trade.is_open.is_(True)).first() + self.assertTrue(trade) + handle_trade(trade) + self.assertEqual(trade.close_rate, 0.17256061) + self.assertEqual(trade.close_profit, 137.4872490056564) + self.assertIsNotNone(trade.close_date) + self.assertEqual(trade.open_order_id, 'dry_run') + + def test_3_close_trade(self): + with patch.dict('main._conf', self.conf): + trade = Trade.query.filter(Trade.is_open.is_(True)).first() + self.assertTrue(trade) + + # Simulate that there is no open order + trade.open_order_id = None + + closed = close_trade_if_fulfilled(trade) + self.assertTrue(closed) + self.assertEqual(trade.is_open, False) + + @classmethod + def setUpClass(cls): + validate(cls.conf, conf_schema) + + @classmethod + def tearDownClass(cls): + try: + os.remove('./tradesv2.dry_run.sqlite') + except FileNotFoundError: + pass + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_persistence.py b/test/test_persistence.py new file mode 100644 index 000000000..25bdfd4e1 --- /dev/null +++ b/test/test_persistence.py @@ -0,0 +1,28 @@ +import unittest +from unittest.mock import patch, Mock + +from exchange import Exchange +from persistence import Trade + + +class TestTrade(unittest.TestCase): + def test_1_exec_sell_order(self): + with patch('main.exchange.sell', side_effect='mocked_order_id') as api_mock: + trade = Trade( + pair='BTC_ETH', + btc_amount=1.00, + open_rate=0.50, + amount=10.00, + exchange=Exchange.BITTREX, + open_order_id='mocked' + ) + profit = trade.exec_sell_order(1.00, 10.00) + api_mock.assert_called_once_with('BTC_ETH', 1.0, 10.0) + self.assertEqual(profit, 100.0) + self.assertEqual(trade.close_rate, 1.0) + self.assertEqual(trade.close_profit, profit) + self.assertIsNotNone(trade.close_date) + + +if __name__ == '__main__': + unittest.main() From 8a722857981c98c4f34e66782bb1455cd0fe6287 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 16:39:17 +0200 Subject: [PATCH 02/28] enable ci --- .travis.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..83c292a1d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,27 @@ +sudo: false +os: + - linux + +language: python +python: + - 3.6 + - nightly +matrix: + allow_failures: + - python: nightly + +addons: + apt: + packages: + - libelf-dev + - libdw-dev + - binutils-dev + +install: + - wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz + - tar zxvf ta-lib-0.4.0-src.tar.gz + - cd ta-lib && ./configure && make && make install + - pip install -r requirements.txt + +script: + - python -m unittest \ No newline at end of file From 83b750f8af49526fdf59bebc803b4015711231be Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 16:45:22 +0200 Subject: [PATCH 03/28] use sudo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 83c292a1d..3dcf94719 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ addons: install: - wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz - tar zxvf ta-lib-0.4.0-src.tar.gz - - cd ta-lib && ./configure && make && make install + - cd ta-lib && ./configure && sudo make && sudo make install - pip install -r requirements.txt script: From 1528041520a6a399833c310210962676ad92819a Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 16:50:08 +0200 Subject: [PATCH 04/28] fix path issue --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3dcf94719..6172f52c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,8 @@ addons: install: - wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz - tar zxvf ta-lib-0.4.0-src.tar.gz - - cd ta-lib && ./configure && sudo make && sudo make install + - cd ta-lib && ./configure && sudo make && sudo make install && cd .. + - ENV LD_LIBRARY_PATH /usr/local/lib - pip install -r requirements.txt script: From 3dade54b67ceaf6c716c7484e378b9eb84319dc1 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 16:55:03 +0200 Subject: [PATCH 05/28] export lib path --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6172f52c2..b9f41da0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ install: - wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz - tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && ./configure && sudo make && sudo make install && cd .. - - ENV LD_LIBRARY_PATH /usr/local/lib + - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install -r requirements.txt script: From 13786ee32c980696154108fb1385b35ce25ee068 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 17:35:21 +0200 Subject: [PATCH 06/28] add Build Status badge --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index bfd640060..e0f79f4d0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ # freqtrade + +[![Build Status](https://travis-ci.org/gcarq/freqtrade.svg?branch=develop)](https://travis-ci.org/gcarq/freqtrade) + Simple High frequency trading bot for crypto currencies. Currently supported exchanges: bittrex, poloniex (partly implemented) From 470f9b76bf90356486bd00ab3cf0a39d099d42ef Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 17:40:29 +0200 Subject: [PATCH 07/28] remove unneeded command --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d29d81cf2..f03d75e4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,6 @@ RUN apt-get -y install build-essential RUN wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz RUN tar zxvf ta-lib-0.4.0-src.tar.gz RUN cd ta-lib && ./configure && make && make install -RUN pip install TA-Lib ENV LD_LIBRARY_PATH /usr/local/lib RUN mkdir -p /freqtrade From efa04443044cacffe0c5add6e50ddb0ca68e7ce0 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 17:42:23 +0200 Subject: [PATCH 08/28] add test-suite usage to README.md --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e0f79f4d0..d3d3ae36c 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ and enter the telegram `token` and your `chat_id` in `config.json` Persistence is achieved through sqlite. -##### Telegram RPC commands: +#### Telegram RPC commands: * /start: Starts the trader * /stop: Stops the trader * /status: Lists all open trades @@ -22,7 +22,7 @@ Persistence is achieved through sqlite. * /forcesell : Instantly sells the given trade (Ignoring `minimum_roi`). * /performance: Show performance of each finished trade grouped by pair -##### Config +#### Config `minimal_roi` is a JSON object where the key is a duration in minutes and the value is the minimum ROI in percent. See the example below: @@ -48,7 +48,7 @@ if not feel free to raise a github issue. * sqlite * [TA-lib](https://github.com/mrjbq7/ta-lib#dependencies) binaries -##### Install +#### Install ``` $ cd freqtrade/ # copy example config. Dont forget to insert your api keys @@ -59,6 +59,11 @@ $ pip install -r requirements.txt $ ./main.py ``` +#### Execute tests + +``` +$ python -m unittest +``` #### Docker ``` From c737041acb294bbdc51b9ade5b61bce9343c88d4 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 19:25:12 +0200 Subject: [PATCH 09/28] add some tests for rpc.teleram --- test/test_main.py | 6 +- test/test_persistence.py | 2 +- test/test_telegram.py | 118 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 test/test_telegram.py diff --git a/test/test_main.py b/test/test_main.py index 99094f72b..8bd06c316 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -99,14 +99,12 @@ class TestMain(unittest.TestCase): @classmethod def setUpClass(cls): - validate(cls.conf, conf_schema) - - @classmethod - def tearDownClass(cls): try: os.remove('./tradesv2.dry_run.sqlite') except FileNotFoundError: pass + validate(cls.conf, conf_schema) + if __name__ == '__main__': unittest.main() diff --git a/test/test_persistence.py b/test/test_persistence.py index 25bdfd4e1..65af0ad8a 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -1,5 +1,5 @@ import unittest -from unittest.mock import patch, Mock +from unittest.mock import patch from exchange import Exchange from persistence import Trade diff --git a/test/test_telegram.py b/test/test_telegram.py new file mode 100644 index 000000000..ceb9e3e7f --- /dev/null +++ b/test/test_telegram.py @@ -0,0 +1,118 @@ +import unittest +from datetime import datetime +from unittest.mock import patch, MagicMock + +import os +from jsonschema import validate +from telegram import Bot, Update, Message, Chat + +import exchange +from main import init, create_trade +from misc import conf_schema +from persistence import Trade +from rpc.telegram import _status, _profit + + +class MagicBot(MagicMock, Bot): + pass + + +class TestTelegram(unittest.TestCase): + + conf = { + "max_open_trades": 3, + "stake_currency": "BTC", + "stake_amount": 0.05, + "dry_run": True, + "minimal_roi": { + "2880": 0.005, + "720": 0.01, + "0": 0.02 + }, + "poloniex": { + "enabled": False, + "key": "key", + "secret": "secret", + "pair_whitelist": [] + }, + "bittrex": { + "enabled": True, + "key": "key", + "secret": "secret", + "pair_whitelist": [ + "BTC_ETH" + ] + }, + "telegram": { + "enabled": True, + "token": "token", + "chat_id": "0" + } + } + + def test_1_status_handle(self): + with patch.dict('main._conf', self.conf): + with patch('main.get_buy_signal', side_effect=lambda _: True): + msg_mock = MagicMock() + with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + with patch.multiple('main.exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.07256061, + 'ask': 0.072661, + 'last': 0.07256061 + }), + buy=MagicMock(return_value='mocked_order_id')): + init(self.conf) + + # Create some test data + trade = create_trade(15.0, exchange.Exchange.BITTREX) + Trade.session.add(trade) + Trade.session.flush() + + _status(bot=MagicBot(), update=self.update) + self.assertEqual(msg_mock.call_count, 2) + self.assertIn('[BTC_ETH]', msg_mock.call_args_list[-1][0][0]) + + def test_2_profit_handle(self): + with patch.dict('main._conf', self.conf): + with patch('main.get_buy_signal', side_effect=lambda _: True): + msg_mock = MagicMock() + with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + with patch.multiple('main.exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.07256061, + 'ask': 0.072661, + 'last': 0.07256061 + }), + buy=MagicMock(return_value='mocked_order_id')): + init(self.conf) + + # Create some test data + trade = create_trade(15.0, exchange.Exchange.BITTREX) + trade.close_rate = 0.07256061 + trade.close_profit = 137.4872490056564 + trade.close_date = datetime.utcnow() + trade.open_order_id = None + trade.is_open = False + Trade.session.add(trade) + Trade.session.flush() + + _profit(bot=MagicBot(), update=self.update) + self.assertEqual(msg_mock.call_count, 2) + self.assertIn('(137.49%)', msg_mock.call_args_list[-1][0][0]) + + def setUp(self): + try: + os.remove('./tradesv2.dry_run.sqlite') + except FileNotFoundError: + pass + self.update = Update(0) + self.update.message = Message(0, 0, MagicMock(), Chat(0, 0)) + + @classmethod + def setUpClass(cls): + validate(cls.conf, conf_schema) + + +if __name__ == '__main__': + unittest.main() From 9aaa169ec321d4cdaf8d6282664db0d63262e162 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 19:25:39 +0200 Subject: [PATCH 10/28] fix NoneType issues --- rpc/telegram.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/rpc/telegram.py b/rpc/telegram.py index bd44cd190..807914f7e 100644 --- a/rpc/telegram.py +++ b/rpc/telegram.py @@ -3,7 +3,7 @@ from datetime import timedelta from typing import Callable, Any import arrow -from sqlalchemy import and_, func +from sqlalchemy import and_, func, text from telegram.error import NetworkError from telegram.ext import CommandHandler, Updater from telegram import ParseMode, Bot, Update @@ -60,7 +60,8 @@ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[ :return: decorated function """ def wrapper(*args, **kwargs): - bot, update = args[0], args[1] + bot, update = (args[0], args[1]) if args else (kwargs['bot'], kwargs['update']) + if not isinstance(bot, Bot) or not isinstance(update, Update): raise ValueError('Received invalid Arguments: {}'.format(*args)) @@ -151,12 +152,17 @@ def _profit(bot: Bot, update: Update) -> None: profit_amounts.append((profit / 100) * trade.btc_amount) profits.append(profit) - bp_pair, bp_rate = Trade.session().query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ + best_pair = Trade.session.query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ .filter(Trade.is_open.is_(False)) \ .group_by(Trade.pair) \ - .order_by('profit_sum DESC') \ + .order_by(text('profit_sum DESC')) \ .first() + if not best_pair: + send_msg('*Status:* `no closed trade`', bot=bot) + return + + bp_pair, bp_rate = best_pair markdown_msg = """ *ROI:* `{profit_btc} ({profit}%)` *Trade Count:* `{trade_count}` @@ -272,7 +278,7 @@ def _performance(bot: Bot, update: Update) -> None: send_msg('`trader is not running`', bot=bot) return - pair_rates = Trade.session().query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ + pair_rates = Trade.session.query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ .filter(Trade.is_open.is_(False)) \ .group_by(Trade.pair) \ .order_by('profit_sum DESC') \ From dad76c886fe8504a5bea0f57128ecc4d7b608088 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 19:45:54 +0200 Subject: [PATCH 11/28] show two digits of precision --- rpc/telegram.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpc/telegram.py b/rpc/telegram.py index 807914f7e..44d8addde 100644 --- a/rpc/telegram.py +++ b/rpc/telegram.py @@ -107,7 +107,7 @@ def _status(bot: Bot, update: Update) -> None: *Close Rate:* `{close_rate}` *Current Rate:* `{current_rate}` *Close Profit:* `{close_profit}` -*Current Profit:* `{current_profit}%` +*Current Profit:* `{current_profit:.2f}%` *Open Order:* `{open_order}` """.format( trade_id=trade.id, @@ -164,12 +164,12 @@ def _profit(bot: Bot, update: Update) -> None: bp_pair, bp_rate = best_pair markdown_msg = """ -*ROI:* `{profit_btc} ({profit}%)` +*ROI:* `{profit_btc:.2f} ({profit:.2f}%)` *Trade Count:* `{trade_count}` *First Trade opened:* `{first_trade_date}` *Latest Trade opened:* `{latest_trade_date}` *Avg. Duration:* `{avg_duration}` -*Best Performing:* `{best_pair}: {best_rate}%` +*Best Performing:* `{best_pair}: {best_rate:.2f}%` """.format( profit_btc=round(sum(profit_amounts), 8), profit=round(sum(profits), 2), From e38f73c0dc3c7de4eb4ecd62d66d6d36f5c89beb Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 19:50:48 +0200 Subject: [PATCH 12/28] show two digits of precision for /performance --- rpc/telegram.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rpc/telegram.py b/rpc/telegram.py index 44d8addde..e3da0a00d 100644 --- a/rpc/telegram.py +++ b/rpc/telegram.py @@ -284,7 +284,11 @@ def _performance(bot: Bot, update: Update) -> None: .order_by('profit_sum DESC') \ .all() - stats = '\n'.join('{}. {}\t{}%'.format(i + 1, pair, round(rate, 2)) for i, (pair, rate) in enumerate(pair_rates)) + stats = '\n'.join('{index}. {pair}\t{profit:.2f}%'.format( + index=i + 1, + pair=pair, + profit=round(rate, 2) + ) for i, (pair, rate) in enumerate(pair_rates)) message = 'Performance:\n{}\n'.format(stats) logger.debug(message) From 3ecfebee77597688c31a1077e96bf3da9f38040b Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 19:52:20 +0200 Subject: [PATCH 13/28] fix SAWarning --- rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/telegram.py b/rpc/telegram.py index e3da0a00d..f42189d07 100644 --- a/rpc/telegram.py +++ b/rpc/telegram.py @@ -281,7 +281,7 @@ def _performance(bot: Bot, update: Update) -> None: pair_rates = Trade.session.query(Trade.pair, func.sum(Trade.close_profit).label('profit_sum')) \ .filter(Trade.is_open.is_(False)) \ .group_by(Trade.pair) \ - .order_by('profit_sum DESC') \ + .order_by(text('profit_sum DESC')) \ .all() stats = '\n'.join('{index}. {pair}\t{profit:.2f}%'.format( From 4ec2ac6d608e0a95b0c66e046e8aea055de171ff Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 20:03:45 +0200 Subject: [PATCH 14/28] add more tests for rpc.telegram --- test/test_telegram.py | 94 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/test/test_telegram.py b/test/test_telegram.py index ceb9e3e7f..fe27fe875 100644 --- a/test/test_telegram.py +++ b/test/test_telegram.py @@ -7,10 +7,10 @@ from jsonschema import validate from telegram import Bot, Update, Message, Chat import exchange -from main import init, create_trade +from main import init, create_trade, update_state, State, get_state from misc import conf_schema from persistence import Trade -from rpc.telegram import _status, _profit +from rpc.telegram import _status, _profit, _forcesell, _performance, _start, _stop class MagicBot(MagicMock, Bot): @@ -66,6 +66,7 @@ class TestTelegram(unittest.TestCase): # Create some test data trade = create_trade(15.0, exchange.Exchange.BITTREX) + self.assertTrue(trade) Trade.session.add(trade) Trade.session.flush() @@ -89,8 +90,9 @@ class TestTelegram(unittest.TestCase): # Create some test data trade = create_trade(15.0, exchange.Exchange.BITTREX) + self.assertTrue(trade) trade.close_rate = 0.07256061 - trade.close_profit = 137.4872490056564 + trade.close_profit = 100.00 trade.close_date = datetime.utcnow() trade.open_order_id = None trade.is_open = False @@ -99,7 +101,89 @@ class TestTelegram(unittest.TestCase): _profit(bot=MagicBot(), update=self.update) self.assertEqual(msg_mock.call_count, 2) - self.assertIn('(137.49%)', msg_mock.call_args_list[-1][0][0]) + self.assertIn('(100.00%)', msg_mock.call_args_list[-1][0][0]) + + def test_3_forcesell_handle(self): + with patch.dict('main._conf', self.conf): + with patch('main.get_buy_signal', side_effect=lambda _: True): + msg_mock = MagicMock() + with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + with patch.multiple('main.exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.07256061, + 'ask': 0.072661, + 'last': 0.07256061 + }), + buy=MagicMock(return_value='mocked_order_id')): + init(self.conf) + + # Create some test data + trade = create_trade(15.0, exchange.Exchange.BITTREX) + self.assertTrue(trade) + Trade.session.add(trade) + Trade.session.flush() + + self.update.message.text = '/forcesell 1' + _forcesell(bot=MagicBot(), update=self.update) + + self.assertEqual(msg_mock.call_count, 2) + self.assertIn('Selling [BTC/ETH]', msg_mock.call_args_list[-1][0][0]) + self.assertIn('0.072561', msg_mock.call_args_list[-1][0][0]) + + def test_4_performance_handle(self): + with patch.dict('main._conf', self.conf): + with patch('main.get_buy_signal', side_effect=lambda _: True): + msg_mock = MagicMock() + with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + with patch.multiple('main.exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.07256061, + 'ask': 0.072661, + 'last': 0.07256061 + }), + buy=MagicMock(return_value='mocked_order_id')): + init(self.conf) + + # Create some test data + trade = create_trade(15.0, exchange.Exchange.BITTREX) + self.assertTrue(trade) + trade.close_rate = 0.07256061 + trade.close_profit = 100.00 + trade.close_date = datetime.utcnow() + trade.open_order_id = None + trade.is_open = False + Trade.session.add(trade) + Trade.session.flush() + + _performance(bot=MagicBot(), update=self.update) + self.assertEqual(msg_mock.call_count, 2) + self.assertIn('Performance', msg_mock.call_args_list[-1][0][0]) + self.assertIn('BTC_ETH 100.00%', msg_mock.call_args_list[-1][0][0]) + + def test_5_start_handle(self): + with patch.dict('main._conf', self.conf): + msg_mock = MagicMock() + with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + init(self.conf) + + update_state(State.PAUSED) + self.assertEqual(get_state(), State.PAUSED) + _start(bot=MagicBot(), update=self.update) + self.assertEqual(get_state(), State.RUNNING) + self.assertEqual(msg_mock.call_count, 0) + + def test_6_stop_handle(self): + with patch.dict('main._conf', self.conf): + msg_mock = MagicMock() + with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + init(self.conf) + + update_state(State.RUNNING) + self.assertEqual(get_state(), State.RUNNING) + _stop(bot=MagicBot(), update=self.update) + self.assertEqual(get_state(), State.PAUSED) + self.assertEqual(msg_mock.call_count, 1) + self.assertIn('Stopping trader', msg_mock.call_args_list[0][0][0]) def setUp(self): try: @@ -107,7 +191,7 @@ class TestTelegram(unittest.TestCase): except FileNotFoundError: pass self.update = Update(0) - self.update.message = Message(0, 0, MagicMock(), Chat(0, 0)) + self.update.message = Message(0, 0, datetime.utcnow(), Chat(0, 0)) @classmethod def setUpClass(cls): From fe36f3a552d763711a887eb31ba9067bfa1667a7 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 21:17:13 +0200 Subject: [PATCH 15/28] remove get_updater --- rpc/telegram.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/rpc/telegram.py b/rpc/telegram.py index f42189d07..96adaa739 100644 --- a/rpc/telegram.py +++ b/rpc/telegram.py @@ -7,7 +7,6 @@ from sqlalchemy import and_, func, text from telegram.error import NetworkError from telegram.ext import CommandHandler, Updater from telegram import ParseMode, Bot, Update -from wrapt import synchronized from persistence import Trade @@ -30,6 +29,9 @@ def init(config: dict) -> None: :param config: config to use :return: None """ + global _updater + _updater = Updater(token=config['telegram']['token'], workers=0) + _conf.update(config) # Register command handler and start telegram message polling @@ -42,8 +44,8 @@ def init(config: dict) -> None: CommandHandler('performance', _performance), ] for handle in handles: - get_updater(_conf).dispatcher.add_handler(handle) - get_updater(_conf).start_polling( + _updater.dispatcher.add_handler(handle) + _updater.start_polling( clean=True, bootstrap_retries=3, timeout=30, @@ -295,19 +297,6 @@ def _performance(bot: Bot, update: Update) -> None: send_msg(message, parse_mode=ParseMode.HTML) -@synchronized -def get_updater(config: dict) -> Updater: - """ - Returns the current telegram updater or instantiates a new one - :param config: dict - :return: telegram.ext.Updater - """ - global _updater - if not _updater: - _updater = Updater(token=config['telegram']['token'], workers=0) - return _updater - - def send_msg(msg: str, bot: Bot=None, parse_mode: ParseMode=ParseMode.MARKDOWN) -> None: """ Send given markdown message @@ -318,7 +307,7 @@ def send_msg(msg: str, bot: Bot=None, parse_mode: ParseMode=ParseMode.MARKDOWN) """ if _conf['telegram'].get('enabled', False): try: - bot = bot or get_updater(_conf).bot + bot = bot or _updater.bot try: bot.send_message(_conf['telegram']['chat_id'], msg, parse_mode=parse_mode) except NetworkError as error: From 09e4c6893eae9aebc432f65ddf3fa9bdf3f61830 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 21:17:58 +0200 Subject: [PATCH 16/28] fix close_trade_if_fulfilled; flush sessions manually (fixes #16) --- main.py | 26 ++++++++++++++------------ persistence.py | 16 +++++++--------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/main.py b/main.py index af96640e0..af6fed2b3 100755 --- a/main.py +++ b/main.py @@ -79,6 +79,10 @@ def _process() -> None: logger.exception('Unable to create trade') for trade in trades: + if close_trade_if_fulfilled(trade): + logger.info('No open orders found and trade is fulfilled. Marking %s as closed ...', trade) + + for trade in filter(lambda t: t.is_open, trades): # Check if there is already an open order for this trade orders = exchange.get_open_orders(trade.pair) orders = [o for o in orders if o['id'] == trade.open_order_id] @@ -91,17 +95,11 @@ def _process() -> None: orders[0]['type'], orders[0]['id']) logger.info(msg) - continue - - # Update state - trade.open_order_id = None - # Check if this trade can be marked as closed - if close_trade_if_fulfilled(trade): - logger.info('No open orders found and trade is fulfilled. Marking %s as closed ...', trade) - continue - - # Check if we can sell our current pair - handle_trade(trade) + else: + # Update state + trade.open_order_id = None + # Check if we can sell our current pair + handle_trade(trade) def close_trade_if_fulfilled(trade: Trade) -> bool: @@ -112,7 +110,10 @@ def close_trade_if_fulfilled(trade: Trade) -> bool: """ # If we don't have an open order and the close rate is already set, # we can close this trade. - if trade.close_profit and trade.close_date and trade.close_rate and not trade.open_order_id: + if trade.close_profit is not None \ + and trade.close_date is not None \ + and trade.close_rate is not None \ + and trade.open_order_id is None: trade.is_open = False return True return False @@ -261,6 +262,7 @@ def app(config: dict) -> None: elif state == State.RUNNING: try: _process() + Trade.session.flush() except (ConnectionError, JSONDecodeError, ValueError) as error: msg = 'Got {} during _process()'.format(error.__class__.__name__) logger.exception(msg) diff --git a/persistence.py b/persistence.py index 47bb016a5..ce1a1999a 100644 --- a/persistence.py +++ b/persistence.py @@ -11,7 +11,6 @@ import exchange _db_handle = None -_session = None _conf = {} @@ -26,7 +25,7 @@ def init(config: dict) -> None: :param config: config to use :return: None """ - global _db_handle, _session + global _db_handle _conf.update(config) if _conf.get('dry_run', False): _db_handle = 'sqlite:///tradesv2.dry_run.sqlite' @@ -34,16 +33,12 @@ def init(config: dict) -> None: _db_handle = 'sqlite:///tradesv2.sqlite' engine = create_engine(_db_handle, echo=False) - _session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) - Trade.session = _session() - Trade.query = _session.query_property() + session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) + Trade.session = session() + Trade.query = session.query_property() Base.metadata.create_all(engine) -def get_session(): - return _session - - class Trade(Base): __tablename__ = 'trades' @@ -84,5 +79,8 @@ class Trade(Base): self.close_profit = profit self.close_date = datetime.utcnow() self.open_order_id = order_id + + # Flush changes + Trade.session.flush() return profit From 689cd11a6c435378061f0c275e70b974faa53225 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 21:39:31 +0200 Subject: [PATCH 17/28] use inmemory db for tests --- main.py | 5 +++-- persistence.py | 18 +++++++++--------- test/test_main.py | 6 +----- test/test_telegram.py | 16 ++++++---------- 4 files changed, 19 insertions(+), 26 deletions(-) diff --git a/main.py b/main.py index af6fed2b3..974a20bcc 100755 --- a/main.py +++ b/main.py @@ -230,17 +230,18 @@ def create_trade(stake_amount: float, _exchange: exchange.Exchange) -> Optional[ is_open=True) -def init(config: dict) -> None: +def init(config: dict, db_url: Optional[str]=None) -> None: """ Initializes all modules and updates the config :param config: config as dict + :param db_url: database connector string for sqlalchemy (Optional) :return: None """ global _conf # Initialize all modules telegram.init(config) - persistence.init(config) + persistence.init(config, db_url) exchange.init(config) _conf.update(config) diff --git a/persistence.py b/persistence.py index ce1a1999a..d85a2a27c 100644 --- a/persistence.py +++ b/persistence.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Optional from sqlalchemy import Boolean, Column, DateTime, Float, Integer, String, create_engine from sqlalchemy.ext.declarative import declarative_base @@ -10,29 +11,28 @@ from sqlalchemy.types import Enum import exchange -_db_handle = None _conf = {} - Base = declarative_base() -def init(config: dict) -> None: +def init(config: dict, db_url: Optional[str]=None) -> None: """ Initializes this module with the given config, registers all known command handlers and starts polling for message updates :param config: config to use + :param db_url: database connector string for sqlalchemy (Optional) :return: None """ - global _db_handle _conf.update(config) - if _conf.get('dry_run', False): - _db_handle = 'sqlite:///tradesv2.dry_run.sqlite' - else: - _db_handle = 'sqlite:///tradesv2.sqlite' + if not db_url: + if _conf.get('dry_run', False): + db_url = 'sqlite:///tradesv2.dry_run.sqlite' + else: + db_url = 'sqlite:///tradesv2.sqlite' - engine = create_engine(_db_handle, echo=False) + engine = create_engine(db_url, echo=False) session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() Trade.query = session.query_property() diff --git a/test/test_main.py b/test/test_main.py index 8bd06c316..37aa8d67b 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -53,7 +53,7 @@ class TestMain(unittest.TestCase): 'last': 0.07256061 }), buy=MagicMock(return_value='mocked_order_id')): - init(self.conf) + init(self.conf, 'sqlite://') trade = create_trade(15.0, exchange.Exchange.BITTREX) Trade.session.add(trade) Trade.session.flush() @@ -99,10 +99,6 @@ class TestMain(unittest.TestCase): @classmethod def setUpClass(cls): - try: - os.remove('./tradesv2.dry_run.sqlite') - except FileNotFoundError: - pass validate(cls.conf, conf_schema) diff --git a/test/test_telegram.py b/test/test_telegram.py index fe27fe875..ac96b2e0a 100644 --- a/test/test_telegram.py +++ b/test/test_telegram.py @@ -62,7 +62,7 @@ class TestTelegram(unittest.TestCase): 'last': 0.07256061 }), buy=MagicMock(return_value='mocked_order_id')): - init(self.conf) + init(self.conf, 'sqlite://') # Create some test data trade = create_trade(15.0, exchange.Exchange.BITTREX) @@ -86,7 +86,7 @@ class TestTelegram(unittest.TestCase): 'last': 0.07256061 }), buy=MagicMock(return_value='mocked_order_id')): - init(self.conf) + init(self.conf, 'sqlite://') # Create some test data trade = create_trade(15.0, exchange.Exchange.BITTREX) @@ -115,7 +115,7 @@ class TestTelegram(unittest.TestCase): 'last': 0.07256061 }), buy=MagicMock(return_value='mocked_order_id')): - init(self.conf) + init(self.conf, 'sqlite://') # Create some test data trade = create_trade(15.0, exchange.Exchange.BITTREX) @@ -142,7 +142,7 @@ class TestTelegram(unittest.TestCase): 'last': 0.07256061 }), buy=MagicMock(return_value='mocked_order_id')): - init(self.conf) + init(self.conf, 'sqlite://') # Create some test data trade = create_trade(15.0, exchange.Exchange.BITTREX) @@ -164,7 +164,7 @@ class TestTelegram(unittest.TestCase): with patch.dict('main._conf', self.conf): msg_mock = MagicMock() with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): - init(self.conf) + init(self.conf, 'sqlite://') update_state(State.PAUSED) self.assertEqual(get_state(), State.PAUSED) @@ -176,7 +176,7 @@ class TestTelegram(unittest.TestCase): with patch.dict('main._conf', self.conf): msg_mock = MagicMock() with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): - init(self.conf) + init(self.conf, 'sqlite://') update_state(State.RUNNING) self.assertEqual(get_state(), State.RUNNING) @@ -186,10 +186,6 @@ class TestTelegram(unittest.TestCase): self.assertIn('Stopping trader', msg_mock.call_args_list[0][0][0]) def setUp(self): - try: - os.remove('./tradesv2.dry_run.sqlite') - except FileNotFoundError: - pass self.update = Update(0) self.update.message = Message(0, 0, datetime.utcnow(), Chat(0, 0)) From 996beae7708ee5fa09fb4db844e3039ff50b5cb6 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 8 Sep 2017 23:10:22 +0200 Subject: [PATCH 18/28] pylint fixes --- analyze.py | 1 + exchange.py | 86 +++++++++++++++++++++---------------------- main.py | 69 +++++++++++++++++----------------- misc.py | 2 +- persistence.py | 15 +++++--- requirements.txt | 4 +- rpc/telegram.py | 35 +++++++++++------- test/test_main.py | 11 +++--- test/test_telegram.py | 31 ++++++++-------- 9 files changed, 133 insertions(+), 121 deletions(-) diff --git a/analyze.py b/analyze.py index 10652b615..6ac798757 100644 --- a/analyze.py +++ b/analyze.py @@ -145,6 +145,7 @@ def plot_dataframe(dataframe: DataFrame, pair: str) -> None: if __name__ == '__main__': + # Install PYQT5==5.9 manually if you want to test this helper function while True: pair = 'BTC_ANT' #for pair in ['BTC_ANT', 'BTC_ETH', 'BTC_GNT', 'BTC_ETC']: diff --git a/exchange.py b/exchange.py index f81a9fd7e..e92e5cf7e 100644 --- a/exchange.py +++ b/exchange.py @@ -7,10 +7,10 @@ from poloniex import Poloniex logger = logging.getLogger(__name__) - -cur_exchange = None -_api = None -_conf = {} +# Current selected exchange +EXCHANGE = None +_API = None +_CONF = {} class Exchange(enum.Enum): @@ -26,9 +26,9 @@ def init(config: dict) -> None: :param config: config to use :return: None """ - global _api, cur_exchange + global _API, EXCHANGE - _conf.update(config) + _CONF.update(config) if config['dry_run']: logger.info('Instance is running with dry_run enabled') @@ -37,17 +37,17 @@ def init(config: dict) -> None: use_bittrex = config.get('bittrex', {}).get('enabled', False) if use_poloniex: - cur_exchange = Exchange.POLONIEX - _api = Poloniex(key=config['poloniex']['key'], secret=config['poloniex']['secret']) + EXCHANGE = Exchange.POLONIEX + _API = Poloniex(key=config['poloniex']['key'], secret=config['poloniex']['secret']) elif use_bittrex: - cur_exchange = Exchange.BITTREX - _api = Bittrex(api_key=config['bittrex']['key'], api_secret=config['bittrex']['secret']) + EXCHANGE = Exchange.BITTREX + _API = Bittrex(api_key=config['bittrex']['key'], api_secret=config['bittrex']['secret']) else: raise RuntimeError('No exchange specified. Aborting!') # Check if all pairs are available markets = get_markets() - for pair in config[cur_exchange.name.lower()]['pair_whitelist']: + for pair in config[EXCHANGE.name.lower()]['pair_whitelist']: if pair not in markets: raise RuntimeError('Pair {} is not available at Poloniex'.format(pair)) @@ -60,13 +60,13 @@ def buy(pair: str, rate: float, amount: float) -> str: :param amount: The amount to purchase :return: order_id of the placed buy order """ - if _conf['dry_run']: + if _CONF['dry_run']: return 'dry_run' - elif cur_exchange == Exchange.POLONIEX: - _api.buy(pair, rate, amount) + elif EXCHANGE == Exchange.POLONIEX: + _API.buy(pair, rate, amount) # TODO: return order id - elif cur_exchange == Exchange.BITTREX: - data = _api.buy_limit(pair.replace('_', '-'), amount, rate) + elif EXCHANGE == Exchange.BITTREX: + data = _API.buy_limit(pair.replace('_', '-'), amount, rate) if not data['success']: raise RuntimeError('BITTREX: {}'.format(data['message'])) return data['result']['uuid'] @@ -80,13 +80,13 @@ def sell(pair: str, rate: float, amount: float) -> str: :param amount: The amount to sell :return: None """ - if _conf['dry_run']: + if _CONF['dry_run']: return 'dry_run' - elif cur_exchange == Exchange.POLONIEX: - _api.sell(pair, rate, amount) + elif EXCHANGE == Exchange.POLONIEX: + _API.sell(pair, rate, amount) # TODO: return order id - elif cur_exchange == Exchange.BITTREX: - data = _api.sell_limit(pair.replace('_', '-'), amount, rate) + elif EXCHANGE == Exchange.BITTREX: + data = _API.sell_limit(pair.replace('_', '-'), amount, rate) if not data['success']: raise RuntimeError('BITTREX: {}'.format(data['message'])) return data['result']['uuid'] @@ -98,13 +98,13 @@ def get_balance(currency: str) -> float: :param currency: currency as str, format: BTC :return: float """ - if _conf['dry_run']: + if _CONF['dry_run']: return 999.9 - elif cur_exchange == Exchange.POLONIEX: - data = _api.returnBalances() + elif EXCHANGE == Exchange.POLONIEX: + data = _API.returnBalances() return float(data[currency]) - elif cur_exchange == Exchange.BITTREX: - data = _api.get_balance(currency) + elif EXCHANGE == Exchange.BITTREX: + data = _API.get_balance(currency) if not data['success']: raise RuntimeError('BITTREX: {}'.format(data['message'])) return float(data['result']['Balance'] or 0.0) @@ -116,15 +116,15 @@ def get_ticker(pair: str) -> dict: :param pair: Pair as str, format: BTC_ETC :return: dict """ - if cur_exchange == Exchange.POLONIEX: - data = _api.returnTicker() + if EXCHANGE == Exchange.POLONIEX: + data = _API.returnTicker() return { 'bid': float(data[pair]['highestBid']), 'ask': float(data[pair]['lowestAsk']), 'last': float(data[pair]['last']) } - elif cur_exchange == Exchange.BITTREX: - data = _api.get_ticker(pair.replace('_', '-')) + elif EXCHANGE == Exchange.BITTREX: + data = _API.get_ticker(pair.replace('_', '-')) if not data['success']: raise RuntimeError('BITTREX: {}'.format(data['message'])) return { @@ -140,12 +140,12 @@ def cancel_order(order_id: str) -> None: :param order_id: id as str :return: None """ - if _conf['dry_run']: + if _CONF['dry_run']: pass - elif cur_exchange == Exchange.POLONIEX: + elif EXCHANGE == Exchange.POLONIEX: raise NotImplemented('Not implemented') - elif cur_exchange == Exchange.BITTREX: - data = _api.cancel(order_id) + elif EXCHANGE == Exchange.BITTREX: + data = _API.cancel(order_id) if not data['success']: raise RuntimeError('BITTREX: {}'.format(data['message'])) @@ -156,12 +156,12 @@ def get_open_orders(pair: str) -> List[dict]: :param pair: Pair as str, format: BTC_ETC :return: list of dicts """ - if _conf['dry_run']: + if _CONF['dry_run']: return [] - elif cur_exchange == Exchange.POLONIEX: + elif EXCHANGE == Exchange.POLONIEX: raise NotImplemented('Not implemented') - elif cur_exchange == Exchange.BITTREX: - data = _api.get_open_orders(pair.replace('_', '-')) + elif EXCHANGE == Exchange.BITTREX: + data = _API.get_open_orders(pair.replace('_', '-')) if not data['success']: raise RuntimeError('BITTREX: {}'.format(data['message'])) return [{ @@ -180,9 +180,9 @@ def get_pair_detail_url(pair: str) -> str: :param pair: pair as str, format: BTC_ANT :return: url as str """ - if cur_exchange == Exchange.POLONIEX: + if EXCHANGE == Exchange.POLONIEX: raise NotImplemented('Not implemented') - elif cur_exchange == Exchange.BITTREX: + elif EXCHANGE == Exchange.BITTREX: return 'https://bittrex.com/Market/Index?MarketName={}'.format(pair.replace('_', '-')) @@ -191,11 +191,11 @@ def get_markets() -> List[str]: Returns all available markets :return: list of all available pairs """ - if cur_exchange == Exchange.POLONIEX: + if EXCHANGE == Exchange.POLONIEX: # TODO: implement raise NotImplemented('Not implemented') - elif cur_exchange == Exchange. BITTREX: - data = _api.get_markets() + elif EXCHANGE == Exchange. BITTREX: + data = _API.get_markets() if not data['success']: raise RuntimeError('BITTREX: {}'.format(data['message'])) return [m['MarketName'].replace('-', '_') for m in data['result']] diff --git a/main.py b/main.py index 974a20bcc..6913fcb97 100755 --- a/main.py +++ b/main.py @@ -5,19 +5,17 @@ import logging import time import traceback from datetime import datetime -from json import JSONDecodeError from typing import Optional from jsonschema import validate -from requests import ConnectionError from wrapt import synchronized import exchange import persistence -from rpc import telegram -from analyze import get_buy_signal from persistence import Trade -from misc import conf_schema +from analyze import get_buy_signal +from misc import CONF_SCHEMA +from rpc import telegram logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') @@ -35,8 +33,10 @@ class State(enum.Enum): TERMINATE = 2 -_conf = {} -_cur_state = State.RUNNING +_CONF = {} + +# Current application state +_STATE = State.RUNNING @synchronized @@ -46,8 +46,8 @@ def update_state(state: State) -> None: :param state: new state :return: None """ - global _cur_state - _cur_state = state + global _STATE + _STATE = state @synchronized @@ -56,7 +56,7 @@ def get_state() -> State: Gets the current application state :return: """ - return _cur_state + return _STATE def _process() -> None: @@ -67,10 +67,10 @@ def _process() -> None: """ # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if len(trades) < _conf['max_open_trades']: + if len(trades) < _CONF['max_open_trades']: try: # Create entity and execute trade - trade = create_trade(float(_conf['stake_amount']), exchange.cur_exchange) + trade = create_trade(float(_CONF['stake_amount']), exchange.EXCHANGE) if trade: Trade.session.add(trade) else: @@ -80,14 +80,17 @@ def _process() -> None: for trade in trades: if close_trade_if_fulfilled(trade): - logger.info('No open orders found and trade is fulfilled. Marking %s as closed ...', trade) + logger.info( + 'No open orders found and trade is fulfilled. Marking %s as closed ...', + trade + ) for trade in filter(lambda t: t.is_open, trades): # Check if there is already an open order for this trade orders = exchange.get_open_orders(trade.pair) orders = [o for o in orders if o['id'] == trade.open_order_id] if orders: - msg = 'There exists an open order for {}: Order(total={}, remaining={}, type={}, id={})' \ + msg = 'There is an open order for {}: Order(total={}, remaining={}, type={}, id={})' \ .format( trade, round(orders[0]['amount'], 8), @@ -156,20 +159,20 @@ def handle_trade(trade: Trade) -> None: current_rate = exchange.get_ticker(trade.pair)['bid'] current_profit = 100.0 * ((current_rate - trade.open_rate) / trade.open_rate) - if 'stoploss' in _conf and current_profit < float(_conf['stoploss']) * 100.0: + if 'stoploss' in _CONF and current_profit < float(_CONF['stoploss']) * 100.0: logger.debug('Stop loss hit.') execute_sell(trade, current_rate) return - for duration, threshold in sorted(_conf['minimal_roi'].items()): + for duration, threshold in sorted(_CONF['minimal_roi'].items()): duration, threshold = float(duration), float(threshold) # Check if time matches and current rate is above threshold time_diff = (datetime.utcnow() - trade.open_date).total_seconds() / 60 if time_diff > duration and current_rate > (1 + threshold) * trade.open_rate: execute_sell(trade, current_rate) return - else: - logger.debug('Threshold not reached. (cur_profit: %1.2f%%)', current_profit) + + logger.debug('Threshold not reached. (cur_profit: %1.2f%%)', current_profit) except ValueError: logger.exception('Unable to handle open order') @@ -182,10 +185,12 @@ def create_trade(stake_amount: float, _exchange: exchange.Exchange) -> Optional[ :param _exchange: exchange to use """ logger.info('Creating new trade with stake_amount: %f ...', stake_amount) - whitelist = _conf[_exchange.name.lower()]['pair_whitelist'] + whitelist = _CONF[_exchange.name.lower()]['pair_whitelist'] # Check if btc_amount is fulfilled - if exchange.get_balance(_conf['stake_currency']) < stake_amount: - raise ValueError('stake amount is not fulfilled (currency={}'.format(_conf['stake_currency'])) + if exchange.get_balance(_CONF['stake_currency']) < stake_amount: + raise ValueError( + 'stake amount is not fulfilled (currency={}'.format(_CONF['stake_currency']) + ) # Remove currently opened and latest pairs from whitelist trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -200,9 +205,9 @@ def create_trade(stake_amount: float, _exchange: exchange.Exchange) -> Optional[ raise ValueError('No pair in whitelist') # Pick pair based on StochRSI buy signals - for p in whitelist: - if get_buy_signal(p): - pair = p + for _pair in whitelist: + if get_buy_signal(_pair): + pair = _pair break else: return None @@ -230,20 +235,17 @@ def create_trade(stake_amount: float, _exchange: exchange.Exchange) -> Optional[ is_open=True) -def init(config: dict, db_url: Optional[str]=None) -> None: +def init(config: dict, db_url: Optional[str] = None) -> None: """ Initializes all modules and updates the config :param config: config as dict :param db_url: database connector string for sqlalchemy (Optional) :return: None """ - global _conf - # Initialize all modules telegram.init(config) persistence.init(config, db_url) exchange.init(config) - _conf.update(config) def app(config: dict) -> None: @@ -264,12 +266,12 @@ def app(config: dict) -> None: try: _process() Trade.session.flush() - except (ConnectionError, JSONDecodeError, ValueError) as error: + except (ConnectionError, json.JSONDecodeError, ValueError) as error: msg = 'Got {} during _process()'.format(error.__class__.__name__) logger.exception(msg) finally: time.sleep(25) - except (RuntimeError, JSONDecodeError): + except (RuntimeError, json.JSONDecodeError): telegram.send_msg( '*Status:* Got RuntimeError: ```\n{}\n```'.format(traceback.format_exc()) ) @@ -280,7 +282,6 @@ def app(config: dict) -> None: if __name__ == '__main__': with open('config.json') as file: - conf = json.load(file) - validate(conf, conf_schema) - app(conf) - + _CONF = json.load(file) + validate(_CONF, CONF_SCHEMA) + app(_CONF) diff --git a/misc.py b/misc.py index 4fd92d1d4..0564d0ba0 100644 --- a/misc.py +++ b/misc.py @@ -1,6 +1,6 @@ # Required json-schema for user specified config -conf_schema = { +CONF_SCHEMA = { 'type': 'object', 'properties': { 'max_open_trades': {'type': 'integer', 'minimum': 1}, diff --git a/persistence.py b/persistence.py index d85a2a27c..8ba8079f9 100644 --- a/persistence.py +++ b/persistence.py @@ -11,12 +11,12 @@ from sqlalchemy.types import Enum import exchange -_conf = {} +_CONF = {} Base = declarative_base() -def init(config: dict, db_url: Optional[str]=None) -> None: +def init(config: dict, db_url: Optional[str] = None) -> None: """ Initializes this module with the given config, registers all known command handlers @@ -25,9 +25,9 @@ def init(config: dict, db_url: Optional[str]=None) -> None: :param db_url: database connector string for sqlalchemy (Optional) :return: None """ - _conf.update(config) + _CONF.update(config) if not db_url: - if _conf.get('dry_run', False): + if _CONF.get('dry_run', False): db_url = 'sqlite:///tradesv2.dry_run.sqlite' else: db_url = 'sqlite:///tradesv2.sqlite' @@ -56,12 +56,16 @@ class Trade(Base): open_order_id = Column(String) def __repr__(self): + if self.is_open: + open_since = 'closed' + else: + open_since = round((datetime.utcnow() - self.open_date).total_seconds() / 60, 2) return 'Trade(id={}, pair={}, amount={}, open_rate={}, open_since={})'.format( self.id, self.pair, self.amount, self.open_rate, - 'closed' if not self.is_open else round((datetime.utcnow() - self.open_date).total_seconds() / 60, 2) + open_since ) def exec_sell_order(self, rate: float, amount: float) -> float: @@ -83,4 +87,3 @@ class Trade(Base): # Flush changes Trade.session.flush() return profit - diff --git a/requirements.txt b/requirements.txt index fcd6c3954..7ee7236ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,8 +8,8 @@ urllib3==1.22 wrapt==1.10.11 pandas==0.20.3 matplotlib==2.0.2 -PYQT5==5.9 scikit-learn==0.19.0 scipy==0.19.1 jsonschema==2.6.0 -TA-Lib==0.4.10 \ No newline at end of file +TA-Lib==0.4.10 +#PYQT5==5.9 \ No newline at end of file diff --git a/rpc/telegram.py b/rpc/telegram.py index 96adaa739..cd046b749 100644 --- a/rpc/telegram.py +++ b/rpc/telegram.py @@ -18,7 +18,7 @@ logging.getLogger('telegram').setLevel(logging.INFO) logger = logging.getLogger(__name__) _updater = None -_conf = {} +_CONF = {} def init(config: dict) -> None: @@ -32,7 +32,7 @@ def init(config: dict) -> None: global _updater _updater = Updater(token=config['telegram']['token'], workers=0) - _conf.update(config) + _CONF.update(config) # Register command handler and start telegram message polling handles = [ @@ -51,8 +51,10 @@ def init(config: dict) -> None: timeout=30, read_latency=60, ) - logger.info('rpc.telegram is listening for following commands: {}' - .format([h.command for h in handles])) + logger.info( + 'rpc.telegram is listening for following commands: %s', + [h.command for h in handles] + ) def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[..., Any]: @@ -62,12 +64,12 @@ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[ :return: decorated function """ def wrapper(*args, **kwargs): - bot, update = (args[0], args[1]) if args else (kwargs['bot'], kwargs['update']) + bot, update = kwargs.get('bot') or args[0], kwargs.get('update') or args[1] if not isinstance(bot, Bot) or not isinstance(update, Update): raise ValueError('Received invalid Arguments: {}'.format(*args)) - chat_id = int(_conf['telegram']['chat_id']) + chat_id = int(_CONF['telegram']['chat_id']) if int(update.message.chat_id) == chat_id: logger.info('Executing handler: %s for chat_id: %s', command_handler.__name__, chat_id) return command_handler(*args, **kwargs) @@ -88,7 +90,7 @@ def _status(bot: Bot, update: Update) -> None: # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() from main import get_state, State - if not get_state() == State.RUNNING: + if get_state() != State.RUNNING: send_msg('*Status:* `trader is not running`', bot=bot) elif not trades: send_msg('*Status:* `no active order`', bot=bot) @@ -100,6 +102,10 @@ def _status(bot: Bot, update: Update) -> None: orders = exchange.get_open_orders(trade.pair) orders = [o for o in orders if o['id'] == trade.open_order_id] order = orders[0] if orders else None + + fmt_close_profit = '{:.2f}%'.format( + round(trade.close_profit, 2) + ) if trade.close_profit else None message = """ *Trade ID:* `{trade_id}` *Current Pair:* [{pair}]({market_url}) @@ -120,7 +126,7 @@ def _status(bot: Bot, update: Update) -> None: close_rate=trade.close_rate, current_rate=current_rate, amount=round(trade.amount, 8), - close_profit='{}%'.format(round(trade.close_profit, 2)) if trade.close_profit else None, + close_profit=fmt_close_profit, current_profit=round(current_profit, 2), open_order='{} ({})'.format(order['remaining'], order['type']) if order else None, ) @@ -297,7 +303,7 @@ def _performance(bot: Bot, update: Update) -> None: send_msg(message, parse_mode=ParseMode.HTML) -def send_msg(msg: str, bot: Bot=None, parse_mode: ParseMode=ParseMode.MARKDOWN) -> None: +def send_msg(msg: str, bot: Bot = None, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: """ Send given markdown message :param msg: message @@ -305,15 +311,18 @@ def send_msg(msg: str, bot: Bot=None, parse_mode: ParseMode=ParseMode.MARKDOWN) :param parse_mode: telegram parse mode :return: None """ - if _conf['telegram'].get('enabled', False): + if _CONF['telegram'].get('enabled', False): try: bot = bot or _updater.bot try: - bot.send_message(_conf['telegram']['chat_id'], msg, parse_mode=parse_mode) + bot.send_message(_CONF['telegram']['chat_id'], msg, parse_mode=parse_mode) except NetworkError as error: # Sometimes the telegram server resets the current connection, # if this is the case we send the message again. - logger.warning('Got Telegram NetworkError: %s! Trying one more time.', error.message) - bot.send_message(_conf['telegram']['chat_id'], msg, parse_mode=parse_mode) + logger.warning( + 'Got Telegram NetworkError: %s! Trying one more time.', + error.message + ) + bot.send_message(_CONF['telegram']['chat_id'], msg, parse_mode=parse_mode) except Exception: logger.exception('Exception occurred within Telegram API') diff --git a/test/test_main.py b/test/test_main.py index 37aa8d67b..12b9db5ec 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,12 +1,11 @@ import unittest from unittest.mock import patch, MagicMock -import os from jsonschema import validate import exchange from main import create_trade, handle_trade, close_trade_if_fulfilled, init -from misc import conf_schema +from misc import CONF_SCHEMA from persistence import Trade @@ -43,7 +42,7 @@ class TestMain(unittest.TestCase): } def test_1_create_trade(self): - with patch.dict('main._conf', self.conf): + with patch.dict('main._CONF', self.conf): with patch('main.get_buy_signal', side_effect=lambda _: True) as buy_signal: with patch.multiple('main.telegram', init=MagicMock(), send_msg=MagicMock()): with patch.multiple('main.exchange', @@ -68,7 +67,7 @@ class TestMain(unittest.TestCase): buy_signal.assert_called_once_with('BTC_ETH') def test_2_handle_trade(self): - with patch.dict('main._conf', self.conf): + with patch.dict('main._CONF', self.conf): with patch.multiple('main.telegram', init=MagicMock(), send_msg=MagicMock()): with patch.multiple('main.exchange', get_ticker=MagicMock(return_value={ @@ -86,7 +85,7 @@ class TestMain(unittest.TestCase): self.assertEqual(trade.open_order_id, 'dry_run') def test_3_close_trade(self): - with patch.dict('main._conf', self.conf): + with patch.dict('main._CONF', self.conf): trade = Trade.query.filter(Trade.is_open.is_(True)).first() self.assertTrue(trade) @@ -99,7 +98,7 @@ class TestMain(unittest.TestCase): @classmethod def setUpClass(cls): - validate(cls.conf, conf_schema) + validate(cls.conf, CONF_SCHEMA) if __name__ == '__main__': diff --git a/test/test_telegram.py b/test/test_telegram.py index ac96b2e0a..de6e3ed74 100644 --- a/test/test_telegram.py +++ b/test/test_telegram.py @@ -1,14 +1,13 @@ import unittest -from datetime import datetime from unittest.mock import patch, MagicMock +from datetime import datetime -import os from jsonschema import validate from telegram import Bot, Update, Message, Chat import exchange from main import init, create_trade, update_state, State, get_state -from misc import conf_schema +from misc import CONF_SCHEMA from persistence import Trade from rpc.telegram import _status, _profit, _forcesell, _performance, _start, _stop @@ -51,10 +50,10 @@ class TestTelegram(unittest.TestCase): } def test_1_status_handle(self): - with patch.dict('main._conf', self.conf): + with patch.dict('main._CONF', self.conf): with patch('main.get_buy_signal', side_effect=lambda _: True): msg_mock = MagicMock() - with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + with patch.multiple('main.telegram', _CONF=self.conf, init=MagicMock(), send_msg=msg_mock): with patch.multiple('main.exchange', get_ticker=MagicMock(return_value={ 'bid': 0.07256061, @@ -75,10 +74,10 @@ class TestTelegram(unittest.TestCase): self.assertIn('[BTC_ETH]', msg_mock.call_args_list[-1][0][0]) def test_2_profit_handle(self): - with patch.dict('main._conf', self.conf): + with patch.dict('main._CONF', self.conf): with patch('main.get_buy_signal', side_effect=lambda _: True): msg_mock = MagicMock() - with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + with patch.multiple('main.telegram', _CONF=self.conf, init=MagicMock(), send_msg=msg_mock): with patch.multiple('main.exchange', get_ticker=MagicMock(return_value={ 'bid': 0.07256061, @@ -104,10 +103,10 @@ class TestTelegram(unittest.TestCase): self.assertIn('(100.00%)', msg_mock.call_args_list[-1][0][0]) def test_3_forcesell_handle(self): - with patch.dict('main._conf', self.conf): + with patch.dict('main._CONF', self.conf): with patch('main.get_buy_signal', side_effect=lambda _: True): msg_mock = MagicMock() - with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + with patch.multiple('main.telegram', _CONF=self.conf, init=MagicMock(), send_msg=msg_mock): with patch.multiple('main.exchange', get_ticker=MagicMock(return_value={ 'bid': 0.07256061, @@ -131,10 +130,10 @@ class TestTelegram(unittest.TestCase): self.assertIn('0.072561', msg_mock.call_args_list[-1][0][0]) def test_4_performance_handle(self): - with patch.dict('main._conf', self.conf): + with patch.dict('main._CONF', self.conf): with patch('main.get_buy_signal', side_effect=lambda _: True): msg_mock = MagicMock() - with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + with patch.multiple('main.telegram', _CONF=self.conf, init=MagicMock(), send_msg=msg_mock): with patch.multiple('main.exchange', get_ticker=MagicMock(return_value={ 'bid': 0.07256061, @@ -161,9 +160,9 @@ class TestTelegram(unittest.TestCase): self.assertIn('BTC_ETH 100.00%', msg_mock.call_args_list[-1][0][0]) def test_5_start_handle(self): - with patch.dict('main._conf', self.conf): + with patch.dict('main._CONF', self.conf): msg_mock = MagicMock() - with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + with patch.multiple('main.telegram', _CONF=self.conf, init=MagicMock(), send_msg=msg_mock): init(self.conf, 'sqlite://') update_state(State.PAUSED) @@ -173,9 +172,9 @@ class TestTelegram(unittest.TestCase): self.assertEqual(msg_mock.call_count, 0) def test_6_stop_handle(self): - with patch.dict('main._conf', self.conf): + with patch.dict('main._CONF', self.conf): msg_mock = MagicMock() - with patch.multiple('main.telegram', _conf=self.conf, init=MagicMock(), send_msg=msg_mock): + with patch.multiple('main.telegram', _CONF=self.conf, init=MagicMock(), send_msg=msg_mock): init(self.conf, 'sqlite://') update_state(State.RUNNING) @@ -191,7 +190,7 @@ class TestTelegram(unittest.TestCase): @classmethod def setUpClass(cls): - validate(cls.conf, conf_schema) + validate(cls.conf, CONF_SCHEMA) if __name__ == '__main__': From a4b2f4e4b9aabe10b58a13ac7fe25deb81f37e2c Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 9 Sep 2017 00:31:40 +0200 Subject: [PATCH 19/28] Fix application state and add new optional config attribute: "initial_state" * Move State handling to misc, to avoid circular imports * Add optional config attribute "initial_state" --- README.md | 6 +- config.json.example | 3 +- main.py | 159 ++++++++++++++++++------------------------ misc.py | 36 +++++++++- rpc/telegram.py | 8 +-- test/test_telegram.py | 13 ++-- 6 files changed, 118 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index d3d3ae36c..1569eabea 100644 --- a/README.md +++ b/README.md @@ -40,10 +40,14 @@ See the example below: For example value `-0.10` will cause immediate sell if the profit dips below -10% for a given trade. This parameter is optional. +`initial_state` is an optional field that defines the initial application state. +Possible values are `running` or `stopped`. (default=`running`) +If the value is `stopped` the bot has to be started with `/start` first. + The other values should be self-explanatory, if not feel free to raise a github issue. -##### Prerequisites +#### Prerequisites * python3.6 * sqlite * [TA-lib](https://github.com/mrjbq7/ta-lib#dependencies) binaries diff --git a/config.json.example b/config.json.example index a9fc3be79..227d4091e 100644 --- a/config.json.example +++ b/config.json.example @@ -35,5 +35,6 @@ "enabled": true, "token": "token", "chat_id": "chat_id" - } + }, + "initial_state": "running" } \ No newline at end of file diff --git a/main.py b/main.py index 6913fcb97..05126a8d7 100755 --- a/main.py +++ b/main.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -import enum import json import logging import time @@ -8,13 +7,12 @@ from datetime import datetime from typing import Optional from jsonschema import validate -from wrapt import synchronized import exchange import persistence from persistence import Trade from analyze import get_buy_signal -from misc import CONF_SCHEMA +from misc import CONF_SCHEMA, get_state, State, update_state from rpc import telegram logging.basicConfig(level=logging.DEBUG, @@ -26,38 +24,8 @@ __copyright__ = "gcarq 2017" __license__ = "GPLv3" __version__ = "0.8.0" - -class State(enum.Enum): - RUNNING = 0 - PAUSED = 1 - TERMINATE = 2 - - _CONF = {} -# Current application state -_STATE = State.RUNNING - - -@synchronized -def update_state(state: State) -> None: - """ - Updates the application state - :param state: new state - :return: None - """ - global _STATE - _STATE = state - - -@synchronized -def get_state() -> State: - """ - Gets the current application state - :return: - """ - return _STATE - def _process() -> None: """ @@ -65,44 +33,43 @@ def _process() -> None: otherwise a new trade is created. :return: None """ - # Query trades from persistence layer - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if len(trades) < _CONF['max_open_trades']: - try: - # Create entity and execute trade - trade = create_trade(float(_CONF['stake_amount']), exchange.EXCHANGE) - if trade: - Trade.session.add(trade) + try: + # Query trades from persistence layer + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + if len(trades) < _CONF['max_open_trades']: + try: + # Create entity and execute trade + trade = create_trade(float(_CONF['stake_amount']), exchange.EXCHANGE) + if trade: + Trade.session.add(trade) + else: + logging.info('Got no buy signal...') + except ValueError: + logger.exception('Unable to create trade') + + for trade in trades: + if close_trade_if_fulfilled(trade): + logger.info( + 'No open orders found and trade is fulfilled. Marking %s as closed ...', + trade + ) + Trade.session.flush() + + for trade in filter(lambda t: t.is_open, trades): + # Check if there is already an open order for this trade + orders = exchange.get_open_orders(trade.pair) + orders = [o for o in orders if o['id'] == trade.open_order_id] + if orders: + logger.info('There is an open order for: %s', orders[0]) else: - logging.info('Got no buy signal...') - except ValueError: - logger.exception('Unable to create trade') - - for trade in trades: - if close_trade_if_fulfilled(trade): - logger.info( - 'No open orders found and trade is fulfilled. Marking %s as closed ...', - trade - ) - - for trade in filter(lambda t: t.is_open, trades): - # Check if there is already an open order for this trade - orders = exchange.get_open_orders(trade.pair) - orders = [o for o in orders if o['id'] == trade.open_order_id] - if orders: - msg = 'There is an open order for {}: Order(total={}, remaining={}, type={}, id={})' \ - .format( - trade, - round(orders[0]['amount'], 8), - round(orders[0]['remaining'], 8), - orders[0]['type'], - orders[0]['id']) - logger.info(msg) - else: - # Update state - trade.open_order_id = None - # Check if we can sell our current pair - handle_trade(trade) + # Update state + trade.open_order_id = None + # Check if we can sell our current pair + handle_trade(trade) + Trade.session.flush() + except (ConnectionError, json.JSONDecodeError) as error: + msg = 'Got {} in _process()'.format(error.__class__.__name__) + logger.exception(msg) def close_trade_if_fulfilled(trade: Trade) -> bool: @@ -247,35 +214,43 @@ def init(config: dict, db_url: Optional[str] = None) -> None: persistence.init(config, db_url) exchange.init(config) + # Set initial application state + initial_state = config.get('initial_state') + if initial_state: + update_state(State[initial_state.upper()]) + else: + update_state(State.STOPPED) + def app(config: dict) -> None: - + """ + Main function which handles the application state + :param config: config as dict + :return: None + """ logger.info('Starting freqtrade %s', __version__) init(config) - try: - telegram.send_msg('*Status:* `trader started`') - logger.info('Trader started') + old_state = get_state() + logger.info('Initial State: %s', old_state) + telegram.send_msg('*Status:* `{}`'.format(old_state.name.lower())) while True: - state = get_state() - if state == State.TERMINATE: - return - elif state == State.PAUSED: + new_state = get_state() + # Log state transition + if new_state != old_state: + telegram.send_msg('*Status:* `{}`'.format(new_state.name.lower())) + logging.info('Changing state to: %s', new_state.name) + + if new_state == State.STOPPED: time.sleep(1) - elif state == State.RUNNING: - try: - _process() - Trade.session.flush() - except (ConnectionError, json.JSONDecodeError, ValueError) as error: - msg = 'Got {} during _process()'.format(error.__class__.__name__) - logger.exception(msg) - finally: - time.sleep(25) - except (RuntimeError, json.JSONDecodeError): - telegram.send_msg( - '*Status:* Got RuntimeError: ```\n{}\n```'.format(traceback.format_exc()) - ) - logger.exception('RuntimeError. Stopping trader ...') + elif new_state == State.RUNNING: + _process() + # We need to sleep here because otherwise we would run into bittrex rate limit + time.sleep(25) + old_state = new_state + except RuntimeError: + telegram.send_msg('*Status:* Got RuntimeError: ```\n{}\n```'.format(traceback.format_exc())) + logger.exception('RuntimeError. Trader stopped!') finally: telegram.send_msg('*Status:* `Trader has stopped`') diff --git a/misc.py b/misc.py index 0564d0ba0..2d653651d 100644 --- a/misc.py +++ b/misc.py @@ -1,3 +1,36 @@ +import enum + +from wrapt import synchronized + + +class State(enum.Enum): + RUNNING = 0 + STOPPED = 1 + + +# Current application state +_STATE = State.STOPPED + + +@synchronized +def update_state(state: State) -> None: + """ + Updates the application state + :param state: new state + :return: None + """ + global _STATE + _STATE = state + + +@synchronized +def get_state() -> State: + """ + Gets the current application state + :return: + """ + return _STATE + # Required json-schema for user specified config CONF_SCHEMA = { @@ -25,7 +58,8 @@ CONF_SCHEMA = { 'chat_id': {'type': 'string'}, }, 'required': ['enabled', 'token', 'chat_id'] - } + }, + 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, }, 'definitions': { 'exchange': { diff --git a/rpc/telegram.py b/rpc/telegram.py index cd046b749..80d557406 100644 --- a/rpc/telegram.py +++ b/rpc/telegram.py @@ -8,6 +8,7 @@ from telegram.error import NetworkError from telegram.ext import CommandHandler, Updater from telegram import ParseMode, Bot, Update +from misc import get_state, State, update_state from persistence import Trade import exchange @@ -89,7 +90,6 @@ def _status(bot: Bot, update: Update) -> None: """ # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() - from main import get_state, State if get_state() != State.RUNNING: send_msg('*Status:* `trader is not running`', bot=bot) elif not trades: @@ -200,7 +200,6 @@ def _start(bot: Bot, update: Update) -> None: :param update: message update :return: None """ - from main import get_state, State, update_state if get_state() == State.RUNNING: send_msg('*Status:* `already running`', bot=bot) else: @@ -216,10 +215,9 @@ def _stop(bot: Bot, update: Update) -> None: :param update: message update :return: None """ - from main import get_state, State, update_state if get_state() == State.RUNNING: send_msg('`Stopping trader ...`', bot=bot) - update_state(State.PAUSED) + update_state(State.STOPPED) else: send_msg('*Status:* `already stopped`', bot=bot) @@ -233,7 +231,6 @@ def _forcesell(bot: Bot, update: Update) -> None: :param update: message update :return: None """ - from main import get_state, State if get_state() != State.RUNNING: send_msg('`trader is not running`', bot=bot) return @@ -281,7 +278,6 @@ def _performance(bot: Bot, update: Update) -> None: :param update: message update :return: None """ - from main import get_state, State if get_state() != State.RUNNING: send_msg('`trader is not running`', bot=bot) return diff --git a/test/test_telegram.py b/test/test_telegram.py index de6e3ed74..55b9c73f2 100644 --- a/test/test_telegram.py +++ b/test/test_telegram.py @@ -6,8 +6,8 @@ from jsonschema import validate from telegram import Bot, Update, Message, Chat import exchange -from main import init, create_trade, update_state, State, get_state -from misc import CONF_SCHEMA +from main import init, create_trade +from misc import CONF_SCHEMA, update_state, State, get_state from persistence import Trade from rpc.telegram import _status, _profit, _forcesell, _performance, _start, _stop @@ -46,7 +46,8 @@ class TestTelegram(unittest.TestCase): "enabled": True, "token": "token", "chat_id": "0" - } + }, + "initial_state": "running" } def test_1_status_handle(self): @@ -165,8 +166,8 @@ class TestTelegram(unittest.TestCase): with patch.multiple('main.telegram', _CONF=self.conf, init=MagicMock(), send_msg=msg_mock): init(self.conf, 'sqlite://') - update_state(State.PAUSED) - self.assertEqual(get_state(), State.PAUSED) + update_state(State.STOPPED) + self.assertEqual(get_state(), State.STOPPED) _start(bot=MagicBot(), update=self.update) self.assertEqual(get_state(), State.RUNNING) self.assertEqual(msg_mock.call_count, 0) @@ -180,7 +181,7 @@ class TestTelegram(unittest.TestCase): update_state(State.RUNNING) self.assertEqual(get_state(), State.RUNNING) _stop(bot=MagicBot(), update=self.update) - self.assertEqual(get_state(), State.PAUSED) + self.assertEqual(get_state(), State.STOPPED) self.assertEqual(msg_mock.call_count, 1) self.assertIn('Stopping trader', msg_mock.call_args_list[0][0][0]) From 52bda0f8861facaafe5aff5908f83e8051abadb8 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 9 Sep 2017 15:30:08 +0200 Subject: [PATCH 20/28] restructure _process to eliminate race conditions --- main.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index 05126a8d7..6722d0b0a 100755 --- a/main.py +++ b/main.py @@ -48,14 +48,6 @@ def _process() -> None: logger.exception('Unable to create trade') for trade in trades: - if close_trade_if_fulfilled(trade): - logger.info( - 'No open orders found and trade is fulfilled. Marking %s as closed ...', - trade - ) - Trade.session.flush() - - for trade in filter(lambda t: t.is_open, trades): # Check if there is already an open order for this trade orders = exchange.get_open_orders(trade.pair) orders = [o for o in orders if o['id'] == trade.open_order_id] @@ -64,9 +56,11 @@ def _process() -> None: else: # Update state trade.open_order_id = None - # Check if we can sell our current pair - handle_trade(trade) - Trade.session.flush() + # Check if this trade can be closed + if not close_trade_if_fulfilled(trade): + # Check if we can sell our current pair + handle_trade(trade) + Trade.session.flush() except (ConnectionError, json.JSONDecodeError) as error: msg = 'Got {} in _process()'.format(error.__class__.__name__) logger.exception(msg) @@ -85,6 +79,7 @@ def close_trade_if_fulfilled(trade: Trade) -> bool: and trade.close_rate is not None \ and trade.open_order_id is None: trade.is_open = False + logger.info('No open orders found and trade is fulfilled. Marking %s as closed ...', trade) return True return False From 00400d906df7f85ee573094319a0c02e1363ffe0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 9 Sep 2017 12:26:33 +0300 Subject: [PATCH 21/28] separate calling ticker api from parsing the result --- analyze.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/analyze.py b/analyze.py index 6ac798757..23095cc6b 100644 --- a/analyze.py +++ b/analyze.py @@ -13,13 +13,10 @@ logging.basicConfig(level=logging.DEBUG, logger = logging.getLogger(__name__) -def get_ticker_dataframe(pair: str) -> DataFrame: +def get_ticker(pair: str, minimum_date: arrow.Arrow) -> dict: """ - Analyses the trend for the given pair - :param pair: pair as str in format BTC_ETH or BTC-ETH - :return: DataFrame + Request ticker data from Bittrex for a given currency pair """ - minimum_date = arrow.now() - timedelta(hours=6) url = 'https://bittrex.com/Api/v2.0/pub/market/GetTicks' headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', @@ -32,6 +29,15 @@ def get_ticker_dataframe(pair: str) -> DataFrame: data = requests.get(url, params=params, headers=headers).json() if not data['success']: raise RuntimeError('BITTREX: {}'.format(data['message'])) + return data + + +def parse_ticker_dataframe(ticker: list, minimum_date: arrow.Arrow) -> DataFrame: + """ + Analyses the trend for the given pair + :param pair: pair as str in format BTC_ETH or BTC-ETH + :return: DataFrame + """ data = [{ 'close': t['C'], @@ -40,7 +46,7 @@ def get_ticker_dataframe(pair: str) -> DataFrame: 'high': t['H'], 'low': t['L'], 'date': t['T'], - } for t in sorted(data['result'], key=lambda k: k['T']) if arrow.get(t['T']) > minimum_date] + } for t in sorted(ticker, key=lambda k: k['T']) if arrow.get(t['T']) > minimum_date] dataframe = DataFrame(json_normalize(data)) dataframe['close_30_ema'] = ta.EMA(dataframe, timeperiod=30) @@ -89,7 +95,9 @@ def get_buy_signal(pair: str) -> bool: :param pair: pair in format BTC_ANT or BTC-ANT :return: True if pair is underpriced, False otherwise """ - dataframe = get_ticker_dataframe(pair) + minimum_date = arrow.now() - timedelta(hours=6) + data = get_ticker(pair, minimum_date) + dataframe = parse_ticker_dataframe(data['result'], minimum_date) dataframe = populate_trends(dataframe) latest = dataframe.iloc[-1] @@ -150,7 +158,9 @@ if __name__ == '__main__': pair = 'BTC_ANT' #for pair in ['BTC_ANT', 'BTC_ETH', 'BTC_GNT', 'BTC_ETC']: # get_buy_signal(pair) - dataframe = get_ticker_dataframe(pair) + minimum_date = arrow.now() - timedelta(hours=6) + data = get_ticker(pair, minimum_date) + dataframe = parse_ticker_dataframe(data['result'], minimum_date) dataframe = populate_trends(dataframe) plot_dataframe(dataframe, pair) time.sleep(60) From 8a736ba38dcf89eab8b7cd5b450a7955a1f30982 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 9 Sep 2017 13:02:47 +0300 Subject: [PATCH 22/28] separate calculating indicators from parsing the data --- analyze.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/analyze.py b/analyze.py index 23095cc6b..e4c1903af 100644 --- a/analyze.py +++ b/analyze.py @@ -38,7 +38,6 @@ def parse_ticker_dataframe(ticker: list, minimum_date: arrow.Arrow) -> DataFrame :param pair: pair as str in format BTC_ETH or BTC-ETH :return: DataFrame """ - data = [{ 'close': t['C'], 'volume': t['V'], @@ -47,8 +46,14 @@ def parse_ticker_dataframe(ticker: list, minimum_date: arrow.Arrow) -> DataFrame 'low': t['L'], 'date': t['T'], } for t in sorted(ticker, key=lambda k: k['T']) if arrow.get(t['T']) > minimum_date] - dataframe = DataFrame(json_normalize(data)) + return DataFrame(json_normalize(data)) + + +def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + """ dataframe['close_30_ema'] = ta.EMA(dataframe, timeperiod=30) dataframe['close_90_ema'] = ta.EMA(dataframe, timeperiod=90) @@ -81,7 +86,7 @@ def populate_trends(dataframe: DataFrame) -> DataFrame: """ dataframe.loc[ (dataframe['stochrsi'] < 20) - & (dataframe['macd'] > dataframe['macds']) + & (dataframe['macd'] > dataframe['macds']) & (dataframe['close'] > dataframe['sar']), 'underpriced' ] = 1 @@ -98,6 +103,7 @@ def get_buy_signal(pair: str) -> bool: minimum_date = arrow.now() - timedelta(hours=6) data = get_ticker(pair, minimum_date) dataframe = parse_ticker_dataframe(data['result'], minimum_date) + dataframe = populate_indicators(dataframe) dataframe = populate_trends(dataframe) latest = dataframe.iloc[-1] @@ -161,6 +167,7 @@ if __name__ == '__main__': minimum_date = arrow.now() - timedelta(hours=6) data = get_ticker(pair, minimum_date) dataframe = parse_ticker_dataframe(data['result'], minimum_date) + dataframe = populate_indicators(dataframe) dataframe = populate_trends(dataframe) plot_dataframe(dataframe, pair) time.sleep(60) From 507f12e92a1871c07fa8fe65e49c5d800b7fe945 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 9 Sep 2017 13:16:14 +0300 Subject: [PATCH 23/28] combine analyzation steps to one method --- analyze.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/analyze.py b/analyze.py index e4c1903af..348a20e8a 100644 --- a/analyze.py +++ b/analyze.py @@ -94,17 +94,21 @@ def populate_trends(dataframe: DataFrame) -> DataFrame: return dataframe +def analyze_ticker(pair: str) -> DataFrame: + minimum_date = arrow.now() - timedelta(hours=6) + data = get_ticker(pair, minimum_date) + dataframe = parse_ticker_dataframe(data['result'], minimum_date) + dataframe = populate_indicators(dataframe) + dataframe = populate_trends(dataframe) + return dataframe + def get_buy_signal(pair: str) -> bool: """ Calculates a buy signal based on StochRSI indicator :param pair: pair in format BTC_ANT or BTC-ANT :return: True if pair is underpriced, False otherwise """ - minimum_date = arrow.now() - timedelta(hours=6) - data = get_ticker(pair, minimum_date) - dataframe = parse_ticker_dataframe(data['result'], minimum_date) - dataframe = populate_indicators(dataframe) - dataframe = populate_trends(dataframe) + dataframe = analyze_ticker(pair) latest = dataframe.iloc[-1] # Check if dataframe is out of date @@ -164,10 +168,5 @@ if __name__ == '__main__': pair = 'BTC_ANT' #for pair in ['BTC_ANT', 'BTC_ETH', 'BTC_GNT', 'BTC_ETC']: # get_buy_signal(pair) - minimum_date = arrow.now() - timedelta(hours=6) - data = get_ticker(pair, minimum_date) - dataframe = parse_ticker_dataframe(data['result'], minimum_date) - dataframe = populate_indicators(dataframe) - dataframe = populate_trends(dataframe) - plot_dataframe(dataframe, pair) + plot_dataframe(analyze_ticker(pair), pair) time.sleep(60) From 1bcd51d6e064061d1aeb1a212c6d76242eeaa99f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 9 Sep 2017 14:48:49 +0300 Subject: [PATCH 24/28] first unit tests for analyze.py --- test/test_analyze.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 test/test_analyze.py diff --git a/test/test_analyze.py b/test/test_analyze.py new file mode 100644 index 000000000..7ba3345ab --- /dev/null +++ b/test/test_analyze.py @@ -0,0 +1,34 @@ +# pragma pylint: disable=missing-docstring +import unittest +import arrow +from analyze import parse_ticker_dataframe + +RESULT_BITTREX = { + 'success': True, + 'message': '', + 'result': [ + {'O': 0.00065311, 'H': 0.00065311, 'L': 0.00065311, 'C': 0.00065311, 'V': 22.17210568, 'T': '2017-08-30T10:40:00', 'BV': 0.01448082}, + {'O': 0.00066194, 'H': 0.00066195, 'L': 0.00066194, 'C': 0.00066195, 'V': 33.4727437, 'T': '2017-08-30T10:34:00', 'BV': 0.02215696}, + {'O': 0.00065311, 'H': 0.00065311, 'L': 0.00065311, 'C': 0.00065311, 'V': 53.85127609, 'T': '2017-08-30T10:37:00', 'BV': 0.0351708}, + {'O': 0.00066194, 'H': 0.00066194, 'L': 0.00065311, 'C': 0.00065311, 'V': 46.29210665, 'T': '2017-08-30T10:42:00', 'BV': 0.03063118}, + ] +} + +class TestAnalyze(unittest.TestCase): + def setUp(self): + self.result = parse_ticker_dataframe(RESULT_BITTREX['result'], arrow.get('2017-08-30T10:00:00')) + + def test_1_dataframe_has_correct_columns(self): + self.assertEqual(self.result.columns.tolist(), + ['close', 'date', 'high', 'low', 'open', 'volume']) + + def test_2_orders_by_date(self): + self.assertEqual(self.result['date'].tolist(), + ['2017-08-30T10:34:00', + '2017-08-30T10:37:00', + '2017-08-30T10:40:00', + '2017-08-30T10:42:00']) + + +if __name__ == '__main__': + unittest.main() From 2f3fd1de8aa0a39e3fe495c9137fa87094dcbdb9 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 9 Sep 2017 16:32:53 +0300 Subject: [PATCH 25/28] rename populate_trends to populate_buy_trend. make it produce buy and buy_price columns --- analyze.py | 32 +++++++++++++++----------------- test/test_analyze.py | 6 +++++- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/analyze.py b/analyze.py index 348a20e8a..8d62aac5d 100644 --- a/analyze.py +++ b/analyze.py @@ -71,42 +71,40 @@ def populate_indicators(dataframe: DataFrame) -> DataFrame: return dataframe -def populate_trends(dataframe: DataFrame) -> DataFrame: +def populate_buy_trend(dataframe: DataFrame) -> DataFrame: """ - Populates the trends for the given dataframe + Based on TA indicators, populates the buy trend for the given dataframe :param dataframe: DataFrame - :return: DataFrame with populated trends - """ - """ - dataframe.loc[ - (dataframe['stochrsi'] < 20) - & (dataframe['close_30_ema'] > (1 + 0.0025) * dataframe['close_60_ema']), - 'underpriced' - ] = 1 + :return: DataFrame with buy column """ dataframe.loc[ (dataframe['stochrsi'] < 20) & (dataframe['macd'] > dataframe['macds']) & (dataframe['close'] > dataframe['sar']), - 'underpriced' + 'buy' ] = 1 - dataframe.loc[dataframe['underpriced'] == 1, 'buy'] = dataframe['close'] + dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close'] return dataframe def analyze_ticker(pair: str) -> DataFrame: + """ + Get ticker data for given currency pair, push it to a DataFrame and + add several TA indicators and buy signal to it + :return DataFrame with ticker data and indicator data + """ minimum_date = arrow.now() - timedelta(hours=6) data = get_ticker(pair, minimum_date) dataframe = parse_ticker_dataframe(data['result'], minimum_date) dataframe = populate_indicators(dataframe) - dataframe = populate_trends(dataframe) + dataframe = populate_buy_trend(dataframe) return dataframe def get_buy_signal(pair: str) -> bool: """ - Calculates a buy signal based on StochRSI indicator + Calculates a buy signal based several technical analysis indicators :param pair: pair in format BTC_ANT or BTC-ANT - :return: True if pair is underpriced, False otherwise + :return: True if pair is good for buying, False otherwise """ dataframe = analyze_ticker(pair) latest = dataframe.iloc[-1] @@ -116,7 +114,7 @@ def get_buy_signal(pair: str) -> bool: if signal_date < arrow.now() - timedelta(minutes=10): return False - signal = latest['underpriced'] == 1 + signal = latest['buy'] == 1 logger.debug('buy_trigger: %s (pair=%s, signal=%s)', latest['date'], pair, signal) return signal @@ -141,7 +139,7 @@ def plot_dataframe(dataframe: DataFrame, pair: str) -> None: ax1.plot(dataframe.index.values, dataframe['close_30_ema'], label='EMA(30)') ax1.plot(dataframe.index.values, dataframe['close_90_ema'], label='EMA(90)') # ax1.plot(dataframe.index.values, dataframe['sell'], 'ro', label='sell') - ax1.plot(dataframe.index.values, dataframe['buy'], 'bo', label='buy') + ax1.plot(dataframe.index.values, dataframe['buy_price'], 'bo', label='buy') ax1.legend() ax2.plot(dataframe.index.values, dataframe['macd'], label='MACD') diff --git a/test/test_analyze.py b/test/test_analyze.py index 7ba3345ab..48cca6fd5 100644 --- a/test/test_analyze.py +++ b/test/test_analyze.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring import unittest import arrow -from analyze import parse_ticker_dataframe +from analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators RESULT_BITTREX = { 'success': True, @@ -29,6 +29,10 @@ class TestAnalyze(unittest.TestCase): '2017-08-30T10:40:00', '2017-08-30T10:42:00']) + def test_3_populates_buy_trend(self): + dataframe = populate_buy_trend(populate_indicators(self.result)) + self.assertTrue('buy' in dataframe.columns) + self.assertTrue('buy_price' in dataframe.columns) if __name__ == '__main__': unittest.main() From 4069e730395c9c88899b788d504a0194f1c03fa0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 9 Sep 2017 19:18:53 +0300 Subject: [PATCH 26/28] test for buy signal --- test/test_analyze.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/test_analyze.py b/test/test_analyze.py index 48cca6fd5..d57f0b108 100644 --- a/test/test_analyze.py +++ b/test/test_analyze.py @@ -1,7 +1,9 @@ # pragma pylint: disable=missing-docstring import unittest +from unittest.mock import patch +from pandas import DataFrame import arrow -from analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators +from analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators, analyze_ticker, get_buy_signal RESULT_BITTREX = { 'success': True, @@ -34,5 +36,14 @@ class TestAnalyze(unittest.TestCase): self.assertTrue('buy' in dataframe.columns) self.assertTrue('buy_price' in dataframe.columns) + def test_4_returns_latest_buy_signal(self): + buydf = DataFrame([{'buy': 1, 'date': arrow.utcnow()}]) + with patch('analyze.analyze_ticker', return_value=buydf): + self.assertEqual(get_buy_signal('BTC-ETH'), True) + buydf = DataFrame([{'buy': 0, 'date': arrow.utcnow()}]) + with patch('analyze.analyze_ticker', return_value=buydf): + self.assertEqual(get_buy_signal('BTC-ETH'), False) + + if __name__ == '__main__': unittest.main() From 8bf5f15125e657df082ff0ae861cec7c2e5fc6d8 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 10 Sep 2017 09:51:56 +0300 Subject: [PATCH 27/28] use DataFrames own functions to manipulate the Bittrex JSON --- analyze.py | 17 ++++++----------- test/test_analyze.py | 2 +- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/analyze.py b/analyze.py index 8d62aac5d..4878333b6 100644 --- a/analyze.py +++ b/analyze.py @@ -38,16 +38,11 @@ def parse_ticker_dataframe(ticker: list, minimum_date: arrow.Arrow) -> DataFrame :param pair: pair as str in format BTC_ETH or BTC-ETH :return: DataFrame """ - data = [{ - 'close': t['C'], - 'volume': t['V'], - 'open': t['O'], - 'high': t['H'], - 'low': t['L'], - 'date': t['T'], - } for t in sorted(ticker, key=lambda k: k['T']) if arrow.get(t['T']) > minimum_date] - - return DataFrame(json_normalize(data)) + df = DataFrame(ticker) \ + .drop('BV', 1) \ + .rename(columns={'C':'close', 'V':'volume', 'O':'open', 'H':'high', 'L':'low', 'T':'date'}) \ + .sort_values('date') + return df[df['date'].map(arrow.get) > minimum_date] def populate_indicators(dataframe: DataFrame) -> DataFrame: @@ -93,7 +88,7 @@ def analyze_ticker(pair: str) -> DataFrame: add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ - minimum_date = arrow.now() - timedelta(hours=6) + minimum_date = arrow.utcnow().shift(hours=-6) data = get_ticker(pair, minimum_date) dataframe = parse_ticker_dataframe(data['result'], minimum_date) dataframe = populate_indicators(dataframe) diff --git a/test/test_analyze.py b/test/test_analyze.py index d57f0b108..9fdc16d7a 100644 --- a/test/test_analyze.py +++ b/test/test_analyze.py @@ -22,7 +22,7 @@ class TestAnalyze(unittest.TestCase): def test_1_dataframe_has_correct_columns(self): self.assertEqual(self.result.columns.tolist(), - ['close', 'date', 'high', 'low', 'open', 'volume']) + ['close', 'high', 'low', 'open', 'date', 'volume']) def test_2_orders_by_date(self): self.assertEqual(self.result['date'].tolist(), From 78a0f7496daa097cc3abfcf6d1db08c51cdfb079 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 10 Sep 2017 22:56:43 +0200 Subject: [PATCH 28/28] version bump --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 6722d0b0a..6de813653 100755 --- a/main.py +++ b/main.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) __author__ = "gcarq" __copyright__ = "gcarq 2017" __license__ = "GPLv3" -__version__ = "0.8.0" +__version__ = "0.9.0" _CONF = {}