diff --git a/bin/freqtrade b/bin/freqtrade index 7476152fd..e7ae7a4ca 100755 --- a/bin/freqtrade +++ b/bin/freqtrade @@ -1,4 +1,7 @@ #!/usr/bin/env python3 -from freqtrade.main import main -main() \ No newline at end of file +import sys + +from freqtrade.main import main, set_loggers +set_loggers() +main(sys.argv[1:]) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 204b35c3a..cf3258465 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -65,8 +65,8 @@ load it: python3 ./freqtrade/main.py --strategy my_awesome_strategy ``` -If the bot does not find your strategy file, it will display in an error - message the reason (File not found, or errors in your code). +If the bot does not find your strategy file, it will display in an error +message the reason (File not found, or errors in your code). Learn more about strategy file in [optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md). diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b056cdf80..8bc552d74 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -1,122 +1,205 @@ """ Functions to analyze ticker data with indicators and produce buy and sell signals """ -import logging -from datetime import timedelta +from datetime import datetime, timedelta from enum import Enum -from typing import Dict, List +from typing import Dict, List, Tuple import arrow from pandas import DataFrame, to_datetime from freqtrade.exchange import get_ticker_history +from freqtrade.logger import Logger +from freqtrade.persistence import Trade from freqtrade.strategy.strategy import Strategy -logger = logging.getLogger(__name__) - class SignalType(Enum): - """ Enum to distinguish between buy and sell signals """ + """ + Enum to distinguish between buy and sell signals + """ BUY = "buy" SELL = "sell" -def parse_ticker_dataframe(ticker: list) -> DataFrame: +class Analyze(object): """ - Analyses the trend for the given ticker history - :param ticker: See exchange.get_ticker_history - :return: DataFrame + Analyze class contains everything the bot need to determine if the situation is good for + buying or selling. """ - columns = {'C': 'close', 'V': 'volume', 'O': 'open', 'H': 'high', 'L': 'low', 'T': 'date'} - frame = DataFrame(ticker) \ - .rename(columns=columns) - if 'BV' in frame: - frame.drop('BV', 1, inplace=True) - frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) - frame.sort_values('date', inplace=True) - return frame + def __init__(self, config: dict) -> None: + """ + Init Analyze + :param config: Bot configuration (use the one from Configuration()) + """ + self.logger = Logger(name=__name__, level=config.get('loglevel')).get_logger() + self.config = config + self.strategy = Strategy(self.config) -def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame + @staticmethod + def parse_ticker_dataframe(ticker: list) -> DataFrame: + """ + Analyses the trend for the given ticker history + :param ticker: See exchange.get_ticker_history + :return: DataFrame + """ + columns = {'C': 'close', 'V': 'volume', 'O': 'open', 'H': 'high', 'L': 'low', 'T': 'date'} + frame = DataFrame(ticker) \ + .rename(columns=columns) + if 'BV' in frame: + frame.drop('BV', 1, inplace=True) + frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) + frame.sort_values('date', inplace=True) + return frame - Performance Note: For the best performance be frugal on the number of indicators - you are using. Let uncomment only the indicator you are using in your strategies - or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - """ - strategy = Strategy() - return strategy.populate_indicators(dataframe=dataframe) + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + Performance Note: For the best performance be frugal on the number of indicators + you are using. Let uncomment only the indicator you are using in your strategies + or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + """ + return self.strategy.populate_indicators(dataframe=dataframe) -def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - strategy = Strategy() - return strategy.populate_buy_trend(dataframe=dataframe) + def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + """ + Based on TA indicators, populates the buy signal for the given dataframe + :param dataframe: DataFrame + :return: DataFrame with buy column + """ + return self.strategy.populate_buy_trend(dataframe=dataframe) + def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + """ + Based on TA indicators, populates the sell signal for the given dataframe + :param dataframe: DataFrame + :return: DataFrame with buy column + """ + return self.strategy.populate_sell_trend(dataframe=dataframe) -def populate_sell_trend(dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - strategy = Strategy() - return strategy.populate_sell_trend(dataframe=dataframe) + def get_ticker_interval(self) -> int: + """ + Return ticker interval to use + :return: Ticker interval value to use + """ + return self.strategy.ticker_interval + def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: + """ + Parses the given ticker history and returns a populated DataFrame + add several TA indicators and buy signal to it + :return DataFrame with ticker data and indicator data + """ + dataframe = self.parse_ticker_dataframe(ticker_history) + dataframe = self.populate_indicators(dataframe) + dataframe = self.populate_buy_trend(dataframe) + dataframe = self.populate_sell_trend(dataframe) + return dataframe -def analyze_ticker(ticker_history: List[Dict]) -> DataFrame: - """ - Parses the given ticker history and returns a populated DataFrame - add several TA indicators and buy signal to it - :return DataFrame with ticker data and indicator data - """ - dataframe = parse_ticker_dataframe(ticker_history) - dataframe = populate_indicators(dataframe) - dataframe = populate_buy_trend(dataframe) - dataframe = populate_sell_trend(dataframe) - return dataframe + def get_signal(self, pair: str, interval: int) -> Tuple[bool, bool]: + """ + Calculates current signal based several technical analysis indicators + :param pair: pair in format BTC_ANT or BTC-ANT + :param interval: Interval to use (in min) + :return: (Buy, Sell) A bool-tuple indicating buy/sell signal + """ + ticker_hist = get_ticker_history(pair, interval) + if not ticker_hist: + self.logger.warning('Empty ticker history for pair %s', pair) + return False, False + try: + dataframe = self.analyze_ticker(ticker_hist) + except ValueError as error: + self.logger.warning( + 'Unable to analyze ticker for pair %s: %s', + pair, + str(error) + ) + return False, False + except Exception as error: + self.logger.exception( + 'Unexpected error when analyzing ticker for pair %s: %s', + pair, + str(error) + ) + return False, False -# FIX: Maybe return False, if an error has occured, -# Otherwise we might mask an error as an non-signal-scenario -def get_signal(pair: str, interval: int) -> (bool, bool): - """ - Calculates current signal based several technical analysis indicators - :param pair: pair in format BTC_ANT or BTC-ANT - :return: (Buy, Sell) A bool-tuple indicating buy/sell signal - """ - ticker_hist = get_ticker_history(pair, interval) - if not ticker_hist: - logger.warning('Empty ticker history for pair %s', pair) - return (False, False) # return False ? + if dataframe.empty: + self.logger.warning('Empty dataframe for pair %s', pair) + return False, False - try: - dataframe = analyze_ticker(ticker_hist) - except ValueError as ex: - logger.warning('Unable to analyze ticker for pair %s: %s', pair, str(ex)) - return (False, False) # return False ? - except Exception as ex: - logger.exception('Unexpected error when analyzing ticker for pair %s: %s', pair, str(ex)) - return (False, False) # return False ? + latest = dataframe.iloc[-1] - if dataframe.empty: - logger.warning('Empty dataframe for pair %s', pair) - return (False, False) # return False ? + # Check if dataframe is out of date + signal_date = arrow.get(latest['date']) + if signal_date < arrow.utcnow() - timedelta(minutes=(interval + 5)): + self.logger.warning( + 'Outdated history for pair %s. Last tick is %s minutes old', + pair, + (arrow.utcnow() - signal_date).seconds // 60 + ) + return False, False - latest = dataframe.iloc[-1] + (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 + self.logger.debug( + 'trigger: %s (pair=%s) buy=%s sell=%s', + latest['date'], + pair, + str(buy), + str(sell) + ) + return buy, sell - # Check if dataframe is out of date - signal_date = arrow.get(latest['date']) - if signal_date < arrow.utcnow() - timedelta(minutes=(interval + 5)): - logger.warning('Outdated history for pair %s. Last tick is %s minutes old', - pair, (arrow.utcnow() - signal_date).seconds // 60) - return (False, False) # return False ? + def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool: + """ + This function evaluate if on the condition required to trigger a sell has been reached + if the threshold is reached and updates the trade record. + :return: True if trade should be sold, False otherwise + """ + # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) + if self.min_roi_reached(trade=trade, current_rate=rate, current_time=date): + self.logger.debug('Required profit reached. Selling..') + return True - (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 - logger.debug('trigger: %s (pair=%s) buy=%s sell=%s', latest['date'], pair, str(buy), str(sell)) - return (buy, sell) + # Experimental: Check if the trade is profitable before selling it (avoid selling at loss) + if self.config.get('experimental', {}).get('sell_profit_only', False): + self.logger.debug('Checking if trade is profitable..') + if trade.calc_profit(rate=rate) <= 0: + return False + + if sell and not buy and self.config.get('experimental', {}).get('use_sell_signal', False): + self.logger.debug('Sell signal received. Selling..') + return True + + return False + + def min_roi_reached(self, trade: Trade, current_rate: float, current_time: datetime) -> bool: + """ + Based an earlier trade and current price and ROI configuration, decides whether bot should + sell + :return True if bot should sell at current rate + """ + current_profit = trade.calc_profit_percent(current_rate) + if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: + self.logger.debug('Stop loss hit.') + return True + + # Check if time matches and current rate is above threshold + time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 + for duration, threshold in self.strategy.minimal_roi.items(): + if time_diff <= duration: + return False + if current_profit > threshold: + return True + + return False + + def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: + """ + Creates a dataframe and populates indicators for given ticker data + """ + return {pair: self.populate_indicators(self.parse_ticker_dataframe(pair_data)) + for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py new file mode 100644 index 000000000..c69135117 --- /dev/null +++ b/freqtrade/arguments.py @@ -0,0 +1,251 @@ +""" +This module contains the argument manager class +""" + +import argparse +import logging +import os +import re +from typing import List, Tuple, Optional + +from freqtrade import __version__ +from freqtrade.constants import Constants + + +class Arguments(object): + """ + Arguments Class. Manage the arguments received by the cli + """ + + def __init__(self, args: List[str], description: str): + self.args = args + self.parsed_arg = None + self.parser = argparse.ArgumentParser(description=description) + + def _load_args(self) -> None: + self.common_args_parser() + self._build_subcommands() + + def get_parsed_arg(self) -> argparse.Namespace: + """ + Return the list of arguments + :return: List[str] List of arguments + """ + if self.parsed_arg is None: + self._load_args() + self.parsed_arg = self.parse_args() + + return self.parsed_arg + + def parse_args(self) -> argparse.Namespace: + """ + Parses given arguments and returns an argparse Namespace instance. + """ + parsed_arg = self.parser.parse_args(self.args) + + return parsed_arg + + def common_args_parser(self) -> None: + """ + Parses given common arguments and returns them as a parsed object. + """ + self.parser.add_argument( + '-v', '--verbose', + help='be verbose', + action='store_const', + dest='loglevel', + const=logging.DEBUG, + default=logging.INFO, + ) + self.parser.add_argument( + '--version', + action='version', + version='%(prog)s {}'.format(__version__), + ) + self.parser.add_argument( + '-c', '--config', + help='specify configuration file (default: %(default)s)', + dest='config', + default='config.json', + type=str, + metavar='PATH', + ) + self.parser.add_argument( + '-d', '--datadir', + help='path to backtest data (default: %(default)s', + dest='datadir', + default=os.path.join('freqtrade', 'tests', 'testdata'), + type=str, + metavar='PATH', + ) + self.parser.add_argument( + '-s', '--strategy', + help='specify strategy file (default: %(default)s)', + dest='strategy', + default='default_strategy', + type=str, + metavar='PATH', + ) + self.parser.add_argument( + '--dynamic-whitelist', + help='dynamically generate and update whitelist \ + based on 24h BaseVolume (Default 20 currencies)', # noqa + dest='dynamic_whitelist', + const=Constants.DYNAMIC_WHITELIST, + type=int, + metavar='INT', + nargs='?', + ) + self.parser.add_argument( + '--dry-run-db', + help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \ + instead of memory DB. Work only if dry_run is enabled.', + action='store_true', + dest='dry_run_db', + ) + + @staticmethod + def backtesting_options(parser: argparse.ArgumentParser) -> None: + """ + Parses given arguments for Backtesting scripts. + """ + parser.add_argument( + '-l', '--live', + help='using live data', + action='store_true', + dest='live', + ) + parser.add_argument( + '-r', '--refresh-pairs-cached', + help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \ + Use it if you want to run your backtesting with up-to-date data.', + action='store_true', + dest='refresh_pairs', + ) + parser.add_argument( + '--export', + help='export backtest results, argument are: trades\ + Example --export=trades', + type=str, + default=None, + dest='export', + ) + + @staticmethod + def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + '-i', '--ticker-interval', + help='specify ticker interval in minutes (1, 5, 30, 60, 1440)', + dest='ticker_interval', + type=int, + metavar='INT', + ) + parser.add_argument( + '--realistic-simulation', + help='uses max_open_trades from config to simulate real world limitations', + action='store_true', + dest='realistic_simulation', + ) + parser.add_argument( + '--timerange', + help='specify what timerange of data to use.', + default=None, + type=str, + dest='timerange', + ) + + @staticmethod + def hyperopt_options(parser: argparse.ArgumentParser) -> None: + """ + Parses given arguments for Hyperopt scripts. + """ + parser.add_argument( + '-e', '--epochs', + help='specify number of epochs (default: %(default)d)', + dest='epochs', + default=Constants.HYPEROPT_EPOCH, + type=int, + metavar='INT', + ) + parser.add_argument( + '--use-mongodb', + help='parallelize evaluations with mongodb (requires mongod in PATH)', + dest='mongodb', + action='store_true', + ) + parser.add_argument( + '-s', '--spaces', + help='Specify which parameters to hyperopt. Space separate list. \ + Default: %(default)s', + choices=['all', 'buy', 'roi', 'stoploss'], + default='all', + nargs='+', + dest='spaces', + ) + + def _build_subcommands(self) -> None: + """ + Builds and attaches all subcommands + :return: None + """ + from freqtrade.optimize import backtesting, hyperopt + + subparsers = self.parser.add_subparsers(dest='subparser') + + # Add backtesting subcommand + backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module') + backtesting_cmd.set_defaults(func=backtesting.start) + self.optimizer_shared_options(backtesting_cmd) + self.backtesting_options(backtesting_cmd) + + # Add hyperopt subcommand + hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') + hyperopt_cmd.set_defaults(func=hyperopt.start) + self.optimizer_shared_options(hyperopt_cmd) + self.hyperopt_options(hyperopt_cmd) + + @staticmethod + def parse_timerange(text: str) -> Optional[Tuple[List, int, int]]: + """ + Parse the value of the argument --timerange to determine what is the range desired + :param text: value from --timerange + :return: Start and End range period + """ + if text is None: + return None + syntax = [(r'^-(\d{8})$', (None, 'date')), + (r'^(\d{8})-$', ('date', None)), + (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^(-\d+)$', (None, 'line')), + (r'^(\d+)-$', ('line', None)), + (r'^(\d+)-(\d+)$', ('index', 'index'))] + for rex, stype in syntax: + # Apply the regular expression to text + match = re.match(rex, text) + if match: # Regex has matched + rvals = match.groups() + index = 0 + start = None + stop = None + if stype[0]: + start = rvals[index] + if stype[0] != 'date': + start = int(start) + index += 1 + if stype[1]: + stop = rvals[index] + if stype[1] != 'date': + stop = int(stop) + return stype, start, stop + raise Exception('Incorrect syntax for timerange "%s"' % text) + + def scripts_options(self) -> None: + """ + Parses given arguments for plot scripts. + """ + self.parser.add_argument( + '-p', '--pair', + help='Show profits for only this pairs. Pairs are comma-separated.', + dest='pair', + default=None + ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py new file mode 100644 index 000000000..1f6cea4e6 --- /dev/null +++ b/freqtrade/configuration.py @@ -0,0 +1,200 @@ +""" +This module contains the configuration class +""" + +import json +from argparse import Namespace +from typing import Dict, Any + +from jsonschema import Draft4Validator, validate +from jsonschema.exceptions import ValidationError, best_match + +from freqtrade.constants import Constants +from freqtrade.logger import Logger + + +class Configuration(object): + """ + Class to read and init the bot configuration + Reuse this class for the bot, backtesting, hyperopt and every script that required configuration + """ + def __init__(self, args: Namespace) -> None: + self.args = args + self.logging = Logger(name=__name__) + self.logger = self.logging.get_logger() + self.config = None + + def load_config(self) -> Dict[str, Any]: + """ + Extract information for sys.argv and load the bot configuration + :return: Configuration dictionary + """ + self.logger.info('Using config: %s ...', self.args.config) + config = self._load_config_file(self.args.config) + + # Add the strategy file to use + config.update({'strategy': self.args.strategy}) + + # Load Common configuration + config = self._load_common_config(config) + + # Load Backtesting + config = self._load_backtesting_config(config) + + # Load Hyperopt + config = self._load_hyperopt_config(config) + + return config + + def _load_config_file(self, path: str) -> Dict[str, Any]: + """ + Loads a config file from the given path + :param path: path as str + :return: configuration as dictionary + """ + try: + with open(path) as file: + conf = json.load(file) + except FileNotFoundError: + self.logger.critical( + 'Config file "%s" not found. Please create your config file', + path + ) + exit(0) + + if 'internals' not in conf: + conf['internals'] = {} + self.logger.info('Validating configuration ...') + + return self._validate_config(conf) + + def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv and load common configuration + :return: configuration as dictionary + """ + + # Log level + if 'loglevel' in self.args and self.args.loglevel: + config.update({'loglevel': self.args.loglevel}) + self.logging.set_level(self.args.loglevel) + self.logger.info('Log level set at %s', config['loglevel']) + + # Add dynamic_whitelist if found + if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: + config.update({'dynamic_whitelist': self.args.dynamic_whitelist}) + self.logger.info( + 'Parameter --dynamic-whitelist detected. ' + 'Using dynamically generated whitelist. ' + '(not applicable with Backtesting and Hyperopt)' + ) + + # Add dry_run_db if found and the bot in dry run + if self.args.dry_run_db and config.get('dry_run', False): + config.update({'dry_run_db': True}) + self.logger.info('Parameter --dry-run-db detected ...') + + if config.get('dry_run_db', False): + if config.get('dry_run', False): + self.logger.info('Dry_run will use the DB file: "tradesv3.dry_run.sqlite"') + else: + self.logger.info('Dry run is disabled. (--dry_run_db ignored)') + + return config + + def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv and load Backtesting configuration + :return: configuration as dictionary + """ + + # If -i/--ticker-interval is used we override the configuration parameter + # (that will override the strategy configuration) + if 'ticker_interval' in self.args and self.args.ticker_interval: + config.update({'ticker_interval': self.args.ticker_interval}) + self.logger.info('Parameter -i/--ticker-interval detected ...') + self.logger.info('Using ticker_interval: %d ...', config.get('ticker_interval')) + + # If -l/--live is used we add it to the configuration + if 'live' in self.args and self.args.live: + config.update({'live': True}) + self.logger.info('Parameter -l/--live detected ...') + + # If --realistic-simulation is used we add it to the configuration + if 'realistic_simulation' in self.args and self.args.realistic_simulation: + config.update({'realistic_simulation': True}) + self.logger.info('Parameter --realistic-simulation detected ...') + self.logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) + + # If --timerange is used we add it to the configuration + if 'timerange' in self.args and self.args.timerange: + config.update({'timerange': self.args.timerange}) + self.logger.info('Parameter --timerange detected: %s ...', self.args.timerange) + + # If --datadir is used we add it to the configuration + if 'datadir' in self.args and self.args.datadir: + config.update({'datadir': self.args.datadir}) + self.logger.info('Parameter --datadir detected: %s ...', self.args.datadir) + + # If -r/--refresh-pairs-cached is used we add it to the configuration + if 'refresh_pairs' in self.args and self.args.refresh_pairs: + config.update({'refresh_pairs': True}) + self.logger.info('Parameter -r/--refresh-pairs-cached detected ...') + + # If --export is used we add it to the configuration + if 'export' in self.args and self.args.export: + config.update({'export': self.args.export}) + self.logger.info('Parameter --export detected: %s ...', self.args.export) + + return config + + def _load_hyperopt_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv and load Hyperopt configuration + :return: configuration as dictionary + """ + # If --realistic-simulation is used we add it to the configuration + if 'epochs' in self.args and self.args.epochs: + config.update({'epochs': self.args.epochs}) + self.logger.info('Parameter --epochs detected ...') + self.logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs')) + + # If --mongodb is used we add it to the configuration + if 'mongodb' in self.args and self.args.mongodb: + config.update({'mongodb': self.args.mongodb}) + self.logger.info('Parameter --use-mongodb detected ...') + + # If --spaces is used we add it to the configuration + if 'spaces' in self.args and self.args.spaces: + config.update({'spaces': self.args.spaces}) + self.logger.info('Parameter -s/--spaces detected: %s', config.get('spaces')) + + return config + + def _validate_config(self, conf: Dict[str, Any]) -> Dict[str, Any]: + """ + Validate the configuration follow the Config Schema + :param conf: Config in JSON format + :return: Returns the config if valid, otherwise throw an exception + """ + try: + validate(conf, Constants.CONF_SCHEMA) + return conf + except ValidationError as exception: + self.logger.fatal( + 'Invalid configuration. See config.json.example. Reason: %s', + exception + ) + raise ValidationError( + best_match(Draft4Validator(Constants.CONF_SCHEMA).iter_errors(conf)).message + ) + + def get_config(self) -> Dict[str, Any]: + """ + Return the config. Use this method to get the bot config + :return: Dict: Bot config + """ + if self.config is None: + self.config = self.load_config() + + return self.config diff --git a/freqtrade/constants.py b/freqtrade/constants.py new file mode 100644 index 000000000..a3f91d774 --- /dev/null +++ b/freqtrade/constants.py @@ -0,0 +1,122 @@ +# pragma pylint: disable=too-few-public-methods + +""" +List bot constants +""" + + +class Constants(object): + """ + Static class that contain all bot constants + """ + DYNAMIC_WHITELIST = 20 # pairs + PROCESS_THROTTLE_SECS = 5 # sec + TICKER_INTERVAL = 5 # min + HYPEROPT_EPOCH = 100 # epochs + RETRY_TIMEOUT = 30 # sec + DEFAULT_STRATEGY = 'default_strategy' + + # Required json-schema for user specified config + CONF_SCHEMA = { + 'type': 'object', + 'properties': { + 'max_open_trades': {'type': 'integer', 'minimum': 1}, + 'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]}, + 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, + 'stake_amount': {'type': 'number', 'minimum': 0.0005}, + 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', + 'CLP', 'CNY', 'CZK', 'DKK', + 'EUR', 'GBP', 'HKD', 'HUF', + 'IDR', 'ILS', 'INR', 'JPY', + 'KRW', 'MXN', 'MYR', 'NOK', + 'NZD', 'PHP', 'PKR', 'PLN', + 'RUB', 'SEK', 'SGD', 'THB', + 'TRY', 'TWD', 'ZAR', 'USD']}, + 'dry_run': {'type': 'boolean'}, + 'minimal_roi': { + 'type': 'object', + 'patternProperties': { + '^[0-9.]+$': {'type': 'number'} + }, + 'minProperties': 1 + }, + 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, + 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, + 'bid_strategy': { + 'type': 'object', + 'properties': { + 'ask_last_balance': { + 'type': 'number', + 'minimum': 0, + 'maximum': 1, + 'exclusiveMaximum': False + }, + }, + 'required': ['ask_last_balance'] + }, + 'exchange': {'$ref': '#/definitions/exchange'}, + 'experimental': { + 'type': 'object', + 'properties': { + 'use_sell_signal': {'type': 'boolean'}, + 'sell_profit_only': {'type': 'boolean'} + } + }, + 'telegram': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'token': {'type': 'string'}, + 'chat_id': {'type': 'string'}, + }, + 'required': ['enabled', 'token', 'chat_id'] + }, + 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, + 'internals': { + 'type': 'object', + 'properties': { + 'process_throttle_secs': {'type': 'number'}, + 'interval': {'type': 'integer'} + } + } + }, + 'definitions': { + 'exchange': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'key': {'type': 'string'}, + 'secret': {'type': 'string'}, + 'pair_whitelist': { + 'type': 'array', + 'items': { + 'type': 'string', + 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' + }, + 'uniqueItems': True + }, + 'pair_blacklist': { + 'type': 'array', + 'items': { + 'type': 'string', + 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' + }, + 'uniqueItems': True + } + }, + 'required': ['name', 'key', 'secret', 'pair_whitelist'] + } + }, + 'anyOf': [ + {'required': ['exchange']} + ], + 'required': [ + 'max_open_trades', + 'stake_currency', + 'stake_amount', + 'fiat_display_currency', + 'dry_run', + 'bid_strategy', + 'telegram' + ] + } diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 5aa07e460..0cba621af 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -1,8 +1,8 @@ import logging from typing import Dict, List, Optional -from bittrex.bittrex import Bittrex as _Bittrex from bittrex.bittrex import API_V1_1, API_V2_0 +from bittrex.bittrex import Bittrex as _Bittrex from requests.exceptions import ContentDecodingError from freqtrade import OperationalException diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 4874247db..b86b56ec5 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -5,12 +5,13 @@ e.g BTC to USD import logging import time + from coinmarketcap import Market logger = logging.getLogger(__name__) -class CryptoFiat(): +class CryptoFiat(object): """ Object to describe what is the price of Crypto-currency in a FIAT """ diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py new file mode 100644 index 000000000..e57f177e9 --- /dev/null +++ b/freqtrade/freqtradebot.py @@ -0,0 +1,537 @@ +""" +Freqtrade is the main module of this bot. It contains the class Freqtrade() +""" + +import copy +import json +import time +import traceback +from datetime import datetime +from typing import Dict, List, Optional, Any, Callable + +import arrow +import requests +from cachetools import cached, TTLCache + +from freqtrade import (DependencyException, OperationalException, exchange, persistence) +from freqtrade.analyze import Analyze +from freqtrade.constants import Constants +from freqtrade.fiat_convert import CryptoToFiatConverter +from freqtrade.logger import Logger +from freqtrade.persistence import Trade +from freqtrade.rpc.rpc_manager import RPCManager +from freqtrade.state import State + + +class FreqtradeBot(object): + """ + Freqtrade is the main class of the bot. + This is from here the bot start its logic. + """ + + def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None): + """ + Init all variables and object the bot need to work + :param config: configuration dict, you can use the Configuration.get_config() + method to get the config dict. + :param db_url: database connector string for sqlalchemy (Optional) + """ + + # Init the logger + self.logger = Logger(name=__name__, level=config.get('loglevel')).get_logger() + + # Init bot states + self._state = State.STOPPED + + # Init objects + self.config = config + self.analyze = None + self.fiat_converter = None + self.rpc = None + self.persistence = None + self.exchange = None + + self._init_modules(db_url=db_url) + + def _init_modules(self, db_url: Optional[str] = None) -> None: + """ + Initializes all modules and updates the config + :param db_url: database connector string for sqlalchemy (Optional) + :return: None + """ + # Initialize all modules + self.analyze = Analyze(self.config) + self.fiat_converter = CryptoToFiatConverter() + self.rpc = RPCManager(self) + + persistence.init(self.config, db_url) + exchange.init(self.config) + + # Set initial application state + initial_state = self.config.get('initial_state') + + if initial_state: + self.update_state(State[initial_state.upper()]) + else: + self.update_state(State.STOPPED) + + def clean(self) -> bool: + """ + Cleanup the application state und finish all pending tasks + :return: None + """ + self.rpc.send_msg('*Status:* `Stopping trader...`') + self.logger.info('Stopping trader and cleaning up modules...') + self.update_state(State.STOPPED) + self.rpc.cleanup() + persistence.cleanup() + return True + + def update_state(self, state: State) -> None: + """ + Updates the application state + :param state: new state + :return: None + """ + self._state = state + + def get_state(self) -> State: + """ + Gets the current application state + :return: + """ + return self._state + + def worker(self, old_state: None) -> State: + """ + Trading routine that must be run at each loop + :param old_state: the previous service state from the previous call + :return: current service state + """ + new_state = self.get_state() + # Log state transition + if new_state != old_state: + self.rpc.send_msg('*Status:* `{}`'.format(new_state.name.lower())) + self.logger.info('Changing state to: %s', new_state.name) + + if new_state == State.STOPPED: + time.sleep(1) + elif new_state == State.RUNNING: + min_secs = self.config.get('internals', {}).get( + 'process_throttle_secs', + Constants.PROCESS_THROTTLE_SECS + ) + + nb_assets = self.config.get( + 'dynamic_whitelist', + Constants.DYNAMIC_WHITELIST + ) + + self._throttle(func=self._process, + min_secs=min_secs, + nb_assets=nb_assets) + return new_state + + def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: + """ + Throttles the given callable that it + takes at least `min_secs` to finish execution. + :param func: Any callable + :param min_secs: minimum execution time in seconds + :return: Any + """ + start = time.time() + result = func(*args, **kwargs) + end = time.time() + duration = max(min_secs - (end - start), 0.0) + self.logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) + time.sleep(duration) + return result + + def _process(self, nb_assets: Optional[int] = 0) -> bool: + """ + Queries the persistence layer for open trades and handles them, + otherwise a new trade is created. + :param: nb_assets: the maximum number of pairs to be traded at the same time + :return: True if one or more trades has been created or closed, False otherwise + """ + state_changed = False + try: + # Refresh whitelist based on wallet maintenance + sanitized_list = self._refresh_whitelist( + self._gen_pair_whitelist( + self.config['stake_currency'] + ) if nb_assets else self.config['exchange']['pair_whitelist'] + ) + + # Keep only the subsets of pairs wanted (up to nb_assets) + final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list + self.config['exchange']['pair_whitelist'] = final_list + + # Query trades from persistence layer + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + + # First process current opened trades + for trade in trades: + state_changed |= self.process_maybe_execute_sell(trade) + + # Then looking for buy opportunities + if len(trades) < self.config['max_open_trades']: + state_changed = self.process_maybe_execute_buy() + + if 'unfilledtimeout' in self.config: + # Check and handle any timed out open orders + self.check_handle_timedout(self.config['unfilledtimeout']) + Trade.session.flush() + + except (requests.exceptions.RequestException, json.JSONDecodeError) as error: + self.logger.warning('%s, retrying in 30 seconds...', error) + time.sleep(Constants.RETRY_TIMEOUT) + except OperationalException: + self.rpc.send_msg( + '*Status:* OperationalException:\n```\n{traceback}```{hint}' + .format( + traceback=traceback.format_exc(), + hint='Issue `/start` if you think it is safe to restart.' + ) + ) + self.logger.exception('OperationalException. Stopping trader ...') + self.update_state(State.STOPPED) + return state_changed + + @cached(TTLCache(maxsize=1, ttl=1800)) + def _gen_pair_whitelist(self, base_currency: str, key: str = 'BaseVolume') -> List[str]: + """ + Updates the whitelist with with a dynamically generated list + :param base_currency: base currency as str + :param key: sort key (defaults to 'BaseVolume') + :return: List of pairs + """ + summaries = sorted( + (s for s in exchange.get_market_summaries() if + s['MarketName'].startswith(base_currency)), + key=lambda s: s.get(key) or 0.0, + reverse=True + ) + + return [s['MarketName'].replace('-', '_') for s in summaries] + + def _refresh_whitelist(self, whitelist: List[str]) -> List[str]: + """ + Check wallet health and remove pair from whitelist if necessary + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to + trade + :return: the list of pairs the user wants to trade without the one unavailable or + black_listed + """ + sanitized_whitelist = whitelist + health = exchange.get_wallet_health() + known_pairs = set() + for status in health: + pair = '{}_{}'.format(self.config['stake_currency'], status['Currency']) + # pair is not int the generated dynamic market, or in the blacklist ... ignore it + if pair not in whitelist or pair in self.config['exchange'].get('pair_blacklist', []): + continue + # else the pair is valid + known_pairs.add(pair) + # Market is not active + if not status['IsActive']: + sanitized_whitelist.remove(pair) + self.logger.info( + 'Ignoring %s from whitelist (reason: %s).', + pair, status.get('Notice') or 'wallet is not active' + ) + + # We need to remove pairs that are unknown + final_list = [x for x in sanitized_whitelist if x in known_pairs] + return final_list + + def get_target_bid(self, ticker: Dict[str, float]) -> float: + """ + Calculates bid target between current ask price and last price + :param ticker: Ticker to use for getting Ask and Last Price + :return: float: Price + """ + if ticker['ask'] < ticker['last']: + return ticker['ask'] + balance = self.config['bid_strategy']['ask_last_balance'] + return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + + def create_trade(self) -> bool: + """ + Checks the implemented trading indicator(s) for a randomly picked pair, + if one pair triggers the buy_signal a new trade record gets created + :param stake_amount: amount of btc to spend + :param interval: Ticker interval used for Analyze + :return: True if a trade object has been created and persisted, False otherwise + """ + stake_amount = self.config['stake_amount'] + interval = self.analyze.get_ticker_interval() + + self.logger.info( + 'Checking buy signals to create a new trade with stake_amount: %f ...', + stake_amount + ) + whitelist = copy.deepcopy(self.config['exchange']['pair_whitelist']) + # Check if stake_amount is fulfilled + if exchange.get_balance(self.config['stake_currency']) < stake_amount: + raise DependencyException( + 'stake amount is not fulfilled (currency={})'.format(self.config['stake_currency']) + ) + + # Remove currently opened and latest pairs from whitelist + for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): + if trade.pair in whitelist: + whitelist.remove(trade.pair) + self.logger.debug('Ignoring %s in pair whitelist', trade.pair) + + if not whitelist: + raise DependencyException('No currency pairs in whitelist') + + # Pick pair based on StochRSI buy signals + for _pair in whitelist: + (buy, sell) = self.analyze.get_signal(_pair, interval) + if buy and not sell: + pair = _pair + break + else: + return False + + # Calculate amount + buy_limit = self.get_target_bid(exchange.get_ticker(pair)) + amount = stake_amount / buy_limit + + order_id = exchange.buy(pair, buy_limit, amount) + + stake_amount_fiat = self.fiat_converter.convert_amount( + stake_amount, + self.config['stake_currency'], + self.config['fiat_display_currency'] + ) + + # Create trade entity and return + self.rpc.send_msg( + '*{}:* Buying [{}]({}) with limit `{:.8f} ({:.6f} {}, {:.3f} {})` ' + .format( + exchange.get_name().upper(), + pair.replace('_', '/'), + exchange.get_pair_detail_url(pair), + buy_limit, + stake_amount, + self.config['stake_currency'], + stake_amount_fiat, + self.config['fiat_display_currency'] + ) + ) + # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL + trade = Trade( + pair=pair, + stake_amount=stake_amount, + amount=amount, + fee=exchange.get_fee(), + open_rate=buy_limit, + open_date=datetime.utcnow(), + exchange=exchange.get_name().upper(), + open_order_id=order_id + ) + Trade.session.add(trade) + Trade.session.flush() + return True + + def process_maybe_execute_buy(self) -> bool: + """ + Tries to execute a buy trade in a safe way + :return: True if executed + """ + try: + # Create entity and execute trade + if self.create_trade(): + return True + + self.logger.info('Found no buy signals for whitelisted currencies. Trying again..') + return False + except DependencyException as exception: + self.logger.warning('Unable to create trade: %s', exception) + return False + + def process_maybe_execute_sell(self, trade: Trade) -> bool: + """ + Tries to execute a sell trade + :return: True if executed + """ + # Get order details for actual price per unit + if trade.open_order_id: + # Update trade with order values + self.logger.info('Found open order for %s', trade) + trade.update(exchange.get_order(trade.open_order_id)) + + if trade.is_open and trade.open_order_id is None: + # Check if we can sell our current pair + return self.handle_trade(trade) + return False + + def handle_trade(self, trade: Trade) -> bool: + """ + Sells the current pair if the threshold is reached and updates the trade record. + :return: True if trade has been sold, False otherwise + """ + if not trade.is_open: + raise ValueError('attempt to handle closed trade: {}'.format(trade)) + + self.logger.debug('Handling %s ...', trade) + current_rate = exchange.get_ticker(trade.pair)['bid'] + + (buy, sell) = (False, False) + + if self.config.get('experimental', {}).get('use_sell_signal'): + (buy, sell) = self.analyze.get_signal(trade.pair, self.analyze.get_ticker_interval()) + + if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): + self.execute_sell(trade, current_rate) + return True + + return False + + def check_handle_timedout(self, timeoutvalue: int) -> None: + """ + Check if any orders are timed out and cancel if neccessary + :param timeoutvalue: Number of minutes until order is considered timed out + :return: None + """ + timeoutthreashold = arrow.utcnow().shift(minutes=-timeoutvalue).datetime + + for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): + try: + order = exchange.get_order(trade.open_order_id) + except requests.exceptions.RequestException: + self.logger.info( + 'Cannot query order for %s due to %s', + trade, + traceback.format_exc()) + continue + ordertime = arrow.get(order['opened']) + + # Check if trade is still actually open + if int(order['remaining']) == 0: + continue + + if order['type'] == "LIMIT_BUY" and ordertime < timeoutthreashold: + self.handle_timedout_limit_buy(trade, order) + elif order['type'] == "LIMIT_SELL" and ordertime < timeoutthreashold: + self.handle_timedout_limit_sell(trade, order) + + # FIX: 20180110, why is cancel.order unconditionally here, whereas + # it is conditionally called in the + # handle_timedout_limit_sell()? + def handle_timedout_limit_buy(self, trade: Trade, order: Dict) -> bool: + """Buy timeout - cancel order + :return: True if order was fully cancelled + """ + exchange.cancel_order(trade.open_order_id) + if order['remaining'] == order['amount']: + # if trade is not partially completed, just delete the trade + Trade.session.delete(trade) + # FIX? do we really need to flush, caller of + # check_handle_timedout will flush afterwards + Trade.session.flush() + self.logger.info('Buy order timeout for %s.', trade) + self.rpc.send_msg('*Timeout:* Unfilled buy order for {} cancelled'.format( + trade.pair.replace('_', '/'))) + return True + + # if trade is partially complete, edit the stake details for the trade + # and close the order + trade.amount = order['amount'] - order['remaining'] + trade.stake_amount = trade.amount * trade.open_rate + trade.open_order_id = None + self.logger.info('Partial buy order timeout for %s.', trade) + self.rpc.send_msg('*Timeout:* Remaining buy order for {} cancelled'.format( + trade.pair.replace('_', '/'))) + return False + + # FIX: 20180110, should cancel_order() be cond. or unconditionally called? + def handle_timedout_limit_sell(self, trade: Trade, order: Dict) -> bool: + """ + Sell timeout - cancel order and update trade + :return: True if order was fully cancelled + """ + if order['remaining'] == order['amount']: + # if trade is not partially completed, just cancel the trade + exchange.cancel_order(trade.open_order_id) + trade.close_rate = None + trade.close_profit = None + trade.close_date = None + trade.is_open = True + trade.open_order_id = None + self.rpc.send_msg('*Timeout:* Unfilled sell order for {} cancelled'.format( + trade.pair.replace('_', '/'))) + self.logger.info('Sell order timeout for %s.', trade) + return True + + # TODO: figure out how to handle partially complete sell orders + return False + + def execute_sell(self, trade: Trade, limit: float) -> None: + """ + Executes a limit sell for the given trade and limit + :param trade: Trade instance + :param limit: limit rate for the sell order + :return: None + """ + # Execute sell and update trade record + order_id = exchange.sell(str(trade.pair), limit, trade.amount) + trade.open_order_id = order_id + + fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) + profit_trade = trade.calc_profit(rate=limit) + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + profit = trade.calc_profit_percent(current_rate) + + message = "*{exchange}:* Selling\n" \ + "*Current Pair:* [{pair}]({pair_url})\n" \ + "*Limit:* `{limit}`\n" \ + "*Amount:* `{amount}`\n" \ + "*Open Rate:* `{open_rate:.8f}`\n" \ + "*Current Rate:* `{current_rate:.8f}`\n" \ + "*Profit:* `{profit:.2f}%`" \ + "".format( + exchange=trade.exchange, + pair=trade.pair, + pair_url=exchange.get_pair_detail_url(trade.pair), + limit=limit, + open_rate=trade.open_rate, + current_rate=current_rate, + amount=round(trade.amount, 8), + profit=round(profit * 100, 2), + ) + + # For regular case, when the configuration exists + if 'stake_currency' in self.config and 'fiat_display_currency' in self.config: + fiat_converter = CryptoToFiatConverter() + profit_fiat = fiat_converter.convert_amount( + profit_trade, + self.config['stake_currency'], + self.config['fiat_display_currency'] + ) + message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ + '` / {profit_fiat:.3f} {fiat})`' \ + ''.format( + gain="profit" if fmt_exp_profit > 0 else "loss", + profit_percent=fmt_exp_profit, + profit_coin=profit_trade, + coin=self.config['stake_currency'], + profit_fiat=profit_fiat, + fiat=self.config['fiat_display_currency'], + ) + # Because telegram._forcesell does not have the configuration + # Ignore the FIAT value and does not show the stake_currency as well + else: + message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f})`'.format( + gain="profit" if fmt_exp_profit > 0 else "loss", + profit_percent=fmt_exp_profit, + profit_coin=profit_trade + ) + + # Send the message + self.rpc.send_msg(message) + Trade.session.flush() diff --git a/freqtrade/indicator_helpers.py b/freqtrade/indicator_helpers.py index c3cc42e6a..14519d7a2 100644 --- a/freqtrade/indicator_helpers.py +++ b/freqtrade/indicator_helpers.py @@ -1,19 +1,19 @@ from math import exp, pi, sqrt, cos -import numpy +import numpy as np import talib as ta from pandas import Series -def went_up(series: Series) -> Series: +def went_up(series: Series) -> bool: return series > series.shift(1) -def went_down(series: Series) -> Series: +def went_down(series: Series) -> bool: return series < series.shift(1) -def ehlers_super_smoother(series: Series, smoothing: float = 6): +def ehlers_super_smoother(series: Series, smoothing: float = 6) -> type(Series): magic = pi * sqrt(2) / smoothing a1 = exp(-magic) coeff2 = 2 * a1 * cos(magic) @@ -29,7 +29,7 @@ def ehlers_super_smoother(series: Series, smoothing: float = 6): return filtered -def fishers_inverse(series: Series, smoothing: float = 0): +def fishers_inverse(series: Series, smoothing: float = 0) -> np.ndarray: """ Does a smoothed fishers inverse transformation. Can be used with any oscillator that goes from 0 to 100 like RSI or MFI """ v1 = 0.1 * (series - 50) @@ -37,4 +37,4 @@ def fishers_inverse(series: Series, smoothing: float = 0): v2 = ta.WMA(v1.values, timeperiod=smoothing) else: v2 = v1 - return (numpy.exp(2 * v2)-1) / (numpy.exp(2 * v2) + 1) + return (np.exp(2 * v2)-1) / (np.exp(2 * v2) + 1) diff --git a/freqtrade/logger.py b/freqtrade/logger.py new file mode 100644 index 000000000..95e55e477 --- /dev/null +++ b/freqtrade/logger.py @@ -0,0 +1,83 @@ +# pragma pylint: disable=too-few-public-methods + +""" +This module contains the class for logger and logging messages +""" + +import logging + + +class Logger(object): + """ + Logging class + """ + def __init__(self, name='', level=logging.INFO) -> None: + """ + Init the logger class + :param name: Name of the Logger scope + :param level: Logger level that should be used + :return: None + """ + self.name = name + self.logger = None + + if level is None: + level = logging.INFO + self.level = level + + self._init_logger() + + def _init_logger(self) -> None: + """ + Setup the bot logger configuration + :return: None + """ + logging.basicConfig( + level=self.level, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + ) + + self.logger = self.get_logger() + self.set_level(self.level) + + def get_logger(self) -> logging.RootLogger: + """ + Return the logger instance to use for sending message + :return: the logger instance + """ + return logging.getLogger(self.name) + + def set_name(self, name: str) -> logging.RootLogger: + """ + Set the name of the logger + :param name: Name of the logger + :return: None + """ + self.name = name + self.logger = self.get_logger() + return self.logger + + def set_level(self, level) -> None: + """ + Set the level of the logger + :param level: + :return: None + """ + self.level = level + self.logger.setLevel(self.level) + + def set_format(self, log_format: str, propagate: bool = False) -> None: + """ + Set a new logging format + :return: None + """ + handler = logging.StreamHandler() + + len_handlers = len(self.logger.handlers) + if len_handlers: + self.logger.removeHandler(handler) + + handler.setFormatter(logging.Formatter(log_format)) + self.logger.addHandler(handler) + + self.logger.propagate = propagate diff --git a/freqtrade/main.py b/freqtrade/main.py index 327d89a59..d2cfc6f9f 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -1,564 +1,74 @@ #!/usr/bin/env python3 -import copy -import json +""" +Main Freqtrade bot script. +Read the documentation to know what cli arguments you need. +""" + import logging import sys -import time -import traceback -from datetime import datetime -from typing import Dict, List, Optional, Any +from typing import List -import arrow -import requests -from cachetools import cached, TTLCache +from freqtrade import (__version__) +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.freqtradebot import FreqtradeBot +from freqtrade.logger import Logger -from freqtrade import (DependencyException, OperationalException, __version__, - exchange, persistence, rpc) -from freqtrade.analyze import get_signal -from freqtrade.fiat_convert import CryptoToFiatConverter -from freqtrade.misc import (State, get_state, load_config, parse_args, - throttle, update_state) -from freqtrade.persistence import Trade -from freqtrade.strategy.strategy import Strategy - -logger = logging.getLogger('freqtrade') - -_CONF: Dict[str, Any] = {} +logger = Logger(name='freqtrade').get_logger() -def refresh_whitelist(whitelist: List[str]) -> List[str]: +def main(sysargv: List[str]) -> None: """ - Check wallet health and remove pair from whitelist if necessary - :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to trade - :return: the list of pairs the user wants to trade without the one unavailable or black_listed - """ - sanitized_whitelist = whitelist - health = exchange.get_wallet_health() - known_pairs = set() - for status in health: - pair = '{}_{}'.format(_CONF['stake_currency'], status['Currency']) - # pair is not int the generated dynamic market, or in the blacklist ... ignore it - if pair not in whitelist or pair in _CONF['exchange'].get('pair_blacklist', []): - continue - # else the pair is valid - known_pairs.add(pair) - # Market is not active - if not status['IsActive']: - sanitized_whitelist.remove(pair) - logger.info( - 'Ignoring %s from whitelist (reason: %s).', - pair, status.get('Notice') or 'wallet is not active' - ) - - # We need to remove pairs that are unknown - final_list = [x for x in sanitized_whitelist if x in known_pairs] - return final_list - - -def process_maybe_execute_buy(interval: int) -> bool: - """ - Tries to execute a buy trade in a safe way - :return: True if executed - """ - try: - # Create entity and execute trade - if create_trade(float(_CONF['stake_amount']), interval): - return True - - logger.info('Found no buy signals for whitelisted currencies. Trying again..') - return False - except DependencyException as exception: - logger.warning('Unable to create trade: %s', exception) - return False - - -def process_maybe_execute_sell(trade: Trade, interval: int) -> bool: - """ - Tries to execute a sell trade - :return: True if executed - """ - # Get order details for actual price per unit - if trade.open_order_id: - # Update trade with order values - logger.info('Found open order for %s', trade) - trade.update(exchange.get_order(trade.open_order_id)) - - if trade.is_open and trade.open_order_id is None: - # Check if we can sell our current pair - return handle_trade(trade, interval) - return False - - -def _process(interval: int, nb_assets: Optional[int] = 0) -> bool: - """ - Queries the persistence layer for open trades and handles them, - otherwise a new trade is created. - :param: nb_assets: the maximum number of pairs to be traded at the same time - :return: True if one or more trades has been created or closed, False otherwise - """ - state_changed = False - try: - # Refresh whitelist based on wallet maintenance - sanitized_list = refresh_whitelist( - gen_pair_whitelist( - _CONF['stake_currency'] - ) if nb_assets else _CONF['exchange']['pair_whitelist'] - ) - - # Keep only the subsets of pairs wanted (up to nb_assets) - final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list - _CONF['exchange']['pair_whitelist'] = final_list - - # Query trades from persistence layer - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - - # First process current opened trades - for trade in trades: - state_changed |= process_maybe_execute_sell(trade, interval) - - # Then looking for buy opportunities - if len(trades) < _CONF['max_open_trades']: - state_changed = process_maybe_execute_buy(interval) - - if 'unfilledtimeout' in _CONF: - # Check and handle any timed out open orders - check_handle_timedout(_CONF['unfilledtimeout']) - Trade.session.flush() - - except (requests.exceptions.RequestException, json.JSONDecodeError) as error: - logger.warning('%s, retrying in 30 seconds...', error) - time.sleep(30) - except OperationalException: - rpc.send_msg('*Status:* OperationalException:\n```\n{traceback}```{hint}'.format( - traceback=traceback.format_exc(), - hint='Issue `/start` if you think it is safe to restart.' - )) - logger.exception('OperationalException. Stopping trader ...') - update_state(State.STOPPED) - return state_changed - - -# FIX: 20180110, why is cancel.order unconditionally here, whereas -# it is conditionally called in the -# handle_timedout_limit_sell()? -def handle_timedout_limit_buy(trade: Trade, order: Dict) -> bool: - """Buy timeout - cancel order - :return: True if order was fully cancelled - """ - exchange.cancel_order(trade.open_order_id) - if order['remaining'] == order['amount']: - # if trade is not partially completed, just delete the trade - Trade.session.delete(trade) - # FIX? do we really need to flush, caller of - # check_handle_timedout will flush afterwards - Trade.session.flush() - logger.info('Buy order timeout for %s.', trade) - rpc.send_msg('*Timeout:* Unfilled buy order for {} cancelled'.format( - trade.pair.replace('_', '/'))) - return True - - # if trade is partially complete, edit the stake details for the trade - # and close the order - trade.amount = order['amount'] - order['remaining'] - trade.stake_amount = trade.amount * trade.open_rate - trade.open_order_id = None - logger.info('Partial buy order timeout for %s.', trade) - rpc.send_msg('*Timeout:* Remaining buy order for {} cancelled'.format( - trade.pair.replace('_', '/'))) - return False - - -# FIX: 20180110, should cancel_order() be cond. or unconditionally called? -def handle_timedout_limit_sell(trade: Trade, order: Dict) -> bool: - """ - Sell timeout - cancel order and update trade - :return: True if order was fully cancelled - """ - if order['remaining'] == order['amount']: - # if trade is not partially completed, just cancel the trade - exchange.cancel_order(trade.open_order_id) - trade.close_rate = None - trade.close_profit = None - trade.close_date = None - trade.is_open = True - trade.open_order_id = None - rpc.send_msg('*Timeout:* Unfilled sell order for {} cancelled'.format( - trade.pair.replace('_', '/'))) - logger.info('Sell order timeout for %s.', trade) - return True - - # TODO: figure out how to handle partially complete sell orders - return False - - -def check_handle_timedout(timeoutvalue: int) -> None: - """ - Check if any orders are timed out and cancel if neccessary - :param timeoutvalue: Number of minutes until order is considered timed out + This function will initiate the bot and start the trading loop. :return: None """ - timeoutthreashold = arrow.utcnow().shift(minutes=-timeoutvalue).datetime - - for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): - try: - order = exchange.get_order(trade.open_order_id) - except requests.exceptions.RequestException: - logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) - continue - ordertime = arrow.get(order['opened']) - - # Check if trade is still actually open - if int(order['remaining']) == 0: - continue - - if order['type'] == "LIMIT_BUY" and ordertime < timeoutthreashold: - handle_timedout_limit_buy(trade, order) - elif order['type'] == "LIMIT_SELL" and ordertime < timeoutthreashold: - handle_timedout_limit_sell(trade, order) - - -def execute_sell(trade: Trade, limit: float) -> None: - """ - Executes a limit sell for the given trade and limit - :param trade: Trade instance - :param limit: limit rate for the sell order - :return: None - """ - # Execute sell and update trade record - order_id = exchange.sell(str(trade.pair), limit, trade.amount) - trade.open_order_id = order_id - - fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) - profit_trade = trade.calc_profit(rate=limit) - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - profit = trade.calc_profit_percent(current_rate) - - message = """*{exchange}:* Selling -*Current Pair:* [{pair}]({pair_url}) -*Limit:* `{limit}` -*Amount:* `{amount}` -*Open Rate:* `{open_rate:.8f}` -*Current Rate:* `{current_rate:.8f}` -*Profit:* `{profit:.2f}%` - """.format( - exchange=trade.exchange, - pair=trade.pair, - pair_url=exchange.get_pair_detail_url(trade.pair), - limit=limit, - open_rate=trade.open_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - profit=round(profit * 100, 2), + arguments = Arguments( + sysargv, + 'Simple High Frequency Trading Bot for crypto currencies' ) + args = arguments.get_parsed_arg() - # For regular case, when the configuration exists - if 'stake_currency' in _CONF and 'fiat_display_currency' in _CONF: - fiat_converter = CryptoToFiatConverter() - profit_fiat = fiat_converter.convert_amount( - profit_trade, - _CONF['stake_currency'], - _CONF['fiat_display_currency'] - ) - message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ - '` / {profit_fiat:.3f} {fiat})`'.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade, - coin=_CONF['stake_currency'], - profit_fiat=profit_fiat, - fiat=_CONF['fiat_display_currency'], - ) - # Because telegram._forcesell does not have the configuration - # Ignore the FIAT value and does not show the stake_currency as well - else: - message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f})`'.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade - ) - - # Send the message - rpc.send_msg(message) - Trade.session.flush() - - -def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) -> bool: - """ - Based an earlier trade and current price and ROI configuration, decides whether bot should sell - :return True if bot should sell at current rate - """ - strategy = Strategy() - - current_profit = trade.calc_profit_percent(current_rate) - if strategy.stoploss is not None and current_profit < strategy.stoploss: - logger.debug('Stop loss hit.') - return True - - # Check if time matches and current rate is above threshold - time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 - for duration, threshold in strategy.minimal_roi.items(): - if time_diff <= duration: - return False - if current_profit > threshold: - return True - return False - - -def should_sell(trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool: - """ - This function evaluate if on the condition required to trigger a sell has been reached - if the threshold is reached and updates the trade record. - :return: True if trade should be sold, False otherwise - """ - # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) - if min_roi_reached(trade, rate, date): - logger.debug('Required profit reached. Selling..') - return True - - # Experimental: Check if the trade is profitable before selling it (avoid selling at loss) - if _CONF.get('experimental', {}).get('sell_profit_only', False): - logger.debug('Checking if trade is profitable..') - if trade.calc_profit(rate=rate) <= 0: - return False - - if sell and not buy and _CONF.get('experimental', {}).get('use_sell_signal', False): - logger.debug('Sell signal received. Selling..') - return True - - return False - - -def handle_trade(trade: Trade, interval: int) -> bool: - """ - Sells the current pair if the threshold is reached and updates the trade record. - :return: True if trade has been sold, False otherwise - """ - if not trade.is_open: - raise ValueError('attempt to handle closed trade: {}'.format(trade)) - - logger.debug('Handling %s ...', trade) - current_rate = exchange.get_ticker(trade.pair)['bid'] - - (buy, sell) = (False, False) - - if _CONF.get('experimental', {}).get('use_sell_signal'): - (buy, sell) = get_signal(trade.pair, interval) - - if should_sell(trade, current_rate, datetime.utcnow(), buy, sell): - execute_sell(trade, current_rate) - return True - - return False - - -def get_target_bid(ticker: Dict[str, float]) -> float: - """ Calculates bid target between current ask price and last price """ - if ticker['ask'] < ticker['last']: - return ticker['ask'] - balance = _CONF['bid_strategy']['ask_last_balance'] - return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) - - -def create_trade(stake_amount: float, interval: int) -> bool: - """ - Checks the implemented trading indicator(s) for a randomly picked pair, - if one pair triggers the buy_signal a new trade record gets created - :param stake_amount: amount of btc to spend - :return: True if a trade object has been created and persisted, False otherwise - """ - logger.info( - 'Checking buy signals to create a new trade with stake_amount: %f ...', - stake_amount - ) - whitelist = copy.deepcopy(_CONF['exchange']['pair_whitelist']) - # Check if stake_amount is fulfilled - if exchange.get_balance(_CONF['stake_currency']) < stake_amount: - raise DependencyException( - 'stake amount is not fulfilled (currency={})'.format(_CONF['stake_currency']) - ) - - # Remove currently opened and latest pairs from whitelist - for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): - if trade.pair in whitelist: - whitelist.remove(trade.pair) - logger.debug('Ignoring %s in pair whitelist', trade.pair) - if not whitelist: - raise DependencyException('No currency pairs in whitelist') - - # Pick pair based on StochRSI buy signals - for _pair in whitelist: - (buy, sell) = get_signal(_pair, interval) - if buy and not sell: - pair = _pair - break - else: - return False - - # Calculate amount - buy_limit = get_target_bid(exchange.get_ticker(pair)) - amount = stake_amount / buy_limit - - order_id = exchange.buy(pair, buy_limit, amount) - - fiat_converter = CryptoToFiatConverter() - stake_amount_fiat = fiat_converter.convert_amount( - stake_amount, - _CONF['stake_currency'], - _CONF['fiat_display_currency'] - ) - - # Create trade entity and return - rpc.send_msg('*{}:* Buying [{}]({}) with limit `{:.8f} ({:.6f} {}, {:.3f} {})` '.format( - exchange.get_name().upper(), - pair.replace('_', '/'), - exchange.get_pair_detail_url(pair), - buy_limit, stake_amount, _CONF['stake_currency'], - stake_amount_fiat, _CONF['fiat_display_currency'] - )) - # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL - trade = Trade( - pair=pair, - stake_amount=stake_amount, - amount=amount, - fee=exchange.get_fee(), - open_rate=buy_limit, - open_date=datetime.utcnow(), - exchange=exchange.get_name().upper(), - open_order_id=order_id - ) - Trade.session.add(trade) - Trade.session.flush() - return True - - -def init(config: dict, db_url: Optional[str] = None) -> None: - """ - Initializes all modules and updates the config - :param config: config as dict - :param db_url: database connector string for sqlalchemy (Optional) - :return: None - """ - # Initialize all modules - rpc.init(config) - persistence.init(config, db_url) - exchange.init(config) - - strategy = Strategy() - strategy.init(config) - - # Set initial application state - initial_state = config.get('initial_state') - if initial_state: - update_state(State[initial_state.upper()]) - else: - update_state(State.STOPPED) - - -@cached(TTLCache(maxsize=1, ttl=1800)) -def gen_pair_whitelist(base_currency: str, key: str = 'BaseVolume') -> List[str]: - """ - Updates the whitelist with with a dynamically generated list - :param base_currency: base currency as str - :param key: sort key (defaults to 'BaseVolume') - :return: List of pairs - """ - summaries = sorted( - (s for s in exchange.get_market_summaries() if s['MarketName'].startswith(base_currency)), - key=lambda s: s.get(key) or 0.0, - reverse=True - ) - - return [s['MarketName'].replace('-', '_') for s in summaries] - - -def cleanup() -> None: - """ - Cleanup the application state und finish all pending tasks - :return: None - """ - rpc.send_msg('*Status:* `Stopping trader...`') - logger.info('Stopping trader and cleaning up modules...') - update_state(State.STOPPED) - persistence.cleanup() - rpc.cleanup() - exit(0) - - -def main(sysargv=sys.argv[1:]) -> int: - """ - Loads and validates the config and handles the main loop - :return: None - """ - global _CONF - args = parse_args(sysargv, - 'Simple High Frequency Trading Bot for crypto currencies') - - # A subcommand has been issued + # A subcommand has been issued. + # Means if Backtesting or Hyperopt have been called we exit the bot if hasattr(args, 'func'): args.func(args) return 0 - # Initialize logger - logging.basicConfig( - level=args.loglevel, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - ) - logger.info( 'Starting freqtrade %s (loglevel=%s)', __version__, logging.getLevelName(args.loglevel) ) - # Load and validate configuration - _CONF = load_config(args.config) - - # Add the strategy file to use - _CONF.update({'strategy': args.strategy}) - - # Initialize all modules and start main loop - if args.dynamic_whitelist: - logger.info('Using dynamically generated whitelist. (--dynamic-whitelist detected)') - - # If the user ask for Dry run with a local DB instead of memory - if args.dry_run_db: - if _CONF.get('dry_run', False): - _CONF.update({'dry_run_db': True}) - logger.info( - 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite". (--dry_run_db detected)' - ) - else: - logger.info('Dry run is disabled. (--dry_run_db ignored)') - try: - init(_CONF) - old_state = None + # Load and validate configuration + configuration = Configuration(args) - while True: - new_state = get_state() - # Log state transition - if new_state != old_state: - rpc.send_msg('*Status:* `{}`'.format(new_state.name.lower())) - logger.info('Changing state to: %s', new_state.name) + # Init the bot + freqtrade = FreqtradeBot(configuration.get_config()) + + state = None + while 1: + state = freqtrade.worker(old_state=state) - if new_state == State.STOPPED: - time.sleep(1) - elif new_state == State.RUNNING: - throttle( - _process, - min_secs=_CONF['internals'].get('process_throttle_secs', 10), - nb_assets=args.dynamic_whitelist, - interval=int(_CONF.get('ticker_interval', 5)) - ) - old_state = new_state except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') except BaseException: logger.exception('Fatal exception!') finally: - cleanup() - return 0 + freqtrade.clean() + sys.exit(0) + + +def set_loggers() -> None: + """ + Set the logger level for Third party libs + :return: None + """ + logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) + logging.getLogger('telegram').setLevel(logging.INFO) if __name__ == '__main__': + set_loggers() main(sys.argv[1:]) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 91498a189..f5d045c44 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -1,30 +1,29 @@ -import argparse -import enum +""" +Various tool function for Freqtrade and scripts +""" + import json import logging -import time -import os import re from datetime import datetime -from typing import Any, Callable, Dict, List +from typing import Dict import numpy as np -from jsonschema import Draft4Validator, validate -from jsonschema.exceptions import ValidationError, best_match -from wrapt import synchronized - -from freqtrade import __version__ +from pandas import DataFrame logger = logging.getLogger(__name__) -class State(enum.Enum): - RUNNING = 0 - STOPPED = 1 - - -# Current application state -_STATE = State.STOPPED +def shorten_date(_date: str) -> str: + """ + Trim the date so it fits on small screens + """ + new_date = re.sub('seconds?', 'sec', _date) + new_date = re.sub('minutes?', 'min', new_date) + new_date = re.sub('hours?', 'h', new_date) + new_date = re.sub('days?', 'd', new_date) + new_date = re.sub('^an?', '1', new_date) + return new_date ############################################ @@ -32,8 +31,7 @@ _STATE = State.STOPPED # Matplotlib doesn't support ::datetime64, # # so we need to convert it into ::datetime # ############################################ - -def datesarray_to_datetimearray(dates): +def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray: """ Convert an pandas-array of timestamps into An numpy-array of datetimes @@ -41,13 +39,18 @@ def datesarray_to_datetimearray(dates): """ times = [] dates = dates.astype(datetime) - for i in range(0, dates.size): - date = dates[i].to_pydatetime() + for index in range(0, dates.size): + date = dates[index].to_pydatetime() times.append(date) return np.array(times) -def common_datearray(dfs): +def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray: + """ + Return dates from Dataframe + :param dfs: Dict with format pair: pair_data + :return: List of dates + """ alldates = {} for pair, pair_data in dfs.items(): dates = datesarray_to_datetimearray(pair_data['date']) @@ -61,375 +64,11 @@ def common_datearray(dfs): def file_dump_json(filename, data) -> None: - with open(filename, 'w') as fp: - json.dump(data, fp) - - -@synchronized -def update_state(state: State) -> None: """ - Updates the application state - :param state: new state - :return: None - """ - global _STATE - _STATE = state - - -@synchronized -def get_state() -> State: - """ - Gets the current application state + Dump JSON data into a file + :param filename: file to create + :param data: JSON Data to save :return: """ - return _STATE - - -def load_config(path: str) -> Dict: - """ - Loads a config file from the given path - :param path: path as str - :return: configuration as dictionary - """ - with open(path) as file: - conf = json.load(file) - if 'internals' not in conf: - conf['internals'] = {} - logger.info('Validating configuration ...') - try: - validate(conf, CONF_SCHEMA) - return conf - except ValidationError as exception: - logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception) - raise ValidationError( - best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message - ) - - -def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: - """ - Throttles the given callable that it - takes at least `min_secs` to finish execution. - :param func: Any callable - :param min_secs: minimum execution time in seconds - :return: Any - """ - start = time.time() - result = func(*args, **kwargs) - end = time.time() - duration = max(min_secs - (end - start), 0.0) - logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) - time.sleep(duration) - return result - - -def common_args_parser(description: str): - """ - Parses given common arguments and returns them as a parsed object. - """ - parser = argparse.ArgumentParser( - description=description - ) - parser.add_argument( - '-v', '--verbose', - help='be verbose', - action='store_const', - dest='loglevel', - const=logging.DEBUG, - default=logging.INFO, - ) - parser.add_argument( - '--version', - action='version', - version='%(prog)s {}'.format(__version__), - ) - parser.add_argument( - '-c', '--config', - help='specify configuration file (default: %(default)s)', - dest='config', - default='config.json', - type=str, - metavar='PATH', - ) - parser.add_argument( - '-d', '--datadir', - help='path to backtest data (default: %(default)s', - dest='datadir', - default=os.path.join('freqtrade', 'tests', 'testdata'), - type=str, - metavar='PATH', - ) - parser.add_argument( - '-s', '--strategy', - help='specify strategy file (default: %(default)s)', - dest='strategy', - default='default_strategy', - type=str, - metavar='PATH', - ) - return parser - - -def parse_args(args: List[str], description: str): - """ - Parses given arguments and returns an argparse Namespace instance. - Returns None if a sub command has been selected and executed. - """ - parser = common_args_parser(description) - parser.add_argument( - '--dry-run-db', - help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \ - instead of memory DB. Work only if dry_run is enabled.', - action='store_true', - dest='dry_run_db', - ) - parser.add_argument( - '--dynamic-whitelist', - help='dynamically generate and update whitelist \ - based on 24h BaseVolume (Default 20 currencies)', # noqa - dest='dynamic_whitelist', - const=20, - type=int, - metavar='INT', - nargs='?', - ) - - build_subcommands(parser) - return parser.parse_args(args) - - -def scripts_options(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - '-p', '--pair', - help='Show profits for only this pairs. Pairs are comma-separated.', - dest='pair', - default=None - ) - - -def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - '-i', '--ticker-interval', - help='specify ticker interval in minutes (1, 5, 30, 60, 1440)', - dest='ticker_interval', - type=int, - metavar='INT', - ) - parser.add_argument( - '--realistic-simulation', - help='uses max_open_trades from config to simulate real world limitations', - action='store_true', - dest='realistic_simulation', - ) - parser.add_argument( - '--timerange', - help='Specify what timerange of data to use.', - default=None, - type=str, - dest='timerange', - ) - - -def backtesting_options(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - '-l', '--live', - action='store_true', - dest='live', - help='using live data', - ) - parser.add_argument( - '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \ - Use it if you want to run your backtesting with up-to-date data.', - action='store_true', - dest='refresh_pairs', - ) - parser.add_argument( - '--export', - help='Export backtest results, argument are: trades\ - Example --export=trades', - type=str, - default=None, - dest='export', - ) - - -def hyperopt_options(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - '-e', '--epochs', - help='specify number of epochs (default: %(default)d)', - dest='epochs', - default=100, - type=int, - metavar='INT', - ) - parser.add_argument( - '--use-mongodb', - help='parallelize evaluations with mongodb (requires mongod in PATH)', - dest='mongodb', - action='store_true', - ) - parser.add_argument( - '-s', '--spaces', - help='Specify which parameters to hyperopt. Space separate list. \ - Default: %(default)s', - choices=['all', 'buy', 'roi', 'stoploss'], - default='all', - nargs='+', - dest='spaces', - ) - - -def parse_timerange(text): - if text is None: - return None - syntax = [(r'^-(\d{8})$', (None, 'date')), - (r'^(\d{8})-$', ('date', None)), - (r'^(\d{8})-(\d{8})$', ('date', 'date')), - (r'^(-\d+)$', (None, 'line')), - (r'^(\d+)-$', ('line', None)), - (r'^(\d+)-(\d+)$', ('index', 'index'))] - for rex, stype in syntax: - # Apply the regular expression to text - match = re.match(rex, text) - if match: # Regex has matched - rvals = match.groups() - index = 0 - start = None - stop = None - if stype[0]: - start = rvals[index] - if stype[0] != 'date': - start = int(start) - index += 1 - if stype[1]: - stop = rvals[index] - if stype[1] != 'date': - stop = int(stop) - return (stype, start, stop) - raise Exception('Incorrect syntax for timerange "%s"' % text) - - -def build_subcommands(parser: argparse.ArgumentParser) -> None: - """ Builds and attaches all subcommands """ - from freqtrade.optimize import backtesting, hyperopt - - subparsers = parser.add_subparsers(dest='subparser') - - # Add backtesting subcommand - backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module') - backtesting_cmd.set_defaults(func=backtesting.start) - optimizer_shared_options(backtesting_cmd) - backtesting_options(backtesting_cmd) - - # Add hyperopt subcommand - hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') - hyperopt_cmd.set_defaults(func=hyperopt.start) - optimizer_shared_options(hyperopt_cmd) - hyperopt_options(hyperopt_cmd) - - -# Required json-schema for user specified config -CONF_SCHEMA = { - 'type': 'object', - 'properties': { - 'max_open_trades': {'type': 'integer', 'minimum': 0}, - 'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, - 'stake_amount': {'type': 'number', 'minimum': 0.0005}, - 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', - 'CLP', 'CNY', 'CZK', 'DKK', - 'EUR', 'GBP', 'HKD', 'HUF', - 'IDR', 'ILS', 'INR', 'JPY', - 'KRW', 'MXN', 'MYR', 'NOK', - 'NZD', 'PHP', 'PKR', 'PLN', - 'RUB', 'SEK', 'SGD', 'THB', - 'TRY', 'TWD', 'ZAR', 'USD']}, - 'dry_run': {'type': 'boolean'}, - 'minimal_roi': { - 'type': 'object', - 'patternProperties': { - '^[0-9.]+$': {'type': 'number'} - }, - 'minProperties': 1 - }, - 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, - 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, - 'bid_strategy': { - 'type': 'object', - 'properties': { - 'ask_last_balance': { - 'type': 'number', - 'minimum': 0, - 'maximum': 1, - 'exclusiveMaximum': False - }, - }, - 'required': ['ask_last_balance'] - }, - 'exchange': {'$ref': '#/definitions/exchange'}, - 'experimental': { - 'type': 'object', - 'properties': { - 'use_sell_signal': {'type': 'boolean'}, - 'sell_profit_only': {'type': 'boolean'} - } - }, - 'telegram': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'token': {'type': 'string'}, - 'chat_id': {'type': 'string'}, - }, - 'required': ['enabled', 'token', 'chat_id'] - }, - 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, - 'internals': { - 'type': 'object', - 'properties': { - 'process_throttle_secs': {'type': 'number'}, - 'interval': {'type': 'integer'} - } - } - }, - 'definitions': { - 'exchange': { - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'key': {'type': 'string'}, - 'secret': {'type': 'string'}, - 'pair_whitelist': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' - }, - 'uniqueItems': True - }, - 'pair_blacklist': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' - }, - 'uniqueItems': True - } - }, - 'required': ['name', 'key', 'secret', 'pair_whitelist'] - } - }, - 'anyOf': [ - {'required': ['exchange']} - ], - 'required': [ - 'max_open_trades', - 'stake_currency', - 'stake_amount', - 'fiat_display_currency', - 'dry_run', - 'bid_strategy', - 'telegram' - ] -} + with open(filename, 'w') as fp: + json.dump(data, fp, default=str) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 931c720ee..a26744691 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,22 +1,20 @@ # pragma pylint: disable=missing-docstring -import logging +import gzip import json import os -from typing import Optional, List, Dict -from pandas import DataFrame -from freqtrade.exchange import get_ticker_history -from freqtrade.analyze import populate_indicators, parse_ticker_dataframe +from typing import Optional, List, Dict, Tuple from freqtrade import misc +from freqtrade.exchange import get_ticker_history +from freqtrade.logger import Logger from user_data.hyperopt_conf import hyperopt_optimize_conf -import gzip -logger = logging.getLogger(__name__) +logger = Logger(name=__name__).get_logger() -def trim_tickerlist(tickerlist, timerange): - (stype, start, stop) = timerange +def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) -> List[Dict]: + stype, start, stop = timerange if stype == (None, 'line'): return tickerlist[stop:] elif stype == ('line', None): @@ -27,7 +25,10 @@ def trim_tickerlist(tickerlist, timerange): return tickerlist -def load_tickerdata_file(datadir, pair, ticker_interval, timerange=None): +def load_tickerdata_file( + datadir: str, pair: str, + ticker_interval: int, + timerange: Optional[Tuple[Tuple, int, int]] = None) -> Optional[List[Dict]]: """ Load a pair from file, :return dict OR empty if unsuccesful @@ -57,12 +58,12 @@ def load_tickerdata_file(datadir, pair, ticker_interval, timerange=None): return pairdata -def load_data(datadir: str, ticker_interval: int, pairs: Optional[List[str]] = None, - refresh_pairs: Optional[bool] = False, timerange=None) -> Dict[str, List]: +def load_data(datadir: str, ticker_interval: int, + pairs: Optional[List[str]] = None, + refresh_pairs: Optional[bool] = False, + timerange: Optional[Tuple[Tuple, int, int]] = None) -> Dict[str, List]: """ Loads ticker history data for the given parameters - :param ticker_interval: ticker interval in minutes - :param pairs: list of pairs :return: dict """ result = {} @@ -85,21 +86,13 @@ def load_data(datadir: str, ticker_interval: int, pairs: Optional[List[str]] = N return result -def tickerdata_to_dataframe(data): - preprocessed = preprocess(data) - return preprocessed - - -def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: - """Creates a dataframe and populates indicators for given ticker data""" - return {pair: populate_indicators(parse_ticker_dataframe(pair_data)) - for pair, pair_data in tickerdata.items()} - - def make_testdata_path(datadir: str) -> str: """Return the path where testdata files are stored""" - return datadir or os.path.abspath(os.path.join(os.path.dirname(__file__), - '..', 'tests', 'testdata')) + return datadir or os.path.abspath( + os.path.join( + os.path.dirname(__file__), '..', 'tests', 'testdata' + ) + ) def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool: @@ -108,19 +101,15 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool: try: download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval) except BaseException: - logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format( - pair=pair, - interval=ticker_interval, - )) + logger.info( + 'Failed to download the pair: "%s", Interval: %s min', + pair, + ticker_interval + ) return False return True -def file_dump_json(filename, data): - with open(filename, "wt") as fp: - json.dump(data, fp) - - # FIX: 20180110, suggest rename interval to tick_interval def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool: """ @@ -131,10 +120,11 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> """ path = make_testdata_path(datadir) - logger.info('Download the pair: "{pair}", Interval: {interval} min'.format( - pair=pair, - interval=interval, - )) + logger.info( + 'Download the pair: "%s", Interval: %s min', + pair, + interval + ) filepair = pair.replace("-", "_") filename = os.path.join(path, '{pair}-{interval}.json'.format( @@ -143,10 +133,10 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> )) if os.path.isfile(filename): - with open(filename, "rt") as fp: - data = json.load(fp) - logger.debug("Current Start: {}".format(data[1]['T'])) - logger.debug("Current End: {}".format(data[-1:][0]['T'])) + with open(filename, "rt") as file: + data = json.load(file) + logger.debug("Current Start: %s", data[1]['T']) + logger.debug("Current End: %s", data[-1:][0]['T']) else: data = [] logger.debug("Current Start: None") @@ -156,8 +146,8 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> for row in new_data: if row not in data: data.append(row) - logger.debug("New Start: {}".format(data[1]['T'])) - logger.debug("New End: {}".format(data[-1:][0]['T'])) + logger.debug("New Start: %s", data[1]['T']) + logger.debug("New End: %s", data[-1:][0]['T']) data = sorted(data, key=lambda data: data['T']) misc.file_dump_json(filename, data) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a0dc8a789..d8af47326 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1,231 +1,312 @@ -# pragma pylint: disable=missing-docstring,W0212 +# pragma pylint: disable=missing-docstring, W0212, too-many-arguments -import logging -from typing import Dict, Tuple +""" +This module contains the backtesting logic +""" +from argparse import Namespace +from typing import Dict, Tuple, Any, List, Optional import arrow from pandas import DataFrame, Series from tabulate import tabulate -import freqtrade.misc as misc import freqtrade.optimize as optimize from freqtrade import exchange -from freqtrade.analyze import populate_buy_trend, populate_sell_trend +from freqtrade.analyze import Analyze +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration from freqtrade.exchange import Bittrex -from freqtrade.main import should_sell +from freqtrade.logger import Logger +from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade -from freqtrade.strategy.strategy import Strategy - -logger = logging.getLogger(__name__) -def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: +class Backtesting(object): """ - Get the maximum timeframe for the given backtest data - :param data: dictionary with preprocessed backtesting data - :return: tuple containing min_date, max_date - """ - all_dates = Series([]) - for pair_data in data.values(): - all_dates = all_dates.append(pair_data['date']) - all_dates.sort_values(inplace=True) - return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) + Backtesting class, this class contains all the logic to run a backtest + To run a backtest: + backtesting = Backtesting(config) + backtesting.start() + """ + def __init__(self, config: Dict[str, Any]) -> None: -def generate_text_table( - data: Dict[str, Dict], results: DataFrame, stake_currency) -> str: - """ - Generates and returns a text table for the given backtest data and the results dataframe - :return: pretty printed table with tabulate as str - """ - floatfmt = ('s', 'd', '.2f', '.8f', '.1f') - tabular_data = [] - headers = ['pair', 'buy count', 'avg profit %', - 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] - for pair in data: - result = results[results.currency == pair] + # Init the logger + self.logging = Logger(name=__name__, level=config['loglevel']) + self.logger = self.logging.get_logger() + self.config = config + self.analyze = None + self.ticker_interval = None + self.tickerdata_to_dataframe = None + self.populate_buy_trend = None + self.populate_sell_trend = None + self._init() + + def _init(self) -> None: + """ + Init objects required for backtesting + :return: None + """ + self.analyze = Analyze(self.config) + self.ticker_interval = self.analyze.strategy.ticker_interval + self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe + self.populate_buy_trend = self.analyze.populate_buy_trend + self.populate_sell_trend = self.analyze.populate_sell_trend + exchange._API = Bittrex({'key': '', 'secret': ''}) + + @staticmethod + def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum timeframe for the given backtest data + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + all_dates = Series([]) + for pair_data in data.values(): + all_dates = all_dates.append(pair_data['date']) + all_dates.sort_values(inplace=True) + return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) + + def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: + """ + Generates and returns a text table for the given backtest data and the results dataframe + :return: pretty printed table with tabulate as str + """ + stake_currency = self.config.get('stake_currency') + + floatfmt = ('s', 'd', '.2f', '.8f', '.1f') + tabular_data = [] + headers = ['pair', 'buy count', 'avg profit %', + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] + for pair in data: + result = results[results.currency == pair] + tabular_data.append([ + pair, + len(result.index), + result.profit_percent.mean() * 100.0, + result.profit_BTC.sum(), + result.duration.mean(), + len(result[result.profit_BTC > 0]), + len(result[result.profit_BTC < 0]) + ]) + + # Append Total tabular_data.append([ - pair, - len(result.index), - result.profit_percent.mean() * 100.0, - result.profit_BTC.sum(), - result.duration.mean(), - len(result[result.profit_BTC > 0]), - len(result[result.profit_BTC < 0]) + 'TOTAL', + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_BTC.sum(), + results.duration.mean(), + len(results[results.profit_BTC > 0]), + len(results[results.profit_BTC < 0]) ]) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) - # Append Total - tabular_data.append([ - 'TOTAL', - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), - results.duration.mean(), - len(results[results.profit_BTC > 0]), - len(results[results.profit_BTC < 0]) - ]) - return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) + def _get_sell_trade_entry( + self, pair: str, buy_row: DataFrame, + partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[Tuple]: + stake_amount = args['stake_amount'] + max_open_trades = args.get('max_open_trades', 0) + trade = Trade( + open_rate=buy_row.close, + open_date=buy_row.date, + stake_amount=stake_amount, + amount=stake_amount / buy_row.open, + fee=exchange.get_fee() + ) -def get_sell_trade_entry(pair, buy_row, partial_ticker, trade_count_lock, args): - stake_amount = args['stake_amount'] - max_open_trades = args.get('max_open_trades', 0) - trade = Trade(open_rate=buy_row.close, - open_date=buy_row.date, - stake_amount=stake_amount, - amount=stake_amount / buy_row.open, - fee=exchange.get_fee() - ) - - # calculate win/lose forwards from buy point - for sell_row in partial_ticker: - if max_open_trades > 0: - # Increase trade_count_lock for every iteration - trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 - - buy_signal = sell_row.buy - if should_sell(trade, sell_row.close, sell_row.date, buy_signal, sell_row.sell): - return sell_row, (pair, - trade.calc_profit_percent(rate=sell_row.close), - trade.calc_profit(rate=sell_row.close), - (sell_row.date - buy_row.date).seconds // 60 - ), sell_row.date - return None - - -def backtest(args) -> DataFrame: - """ - Implements backtesting functionality - :param args: a dict containing: - stake_amount: btc amount to use for each trade - processed: a processed dictionary with format {pair, data} - max_open_trades: maximum number of concurrent trades (default: 0, disabled) - realistic: do we try to simulate realistic trades? (default: True) - sell_profit_only: sell if profit only - use_sell_signal: act on sell-signal - :return: DataFrame - """ - headers = ['date', 'buy', 'open', 'close', 'sell'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) - record = args.get('record', None) - records = [] - trades = [] - trade_count_lock: dict = {} - exchange._API = Bittrex({'key': '', 'secret': ''}) - for pair, pair_data in processed.items(): - pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - - ticker_data = populate_sell_trend(populate_buy_trend(pair_data))[headers] - ticker = [x for x in ticker_data.itertuples()] - - lock_pair_until = None - for index, row in enumerate(ticker): - if row.buy == 0 or row.sell == 1: - continue # skip rows where no buy signal or that would immediately sell off - - if realistic: - if lock_pair_until is not None and row.date <= lock_pair_until: - continue + # calculate win/lose forwards from buy point + for sell_row in partial_ticker: if max_open_trades > 0: - # Check if max_open_trades has already been reached for the given date - if not trade_count_lock.get(row.date, 0) < max_open_trades: - continue - trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + # Increase trade_count_lock for every iteration + trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 - ret = get_sell_trade_entry(pair, row, ticker[index+1:], trade_count_lock, args) - if ret: - row2, trade_entry, next_date = ret - lock_pair_until = next_date - trades.append(trade_entry) - if record: - # Note, need to be json.dump friendly - # record a tuple of pair, current_profit_percent, - # entry-date, duration - records.append((pair, trade_entry[1], - row.date.strftime('%s'), - row2.date.strftime('%s'), - row.date, trade_entry[3])) - # For now export inside backtest(), maybe change so that backtest() - # returns a tuple like: (dataframe, records, logs, etc) - if record and record.find('trades') >= 0: - logger.info('Dumping backtest results') - misc.file_dump_json('backtest-result.json', records) - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] - return DataFrame.from_records(trades, columns=labels) + buy_signal = sell_row.buy + if self.analyze.should_sell(trade, sell_row.close, sell_row.date, buy_signal, + sell_row.sell): + return \ + sell_row, \ + ( + pair, + trade.calc_profit_percent(rate=sell_row.close), + trade.calc_profit(rate=sell_row.close), + (sell_row.date - buy_row.date).seconds // 60 + ), \ + sell_row.date + return None + + def backtest(self, args: Dict) -> DataFrame: + """ + Implements backtesting functionality + + NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. + Of course try to not have ugly code. By some accessor are sometime slower than functions. + Avoid, logging on this method + + :param args: a dict containing: + stake_amount: btc amount to use for each trade + processed: a processed dictionary with format {pair, data} + max_open_trades: maximum number of concurrent trades (default: 0, disabled) + realistic: do we try to simulate realistic trades? (default: True) + sell_profit_only: sell if profit only + use_sell_signal: act on sell-signal + :return: DataFrame + """ + headers = ['date', 'buy', 'open', 'close', 'sell'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', False) + record = args.get('record', None) + records = [] + trades = [] + trade_count_lock = {} + for pair, pair_data in processed.items(): + pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run + + ticker_data = self.populate_sell_trend(self.populate_buy_trend(pair_data))[headers] + ticker = [x for x in ticker_data.itertuples()] + + lock_pair_until = None + for index, row in enumerate(ticker): + if row.buy == 0 or row.sell == 1: + continue # skip rows where no buy signal or that would immediately sell off + + if realistic: + if lock_pair_until is not None and row.date <= lock_pair_until: + continue + if max_open_trades > 0: + # Check if max_open_trades has already been reached for the given date + if not trade_count_lock.get(row.date, 0) < max_open_trades: + continue + + trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + + ret = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_count_lock, args) + + if ret: + row2, trade_entry, next_date = ret + lock_pair_until = next_date + trades.append(trade_entry) + if record: + # Note, need to be json.dump friendly + # record a tuple of pair, current_profit_percent, + # entry-date, duration + records.append((pair, trade_entry[1], + row.date.strftime('%s'), + row2.date.strftime('%s'), + row.date, trade_entry[3])) + # For now export inside backtest(), maybe change so that backtest() + # returns a tuple like: (dataframe, records, logs, etc) + if record and record.find('trades') >= 0: + self.logger.info('Dumping backtest results') + file_dump_json('backtest-result.json', records) + labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + return DataFrame.from_records(trades, columns=labels) + + def start(self) -> None: + """ + Run a backtesting end-to-end + :return: None + """ + data = {} + pairs = self.config['exchange']['pair_whitelist'] + self.logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + self.logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + + if self.config.get('live'): + self.logger.info('Downloading data for all pairs in whitelist ...') + for pair in pairs: + data[pair] = exchange.get_ticker_history(pair, self.ticker_interval) + else: + self.logger.info('Using local backtesting data (using whitelist in given config) ...') + + timerange = Arguments.parse_timerange(self.config.get('timerange')) + data = optimize.load_data( + self.config['datadir'], + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=self.config.get('refresh_pairs', False), + timerange=timerange + ) + + # Ignore max_open_trades in backtesting, except realistic flag was passed + if self.config.get('realistic_simulation', False): + max_open_trades = self.config['max_open_trades'] + else: + self.logger.info('Ignoring max_open_trades (realistic_simulation not set) ...') + max_open_trades = 0 + + preprocessed = self.tickerdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) + self.logger.info( + 'Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + + # Execute backtest and print results + sell_profit_only = self.config.get('experimental', {}).get('sell_profit_only', False) + use_sell_signal = self.config.get('experimental', {}).get('use_sell_signal', False) + results = self.backtest( + { + 'stake_amount': self.config.get('stake_amount'), + 'processed': preprocessed, + 'max_open_trades': max_open_trades, + 'realistic': self.config.get('realistic_simulation', False), + 'sell_profit_only': sell_profit_only, + 'use_sell_signal': use_sell_signal, + 'record': self.config.get('export') + } + ) + + self.logging.set_format('%(message)s') + self.logger.info( + '\n==================================== ' + 'BACKTESTING REPORT' + ' ====================================\n' + '%s', + self._generate_text_table( + data, + results + ) + ) -def start(args): +def setup_configuration(args: Namespace) -> Dict[str, Any]: + """ + Prepare the configuration for the backtesting + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args) + config = configuration.get_config() + + # Ensure we do not use Exchange credentials + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + + return config + + +def start(args: Namespace) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + # Initialize logger - logging.basicConfig( - level=args.loglevel, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - ) + logger = Logger(name=__name__).get_logger() + logger.info('Starting freqtrade in Backtesting mode') - exchange._API = Bittrex({'key': '', 'secret': ''}) + # Initialize configuration + config = setup_configuration(args) - logger.info('Using config: %s ...', args.config) - config = misc.load_config(args.config) - - # If -i/--ticker-interval is use we override the configuration parameter - # (that will override the strategy configuration) - if args.ticker_interval: - config.update({'ticker_interval': args.ticker_interval}) - - # init the strategy to use - config.update({'strategy': args.strategy}) - strategy = Strategy() - strategy.init(config) - - logger.info('Using ticker_interval: %d ...', strategy.ticker_interval) - - data = {} - pairs = config['exchange']['pair_whitelist'] - logger.info('Using stake_currency: %s ...', config['stake_currency']) - logger.info('Using stake_amount: %s ...', config['stake_amount']) - - if args.live: - logger.info('Downloading data for all pairs in whitelist ...') - for pair in pairs: - data[pair] = exchange.get_ticker_history(pair, strategy.ticker_interval) - else: - logger.info('Using local backtesting data (using whitelist in given config) ...') - - timerange = misc.parse_timerange(args.timerange) - data = optimize.load_data(args.datadir, - pairs=pairs, - ticker_interval=strategy.ticker_interval, - refresh_pairs=args.refresh_pairs, - timerange=timerange) - max_open_trades = 0 - if args.realistic_simulation: - logger.info('Using max_open_trades: %s ...', config['max_open_trades']) - max_open_trades = config['max_open_trades'] - - # Monkey patch config - from freqtrade import main - main._CONF = config - - preprocessed = optimize.tickerdata_to_dataframe(data) - # Print timeframe - min_date, max_date = get_timeframe(preprocessed) - logger.info('Measuring data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date-min_date).days) - # Execute backtest and print results - sell_profit_only = config.get('experimental', {}).get('sell_profit_only', False) - use_sell_signal = config.get('experimental', {}).get('use_sell_signal', False) - results = backtest({'stake_amount': config['stake_amount'], - 'processed': preprocessed, - 'max_open_trades': max_open_trades, - 'realistic': args.realistic_simulation, - 'sell_profit_only': sell_profit_only, - 'use_sell_signal': use_sell_signal, - 'record': args.export - }) - logger.info( - '\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa - generate_text_table(data, results, config['stake_currency']) - ) + # Initialize backtesting object + backtesting = Backtesting(config) + backtesting.start() diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8138f4caa..7dcd46fd2 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -1,5 +1,8 @@ -# pragma pylint: disable=missing-docstring,W0212,W0603 +# pragma pylint: disable=too-many-instance-attributes, pointless-string-statement +""" +This module contains the hyperopt logic +""" import json import logging @@ -7,6 +10,7 @@ import os import pickle import signal import sys +from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter @@ -19,431 +23,459 @@ from hyperopt.mongoexp import MongoTrials from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib -# Monkey patch config -from freqtrade import main # noqa; noqa -from freqtrade import exchange, misc, optimize -from freqtrade.exchange import Bittrex -from freqtrade.misc import load_config -from freqtrade.optimize import backtesting -from freqtrade.optimize.backtesting import backtest -from freqtrade.strategy.strategy import Strategy +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.logger import Logger +from freqtrade.optimize import load_data +from freqtrade.optimize.backtesting import Backtesting from user_data.hyperopt_conf import hyperopt_optimize_conf -# Remove noisy log messages -logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) -logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) -logger = logging.getLogger(__name__) - -# set TARGET_TRADES to suit your number concurrent trades so its realistic to the number of days -TARGET_TRADES = 600 -TOTAL_TRIES = 0 -_CURRENT_TRIES = 0 -CURRENT_BEST_LOSS = 100 - -# max average trade duration in minutes -# if eval ends with higher value, we consider it a failed eval -MAX_ACCEPTED_TRADE_DURATION = 300 - -# this is expexted avg profit * expected trade count -# for example 3.5%, 1100 trades, EXPECTED_MAX_PROFIT = 3.85 -# check that the reported Σ% values do not exceed this! -EXPECTED_MAX_PROFIT = 3.0 - -# Configuration and data used by hyperopt -PROCESSED = None # optimize.preprocess(optimize.load_data()) -OPTIMIZE_CONFIG = hyperopt_optimize_conf() - -# Hyperopt Trials -TRIALS_FILE = os.path.join('user_data', 'hyperopt_trials.pickle') -TRIALS = Trials() - -main._CONF = OPTIMIZE_CONFIG - - -def populate_indicators(dataframe: DataFrame) -> DataFrame: +class Hyperopt(Backtesting): """ - Adds several different TA indicators to the given DataFrame + Hyperopt class, this class contains all the logic to run a hyperopt simulation + + To run a backtest: + hyperopt = Hyperopt(config) + hyperopt.start() """ - dataframe['adx'] = ta.ADX(dataframe) - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - dataframe['cci'] = ta.CCI(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['roc'] = ta.ROC(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabolic - dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] + def __init__(self, config: Dict[str, Any]) -> None: - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ + super().__init__(config) - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ + # Rename the logging to display Hyperopt file instead of Backtesting + self.logging = Logger(name=__name__, level=config['loglevel']) + self.logger = self.logging.get_logger() - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ + # set TARGET_TRADES to suit your number concurrent trades so its realistic + # to the number of days + self.target_trades = 600 + self.total_tries = config.get('epochs', 0) + self.current_tries = 0 + self.current_best_loss = 100 - # Chart type - # ------------------------------------ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] + # max average trade duration in minutes + # if eval ends with higher value, we consider it a failed eval + self.max_accepted_trade_duration = 300 - return dataframe + # this is expexted avg profit * expected trade count + # for example 3.5%, 1100 trades, self.expected_max_profit = 3.85 + # check that the reported Σ% values do not exceed this! + self.expected_max_profit = 3.0 + # Configuration and data used by hyperopt + self.processed = None -def save_trials(trials, trials_path=TRIALS_FILE): - """Save hyperopt trials to file""" - logger.info('Saving Trials to \'{}\''.format(trials_path)) - pickle.dump(trials, open(trials_path, 'wb')) + # Hyperopt Trials + self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') + self.trials = Trials() + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + """ + dataframe['adx'] = ta.ADX(dataframe) + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + dataframe['cci'] = ta.CCI(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['roc'] = ta.ROC(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + # SAR Parabolic + dataframe['sar'] = ta.SAR(dataframe) + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] -def read_trials(trials_path=TRIALS_FILE): - """Read hyperopt trials file""" - logger.info('Reading Trials from \'{}\''.format(trials_path)) - trials = pickle.load(open(trials_path, 'rb')) - os.remove(trials_path) - return trials + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ -def log_trials_result(trials): - vals = json.dumps(trials.best_trial['misc']['vals'], indent=4) - results = trials.best_trial['result']['result'] - logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ - -def log_results(results): - """ log results if it is better than any previous evaluation """ - global CURRENT_BEST_LOSS - - if results['loss'] < CURRENT_BEST_LOSS: - CURRENT_BEST_LOSS = results['loss'] - logger.info('{:5d}/{}: {}. Loss {:.5f}'.format( - results['current_tries'], - results['total_tries'], - results['result'], - results['loss'])) - else: - print('.', end='') - sys.stdout.flush() - - -def calculate_loss(total_profit: float, trade_count: int, trade_duration: float): - """ objective function, returns smaller number for more optimal results """ - trade_loss = 1 - 0.25 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.8) - profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) - duration_loss = 0.4 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1) - return trade_loss + profit_loss + duration_loss - - -def generate_roi_table(params) -> Dict[int, float]: - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - -def roi_space() -> Dict[str, Any]: - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } - - -def stoploss_space() -> Dict[str, Any]: - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } - - -def indicator_space() -> Dict[str, Any]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return { - 'macd_below_zero': hp.choice('macd_below_zero', [ - {'enabled': False}, - {'enabled': True} - ]), - 'mfi': hp.choice('mfi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} - ]), - 'fastd': hp.choice('fastd', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} - ]), - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} - ]), - 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'over_sar': hp.choice('over_sar', [ - {'enabled': False}, - {'enabled': True} - ]), - 'green_candle': hp.choice('green_candle', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_sma': hp.choice('uptrend_sma', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'lower_bb_tema'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema3_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'ht_sine'}, - {'type': 'heiken_reversal_bull'}, - {'type': 'di_cross'}, - ]), - } - - -def has_space(spaces, space): - if space in spaces or 'all' in spaces: - return True - return False - - -def hyperopt_space(selected_spaces: str) -> Dict[str, Any]: - spaces = {} - if has_space(selected_spaces, 'buy'): - spaces = {**spaces, **indicator_space()} - if has_space(selected_spaces, 'roi'): - spaces = {**spaces, **roi_space()} - if has_space(selected_spaces, 'stoploss'): - spaces = {**spaces, **stoploss_space()} - return spaces - - -def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: - conditions.append(dataframe['ema50'] > dataframe['ema100']) - if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: - conditions.append(dataframe['macd'] < 0) - if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: - conditions.append(dataframe['ema5'] > dataframe['ema10']) - if 'mfi' in params and params['mfi']['enabled']: - conditions.append(dataframe['mfi'] < params['mfi']['value']) - if 'fastd' in params and params['fastd']['enabled']: - conditions.append(dataframe['fastd'] < params['fastd']['value']) - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'rsi' in params and params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) - if 'over_sar' in params and params['over_sar']['enabled']: - conditions.append(dataframe['close'] > dataframe['sar']) - if 'green_candle' in params and params['green_candle']['enabled']: - conditions.append(dataframe['close'] > dataframe['open']) - if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: - prevsma = dataframe['sma'].shift(1) - conditions.append(dataframe['sma'] > prevsma) - - # TRIGGERS - triggers = { - 'lower_bb': ( - dataframe['close'] < dataframe['bb_lowerband'] - ), - 'lower_bb_tema': ( - dataframe['tema'] < dataframe['bb_lowerband'] - ), - 'faststoch10': (qtpylib.crossed_above( - dataframe['fastd'], 10.0 - )), - 'ao_cross_zero': (qtpylib.crossed_above( - dataframe['ao'], 0.0 - )), - 'ema3_cross_ema10': (qtpylib.crossed_above( - dataframe['ema3'], dataframe['ema10'] - )), - 'macd_cross_signal': (qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )), - 'sar_reversal': (qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )), - 'ht_sine': (qtpylib.crossed_above( - dataframe['htleadsine'], dataframe['htsine'] - )), - 'heiken_reversal_bull': ( - (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & - (dataframe['ha_low'] == dataframe['ha_open']) - ), - 'di_cross': (qtpylib.crossed_above( - dataframe['plus_di'], dataframe['minus_di'] - )), - } - conditions.append(triggers.get(params['trigger']['type'])) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 + # Chart type + # ------------------------------------ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] return dataframe - return populate_buy_trend + def save_trials(self) -> None: + """ + Save hyperopt trials to file + """ + self.logger.info('Saving Trials to \'%s\'', self.trials_file) + pickle.dump(self.trials, open(self.trials_file, 'wb')) + def read_trials(self) -> Trials: + """ + Read hyperopt trials file + """ + self.logger.info('Reading Trials from \'%s\'', self.trials_file) + trials = pickle.load(open(self.trials_file, 'rb')) + os.remove(self.trials_file) + return trials -def generate_optimizer(args): - def optimizer(params): - global _CURRENT_TRIES + def log_trials_result(self) -> None: + """ + Display Best hyperopt result + """ + vals = json.dumps(self.trials.best_trial['misc']['vals'], indent=4) + results = self.trials.best_trial['result']['result'] + self.logger.info('Best result:\n%s\nwith values:\n%s', results, vals) - strategy = Strategy() - if has_space(args.spaces, 'roi'): - strategy.minimal_roi = generate_roi_table(params) + def log_results(self, results) -> None: + """ + Log results if it is better than any previous evaluation + """ + if results['loss'] < self.current_best_loss: + self.current_best_loss = results['loss'] + log_msg = '{:5d}/{}: {}. Loss {:.5f}'.format( + results['current_tries'], + results['total_tries'], + results['result'], + results['loss'] + ) + self.logger.info(log_msg) + else: + print('.', end='') + sys.stdout.flush() - if has_space(args.spaces, 'buy'): - backtesting.populate_buy_trend = buy_strategy_generator(params) + def calculate_loss(self, total_profit: float, trade_count: int, trade_duration: float) -> float: + """ + Objective function, returns smaller number for more optimal results + """ + trade_loss = 1 - 0.25 * exp(-(trade_count - self.target_trades) ** 2 / 10 ** 5.8) + profit_loss = max(0, 1 - total_profit / self.expected_max_profit) + duration_loss = 0.4 * min(trade_duration / self.max_accepted_trade_duration, 1) + return trade_loss + profit_loss + duration_loss - if has_space(args.spaces, 'stoploss'): - strategy.stoploss = params['stoploss'] + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table thqt will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - results = backtest({'stake_amount': OPTIMIZE_CONFIG['stake_amount'], - 'processed': PROCESSED, - 'realistic': args.realistic_simulation, - }) - result_explanation = format_results(results) + return roi_table + + @staticmethod + def roi_space() -> Dict[str, Any]: + """ + Values to search for each ROI steps + """ + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + """ + Stoploss Value to search + """ + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + } + + def has_space(self, space: str) -> bool: + """ + Tell if a space value is contained in the configuration + """ + if space in self.config['spaces'] or 'all' in self.config['spaces']: + return True + return False + + def hyperopt_space(self) -> Dict[str, Any]: + """ + Return the space to use during Hyperopt + """ + spaces = {} + if self.has_space('buy'): + spaces = {**spaces, **Hyperopt.indicator_space()} + if self.has_space('roi'): + spaces = {**spaces, **Hyperopt.roi_space()} + if self.has_space('stoploss'): + spaces = {**spaces, **Hyperopt.stoploss_space()} + return spaces + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + def generate_optimizer(self, params: Dict) -> Dict: + if self.has_space('roi'): + self.analyze.strategy.minimal_roi = self.generate_roi_table(params) + + if self.has_space('buy'): + self.populate_buy_trend = self.buy_strategy_generator(params) + + if self.has_space('stoploss'): + self.analyze.strategy.stoploss = params['stoploss'] + + results = self.backtest( + { + 'stake_amount': self.config['stake_amount'], + 'processed': self.processed, + 'realistic': self.config.get('realistic_simulation', False), + } + ) + result_explanation = self.format_results(results) total_profit = results.profit_percent.sum() trade_count = len(results.index) trade_duration = results.duration.mean() - if trade_count == 0 or trade_duration > MAX_ACCEPTED_TRADE_DURATION: + if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: print('.', end='') return { 'status': STATUS_FAIL, 'loss': float('inf') } - loss = calculate_loss(total_profit, trade_count, trade_duration) + loss = self.calculate_loss(total_profit, trade_count, trade_duration) - _CURRENT_TRIES += 1 + self.current_tries += 1 - log_results({ - 'loss': loss, - 'current_tries': _CURRENT_TRIES, - 'total_tries': TOTAL_TRIES, - 'result': result_explanation, - }) + self.log_results( + { + 'loss': loss, + 'current_tries': self.current_tries, + 'total_tries': self.total_tries, + 'result': result_explanation, + } + ) return { 'loss': loss, @@ -451,111 +483,134 @@ def generate_optimizer(args): 'result': result_explanation, } - return optimizer + @staticmethod + def format_results(results: DataFrame) -> str: + """ + Return the format result in a string + """ + return ('{:6d} trades. Avg profit {: 5.2f}%. ' + 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_BTC.sum(), + results.profit_percent.sum(), + results.duration.mean(), + ) + def start(self) -> None: + timerange = Arguments.parse_timerange(self.config.get('timerange')) + data = load_data( + datadir=self.config.get('datadir'), + pairs=self.config['exchange']['pair_whitelist'], + ticker_interval=self.ticker_interval, + timerange=timerange + ) -def format_results(results: DataFrame): - return ('{:6d} trades. Avg profit {: 5.2f}%. ' - 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_BTC.sum(), - results.profit_percent.sum(), - results.duration.mean(), + if self.has_space('buy'): + self.analyze.populate_indicators = Hyperopt.populate_indicators + self.processed = self.tickerdata_to_dataframe(data) + + if self.config.get('mongodb'): + self.logger.info('Using mongodb ...') + self.logger.info( + 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' ) + db_name = 'freqtrade_hyperopt' + self.trials = MongoTrials( + arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), + exp_key='exp1' + ) + else: + self.logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, self.signal_handler) + # read trials file if we have one + if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: + self.trials = self.read_trials() -def start(args): - global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES + self.current_tries = len(self.trials.results) + self.total_tries += self.current_tries + self.logger.info( + 'Continuing with trials. Current: %d, Total: %d', + self.current_tries, + self.total_tries + ) - TOTAL_TRIES = args.epochs + try: + # change the Logging format + self.logging.set_format('\n%(message)s') - exchange._API = Bittrex({'key': '', 'secret': ''}) + best_parameters = fmin( + fn=self.generate_optimizer, + space=self.hyperopt_space(), + algo=tpe.suggest, + max_evals=self.total_tries, + trials=self.trials + ) + + results = sorted(self.trials.results, key=itemgetter('loss')) + best_result = results[0]['result'] + + except ValueError: + best_parameters = {} + best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ + 'try with more epochs (param: -e).' + + # Improve best parameter logging display + if best_parameters: + best_parameters = space_eval( + self.hyperopt_space(), + best_parameters + ) + + self.logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) + if 'roi_t1' in best_parameters: + self.logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) + + self.logger.info('Best Result:\n%s', best_result) + + # Store trials result to file to resume next time + self.save_trials() + + def signal_handler(self, sig, frame) -> None: + """ + Hyperopt SIGINT handler + """ + self.logger.info( + 'Hyperopt received %s', + signal.Signals(sig).name + ) + + self.save_trials() + self.log_trials_result() + sys.exit(0) + + +def start(args: Namespace) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + + # Remove noisy log messages + logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) + logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) # Initialize logger - logging.basicConfig( - level=args.loglevel, - format='\n%(message)s', - ) + logger = Logger(name=__name__).get_logger() + logger.info('Starting freqtrade in Hyperopt mode') - logger.info('Using config: %s ...', args.config) - config = load_config(args.config) - pairs = config['exchange']['pair_whitelist'] + # Initialize configuration + # Monkey patch the configuration with hyperopt_conf.py + configuration = Configuration(args) + optimize_config = hyperopt_optimize_conf() + config = configuration._load_common_config(optimize_config) + config = configuration._load_backtesting_config(config) + config = configuration._load_hyperopt_config(config) + config['exchange']['key'] = '' + config['exchange']['secret'] = '' - # If -i/--ticker-interval is use we override the configuration parameter - # (that will override the strategy configuration) - if args.ticker_interval: - config.update({'ticker_interval': args.ticker_interval}) - - # init the strategy to use - config.update({'strategy': args.strategy}) - strategy = Strategy() - strategy.init(config) - - timerange = misc.parse_timerange(args.timerange) - data = optimize.load_data(args.datadir, pairs=pairs, - ticker_interval=strategy.ticker_interval, - timerange=timerange) - if has_space(args.spaces, 'buy'): - optimize.populate_indicators = populate_indicators - PROCESSED = optimize.tickerdata_to_dataframe(data) - - if args.mongodb: - logger.info('Using mongodb ...') - logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!') - - db_name = 'freqtrade_hyperopt' - TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') - else: - logger.info('Preparing Trials..') - signal.signal(signal.SIGINT, signal_handler) - # read trials file if we have one - if os.path.exists(TRIALS_FILE): - TRIALS = read_trials() - - _CURRENT_TRIES = len(TRIALS.results) - TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES - logger.info( - 'Continuing with trials. Current: {}, Total: {}' - .format(_CURRENT_TRIES, TOTAL_TRIES)) - - try: - best_parameters = fmin( - fn=generate_optimizer(args), - space=hyperopt_space(args.spaces), - algo=tpe.suggest, - max_evals=TOTAL_TRIES, - trials=TRIALS - ) - - results = sorted(TRIALS.results, key=itemgetter('loss')) - best_result = results[0]['result'] - - except ValueError: - best_parameters = {} - best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ - 'try with more epochs (param: -e).' - - # Improve best parameter logging display - if best_parameters: - best_parameters = space_eval( - hyperopt_space(args.spaces), - best_parameters - ) - - logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) - if 'roi_t1' in best_parameters: - logger.info('ROI table:\n%s', generate_roi_table(best_parameters)) - logger.info('Best Result:\n%s', best_result) - - # Store trials result to file to resume next time - save_trials(TRIALS) - - -def signal_handler(sig, frame): - """Hyperopt SIGINT handler""" - logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) - - save_trials(TRIALS) - log_trials_result(TRIALS) - sys.exit(0) + # Initialize backtesting object + hyperopt = Hyperopt(config) + hyperopt.start() diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index e3043089f..56fe336d2 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -1,3 +1,7 @@ +""" +This module contains the class to persist trades into SQLite +""" + import logging from datetime import datetime from decimal import Decimal, getcontext @@ -72,6 +76,9 @@ def clean_dry_run_db() -> None: class Trade(_DECL_BASE): + """ + Class used to define a trade structure + """ __tablename__ = 'trades' id = Column(Integer, primary_key=True) @@ -200,6 +207,7 @@ class Trade(_DECL_BASE): Calculates the profit in percentage (including fee). :param rate: rate to compare with (optional). If rate is not set self.close_rate will be used + :param fee: fee to use on the close rate (optional). :return: profit in percentage as float """ getcontext().prec = 8 diff --git a/freqtrade/rpc/__init__.py b/freqtrade/rpc/__init__.py index 163e0a8aa..e69de29bb 100644 --- a/freqtrade/rpc/__init__.py +++ b/freqtrade/rpc/__init__.py @@ -1,415 +0,0 @@ -import logging -import re -import arrow -from decimal import Decimal -from datetime import datetime, timedelta -from pandas import DataFrame -import sqlalchemy as sql -# from sqlalchemy import and_, func, text - -from freqtrade.persistence import Trade -from freqtrade.misc import State, get_state, update_state -from freqtrade import exchange -from freqtrade.fiat_convert import CryptoToFiatConverter -from . import telegram - -logger = logging.getLogger(__name__) - -_FIAT_CONVERT = CryptoToFiatConverter() -REGISTERED_MODULES = [] - - -def init(config: dict) -> None: - """ - Initializes all enabled rpc modules - :param config: config to use - :return: None - """ - - if config['telegram'].get('enabled', False): - logger.info('Enabling rpc.telegram ...') - REGISTERED_MODULES.append('telegram') - telegram.init(config) - - -def cleanup() -> None: - """ - Stops all enabled rpc modules - :return: None - """ - if 'telegram' in REGISTERED_MODULES: - logger.debug('Cleaning up rpc.telegram ...') - telegram.cleanup() - - -def send_msg(msg: str) -> None: - """ - Send given markdown message to all registered rpc modules - :param msg: message - :return: None - """ - logger.info(msg) - if 'telegram' in REGISTERED_MODULES: - telegram.send_msg(msg) - - -def shorten_date(_date): - """ - Trim the date so it fits on small screens - """ - new_date = re.sub('seconds?', 'sec', _date) - new_date = re.sub('minutes?', 'min', new_date) - new_date = re.sub('hours?', 'h', new_date) - new_date = re.sub('days?', 'd', new_date) - new_date = re.sub('^an?', '1', new_date) - return new_date - - -# -# Below follows the RPC backend -# it is prefixed with rpc_ -# to raise awareness that it is -# a remotely exposed function - - -def rpc_trade_status(): - # Fetch open trade - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if get_state() != State.RUNNING: - return (True, '*Status:* `trader is not running`') - elif not trades: - return (True, '*Status:* `no active trade`') - else: - result = [] - for trade in trades: - order = None - if trade.open_order_id: - order = exchange.get_order(trade.open_order_id) - # calculate profit and send message to user - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - current_profit = trade.calc_profit_percent(current_rate) - fmt_close_profit = '{:.2f}%'.format( - round(trade.close_profit * 100, 2) - ) if trade.close_profit else None - message = """ -*Trade ID:* `{trade_id}` -*Current Pair:* [{pair}]({market_url}) -*Open Since:* `{date}` -*Amount:* `{amount}` -*Open Rate:* `{open_rate:.8f}` -*Close Rate:* `{close_rate}` -*Current Rate:* `{current_rate:.8f}` -*Close Profit:* `{close_profit}` -*Current Profit:* `{current_profit:.2f}%` -*Open Order:* `{open_order}` - """.format( - trade_id=trade.id, - pair=trade.pair, - market_url=exchange.get_pair_detail_url(trade.pair), - date=arrow.get(trade.open_date).humanize(), - open_rate=trade.open_rate, - close_rate=trade.close_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - close_profit=fmt_close_profit, - current_profit=round(current_profit * 100, 2), - open_order='({} rem={:.8f})'.format( - order['type'], order['remaining'] - ) if order else None, - ) - result.append(message) - return (False, result) - - -def rpc_status_table(): - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if get_state() != State.RUNNING: - return (True, '*Status:* `trader is not running`') - elif not trades: - return (True, '*Status:* `no active order`') - else: - trades_list = [] - for trade in trades: - # calculate profit and send message to user - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - trades_list.append([ - trade.id, - trade.pair, - shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), - '{:.2f}%'.format(100 * trade.calc_profit_percent(current_rate)) - ]) - - columns = ['ID', 'Pair', 'Since', 'Profit'] - df_statuses = DataFrame.from_records(trades_list, columns=columns) - df_statuses = df_statuses.set_index(columns[0]) - # The style used throughout is to return a tuple - # consisting of (error_occured?, result) - # Another approach would be to just return the - # result, or raise error - return (False, df_statuses) - - -def rpc_daily_profit(timescale, stake_currency, fiat_display_currency): - today = datetime.utcnow().date() - profit_days = {} - - if not (isinstance(timescale, int) and timescale > 0): - return (True, '*Daily [n]:* `must be an integer greater than 0`') - - fiat = _FIAT_CONVERT - for day in range(0, timescale): - profitday = today - timedelta(days=day) - trades = Trade.query \ - .filter(Trade.is_open.is_(False)) \ - .filter(Trade.close_date >= profitday)\ - .filter(Trade.close_date < (profitday + timedelta(days=1)))\ - .order_by(Trade.close_date)\ - .all() - curdayprofit = sum(trade.calc_profit() for trade in trades) - profit_days[profitday] = { - 'amount': format(curdayprofit, '.8f'), - 'trades': len(trades) - } - - stats = [ - [ - key, - '{value:.8f} {symbol}'.format( - value=float(value['amount']), - symbol=stake_currency - ), - '{value:.3f} {symbol}'.format( - value=fiat.convert_amount( - value['amount'], - stake_currency, - fiat_display_currency - ), - symbol=fiat_display_currency - ), - '{value} trade{s}'.format(value=value['trades'], s='' if value['trades'] < 2 else 's'), - ] - for key, value in profit_days.items() - ] - return (False, stats) - - -def rpc_trade_statistics(stake_currency, fiat_display_currency) -> None: - """ - :return: cumulative profit statistics. - """ - trades = Trade.query.order_by(Trade.id).all() - - profit_all_coin = [] - profit_all_percent = [] - profit_closed_coin = [] - profit_closed_percent = [] - durations = [] - - for trade in trades: - current_rate = None - - if not trade.open_rate: - continue - if trade.close_date: - durations.append((trade.close_date - trade.open_date).total_seconds()) - - if not trade.is_open: - profit_percent = trade.calc_profit_percent() - profit_closed_coin.append(trade.calc_profit()) - profit_closed_percent.append(profit_percent) - else: - # Get current rate - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - profit_percent = trade.calc_profit_percent(rate=current_rate) - - profit_all_coin.append(trade.calc_profit(rate=Decimal(trade.close_rate or current_rate))) - profit_all_percent.append(profit_percent) - - best_pair = Trade.session.query(Trade.pair, - sql.func.sum(Trade.close_profit).label('profit_sum')) \ - .filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')) \ - .first() - - if not best_pair: - return (True, '*Status:* `no closed trade`') - - bp_pair, bp_rate = best_pair - - # FIX: we want to keep fiatconverter in a state/environment, - # doing this will utilize its caching functionallity, instead we reinitialize it here - fiat = _FIAT_CONVERT - # Prepare data to display - profit_closed_coin = round(sum(profit_closed_coin), 8) - profit_closed_percent = round(sum(profit_closed_percent) * 100, 2) - profit_closed_fiat = fiat.convert_amount( - profit_closed_coin, - stake_currency, - fiat_display_currency - ) - profit_all_coin = round(sum(profit_all_coin), 8) - profit_all_percent = round(sum(profit_all_percent) * 100, 2) - profit_all_fiat = fiat.convert_amount( - profit_all_coin, - stake_currency, - fiat_display_currency - ) - num = float(len(durations) or 1) - return (False, - {'profit_closed_coin': profit_closed_coin, - 'profit_closed_percent': profit_closed_percent, - 'profit_closed_fiat': profit_closed_fiat, - 'profit_all_coin': profit_all_coin, - 'profit_all_percent': profit_all_percent, - 'profit_all_fiat': profit_all_fiat, - 'trade_count': len(trades), - 'first_trade_date': arrow.get(trades[0].open_date).humanize(), - 'latest_trade_date': arrow.get(trades[-1].open_date).humanize(), - 'avg_duration': str(timedelta(seconds=sum(durations) / - num)).split('.')[0], - 'best_pair': bp_pair, - 'best_rate': round(bp_rate * 100, 2) - }) - - -def rpc_balance(fiat_display_currency): - """ - :return: current account balance per crypto - """ - balances = [ - c for c in exchange.get_balances() - if c['Balance'] or c['Available'] or c['Pending'] - ] - if not balances: - return (True, '`All balances are zero.`') - - output = [] - total = 0.0 - for currency in balances: - coin = currency['Currency'] - if coin == 'BTC': - currency["Rate"] = 1.0 - else: - if coin == 'USDT': - currency["Rate"] = 1.0 / exchange.get_ticker('USDT_BTC', False)['bid'] - else: - currency["Rate"] = exchange.get_ticker('BTC_' + coin, False)['bid'] - currency['BTC'] = currency["Rate"] * currency["Balance"] - total = total + currency['BTC'] - output.append({'currency': currency['Currency'], - 'available': currency['Available'], - 'balance': currency['Balance'], - 'pending': currency['Pending'], - 'est_btc': currency['BTC'] - }) - fiat = _FIAT_CONVERT - symbol = fiat_display_currency - value = fiat.convert_amount(total, 'BTC', symbol) - return (False, (output, total, symbol, value)) - - -def rpc_start(): - """ - Handler for start. - """ - if get_state() == State.RUNNING: - return (True, '*Status:* `already running`') - else: - update_state(State.RUNNING) - - -def rpc_stop(): - """ - Handler for stop. - """ - if get_state() == State.RUNNING: - update_state(State.STOPPED) - return (False, '`Stopping trader ...`') - else: - return (True, '*Status:* `already stopped`') - - -# FIX: no test for this!!!! -def rpc_forcesell(trade_id) -> None: - """ - Handler for forcesell . - Sells the given trade at current price - :return: error or None - """ - def _exec_forcesell(trade: Trade) -> str: - # Check if there is there is an open order - if trade.open_order_id: - order = exchange.get_order(trade.open_order_id) - - # Cancel open LIMIT_BUY orders and close trade - if order and not order['closed'] and order['type'] == 'LIMIT_BUY': - exchange.cancel_order(trade.open_order_id) - trade.close(order.get('rate') or trade.open_rate) - # TODO: sell amount which has been bought already - return - - # Ignore trades with an attached LIMIT_SELL order - if order and not order['closed'] and order['type'] == 'LIMIT_SELL': - return - - # Get current rate and execute sell - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - from freqtrade.main import execute_sell - execute_sell(trade, current_rate) - # ---- EOF def _exec_forcesell ---- - - if get_state() != State.RUNNING: - return (True, '`trader is not running`') - - if trade_id == 'all': - # Execute sell for all open orders - for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): - _exec_forcesell(trade) - return (False, '') - - # Query for trade - trade = Trade.query.filter(sql.and_( - Trade.id == trade_id, - Trade.is_open.is_(True) - )).first() - if not trade: - logger.warning('forcesell: Invalid argument received') - return (True, 'Invalid argument.') - - _exec_forcesell(trade) - return (False, '') - - -def rpc_performance() -> None: - """ - Handler for performance. - Shows a performance statistic from finished trades - """ - if get_state() != State.RUNNING: - return (True, '`trader is not running`') - - pair_rates = Trade.session.query(Trade.pair, - sql.func.sum(Trade.close_profit).label('profit_sum'), - sql.func.count(Trade.pair).label('count')) \ - .filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')) \ - .all() - trades = [] - for (pair, rate, count) in pair_rates: - trades.append({'pair': pair, 'profit': round(rate * 100, 2), 'count': count}) - - return (False, trades) - - -def rpc_count() -> None: - """ - Returns the number of trades running - :return: None - """ - if get_state() != State.RUNNING: - return (True, '`trader is not running`') - - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - return (False, trades) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py new file mode 100644 index 000000000..b4592f78a --- /dev/null +++ b/freqtrade/rpc/rpc.py @@ -0,0 +1,385 @@ +""" +This module contains class to define a RPC communications +""" + +from datetime import datetime, timedelta +from decimal import Decimal +from typing import Tuple, Any + +import arrow +import sqlalchemy as sql +from pandas import DataFrame + +from freqtrade import exchange +from freqtrade.logger import Logger +from freqtrade.misc import shorten_date +from freqtrade.persistence import Trade +from freqtrade.state import State + + +class RPC(object): + """ + RPC class can be used to have extra feature, like bot data, and access to DB data + """ + def __init__(self, freqtrade) -> None: + """ + Initializes all enabled rpc modules + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + self.freqtrade = freqtrade + self.logger = Logger( + name=__name__, + level=self.freqtrade.config.get('loglevel') + ).get_logger() + + def rpc_trade_status(self) -> Tuple[bool, Any]: + """ + Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is + a remotely exposed function + :return: + """ + # Fetch open trade + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + if self.freqtrade.get_state() != State.RUNNING: + return True, '*Status:* `trader is not running`' + elif not trades: + return True, '*Status:* `no active trade`' + else: + result = [] + for trade in trades: + order = None + if trade.open_order_id: + order = exchange.get_order(trade.open_order_id) + # calculate profit and send message to user + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + current_profit = trade.calc_profit_percent(current_rate) + fmt_close_profit = '{:.2f}%'.format( + round(trade.close_profit * 100, 2) + ) if trade.close_profit else None + message = "*Trade ID:* `{trade_id}`\n" \ + "*Current Pair:* [{pair}]({market_url})\n" \ + "*Open Since:* `{date}`\n" \ + "*Amount:* `{amount}`\n" \ + "*Open Rate:* `{open_rate:.8f}`\n" \ + "*Close Rate:* `{close_rate}`\n" \ + "*Current Rate:* `{current_rate:.8f}`\n" \ + "*Close Profit:* `{close_profit}`\n" \ + "*Current Profit:* `{current_profit:.2f}%`\n" \ + "*Open Order:* `{open_order}`"\ + .format( + trade_id=trade.id, + pair=trade.pair, + market_url=exchange.get_pair_detail_url(trade.pair), + date=arrow.get(trade.open_date).humanize(), + open_rate=trade.open_rate, + close_rate=trade.close_rate, + current_rate=current_rate, + amount=round(trade.amount, 8), + close_profit=fmt_close_profit, + current_profit=round(current_profit * 100, 2), + open_order='({} rem={:.8f})'.format( + order['type'], order['remaining'] + ) if order else None, + ) + result.append(message) + return False, result + + def rpc_status_table(self) -> Tuple[bool, Any]: + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + if self.freqtrade.get_state() != State.RUNNING: + return True, '*Status:* `trader is not running`' + elif not trades: + return True, '*Status:* `no active order`' + else: + trades_list = [] + for trade in trades: + # calculate profit and send message to user + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + trades_list.append([ + trade.id, + trade.pair, + shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), + '{:.2f}%'.format(100 * trade.calc_profit_percent(current_rate)) + ]) + + columns = ['ID', 'Pair', 'Since', 'Profit'] + df_statuses = DataFrame.from_records(trades_list, columns=columns) + df_statuses = df_statuses.set_index(columns[0]) + # The style used throughout is to return a tuple + # consisting of (error_occured?, result) + # Another approach would be to just return the + # result, or raise error + return False, df_statuses + + def rpc_daily_profit( + self, timescale: int, + stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: + today = datetime.utcnow().date() + profit_days = {} + + if not (isinstance(timescale, int) and timescale > 0): + return True, '*Daily [n]:* `must be an integer greater than 0`' + + fiat = self.freqtrade.fiat_converter + for day in range(0, timescale): + profitday = today - timedelta(days=day) + trades = Trade.query \ + .filter(Trade.is_open.is_(False)) \ + .filter(Trade.close_date >= profitday)\ + .filter(Trade.close_date < (profitday + timedelta(days=1)))\ + .order_by(Trade.close_date)\ + .all() + curdayprofit = sum(trade.calc_profit() for trade in trades) + profit_days[profitday] = { + 'amount': format(curdayprofit, '.8f'), + 'trades': len(trades) + } + + stats = [ + [ + key, + '{value:.8f} {symbol}'.format( + value=float(value['amount']), + symbol=stake_currency + ), + '{value:.3f} {symbol}'.format( + value=fiat.convert_amount( + value['amount'], + stake_currency, + fiat_display_currency + ), + symbol=fiat_display_currency + ), + '{value} trade{s}'.format( + value=value['trades'], + s='' if value['trades'] < 2 else 's' + ), + ] + for key, value in profit_days.items() + ] + return False, stats + + def rpc_trade_statistics( + self, stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: + """ + :return: cumulative profit statistics. + """ + trades = Trade.query.order_by(Trade.id).all() + + profit_all_coin = [] + profit_all_percent = [] + profit_closed_coin = [] + profit_closed_percent = [] + durations = [] + + for trade in trades: + current_rate = None + + if not trade.open_rate: + continue + if trade.close_date: + durations.append((trade.close_date - trade.open_date).total_seconds()) + + if not trade.is_open: + profit_percent = trade.calc_profit_percent() + profit_closed_coin.append(trade.calc_profit()) + profit_closed_percent.append(profit_percent) + else: + # Get current rate + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + profit_percent = trade.calc_profit_percent(rate=current_rate) + + profit_all_coin.append( + trade.calc_profit(rate=Decimal(trade.close_rate or current_rate)) + ) + profit_all_percent.append(profit_percent) + + best_pair = Trade.session.query( + Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum') + ).filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ + .order_by(sql.text('profit_sum DESC')).first() + + if not best_pair: + return True, '*Status:* `no closed trade`' + + bp_pair, bp_rate = best_pair + + # FIX: we want to keep fiatconverter in a state/environment, + # doing this will utilize its caching functionallity, instead we reinitialize it here + fiat = self.freqtrade.fiat_converter + # Prepare data to display + profit_closed_coin = round(sum(profit_closed_coin), 8) + profit_closed_percent = round(sum(profit_closed_percent) * 100, 2) + profit_closed_fiat = fiat.convert_amount( + profit_closed_coin, + stake_currency, + fiat_display_currency + ) + profit_all_coin = round(sum(profit_all_coin), 8) + profit_all_percent = round(sum(profit_all_percent) * 100, 2) + profit_all_fiat = fiat.convert_amount( + profit_all_coin, + stake_currency, + fiat_display_currency + ) + num = float(len(durations) or 1) + return ( + False, + { + 'profit_closed_coin': profit_closed_coin, + 'profit_closed_percent': profit_closed_percent, + 'profit_closed_fiat': profit_closed_fiat, + 'profit_all_coin': profit_all_coin, + 'profit_all_percent': profit_all_percent, + 'profit_all_fiat': profit_all_fiat, + 'trade_count': len(trades), + 'first_trade_date': arrow.get(trades[0].open_date).humanize(), + 'latest_trade_date': arrow.get(trades[-1].open_date).humanize(), + 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], + 'best_pair': bp_pair, + 'best_rate': round(bp_rate * 100, 2) + } + ) + + def rpc_balance(self, fiat_display_currency: str) -> Tuple[bool, Any]: + """ + :return: current account balance per crypto + """ + balances = [ + c for c in exchange.get_balances() + if c['Balance'] or c['Available'] or c['Pending'] + ] + if not balances: + return True, '`All balances are zero.`' + + output = [] + total = 0.0 + for currency in balances: + coin = currency['Currency'] + if coin == 'BTC': + currency["Rate"] = 1.0 + else: + if coin == 'USDT': + currency["Rate"] = 1.0 / exchange.get_ticker('USDT_BTC', False)['bid'] + else: + currency["Rate"] = exchange.get_ticker('BTC_' + coin, False)['bid'] + currency['BTC'] = currency["Rate"] * currency["Balance"] + total = total + currency['BTC'] + output.append( + { + 'currency': currency['Currency'], + 'available': currency['Available'], + 'balance': currency['Balance'], + 'pending': currency['Pending'], + 'est_btc': currency['BTC'] + } + ) + fiat = self.freqtrade.fiat_converter + symbol = fiat_display_currency + value = fiat.convert_amount(total, 'BTC', symbol) + return False, (output, total, symbol, value) + + def rpc_start(self) -> (bool, str): + """ + Handler for start. + """ + if self.freqtrade.get_state() == State.RUNNING: + return True, '*Status:* `already running`' + + self.freqtrade.update_state(State.RUNNING) + return False, '`Starting trader ...`' + + def rpc_stop(self) -> (bool, str): + """ + Handler for stop. + """ + if self.freqtrade.get_state() == State.RUNNING: + self.freqtrade.update_state(State.STOPPED) + return False, '`Stopping trader ...`' + + return True, '*Status:* `already stopped`' + + # FIX: no test for this!!!! + def rpc_forcesell(self, trade_id) -> Tuple[bool, Any]: + """ + Handler for forcesell . + Sells the given trade at current price + :return: error or None + """ + def _exec_forcesell(trade: Trade) -> None: + # Check if there is there is an open order + if trade.open_order_id: + order = exchange.get_order(trade.open_order_id) + + # Cancel open LIMIT_BUY orders and close trade + if order and not order['closed'] and order['type'] == 'LIMIT_BUY': + exchange.cancel_order(trade.open_order_id) + trade.close(order.get('rate') or trade.open_rate) + # TODO: sell amount which has been bought already + return + + # Ignore trades with an attached LIMIT_SELL order + if order and not order['closed'] and order['type'] == 'LIMIT_SELL': + return + + # Get current rate and execute sell + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + self.freqtrade.execute_sell(trade, current_rate) + # ---- EOF def _exec_forcesell ---- + + if self.freqtrade.get_state() != State.RUNNING: + return True, '`trader is not running`' + + if trade_id == 'all': + # Execute sell for all open orders + for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): + _exec_forcesell(trade) + return False, '' + + # Query for trade + trade = Trade.query.filter( + sql.and_( + Trade.id == trade_id, + Trade.is_open.is_(True) + ) + ).first() + if not trade: + self.logger.warning('forcesell: Invalid argument received') + return True, 'Invalid argument.' + + _exec_forcesell(trade) + return False, '' + + def rpc_performance(self) -> Tuple[bool, Any]: + """ + Handler for performance. + Shows a performance statistic from finished trades + """ + if self.freqtrade.get_state() != State.RUNNING: + return True, '`trader is not running`' + + pair_rates = Trade.session.query(Trade.pair, + sql.func.sum(Trade.close_profit).label('profit_sum'), + sql.func.count(Trade.pair).label('count')) \ + .filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ + .order_by(sql.text('profit_sum DESC')) \ + .all() + trades = [] + for (pair, rate, count) in pair_rates: + trades.append({'pair': pair, 'profit': round(rate * 100, 2), 'count': count}) + + return False, trades + + def rpc_count(self) -> Tuple[bool, Any]: + """ + Returns the number of trades running + :return: None + """ + if self.freqtrade.get_state() != State.RUNNING: + return True, '`trader is not running`' + + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + return False, trades diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py new file mode 100644 index 000000000..fb18a8d73 --- /dev/null +++ b/freqtrade/rpc/rpc_manager.py @@ -0,0 +1,59 @@ +""" +This module contains class to manage RPC communications (Telegram, Slack, ...) +""" + +from freqtrade.logger import Logger +from freqtrade.rpc.telegram import Telegram + + +class RPCManager(object): + """ + Class to manage RPC objects (Telegram, Slack, ...) + """ + def __init__(self, freqtrade) -> None: + """ + Initializes all enabled rpc modules + :param config: config to use + :return: None + """ + self.freqtrade = freqtrade + + # Init the logger + self.logger = Logger( + name=__name__, + level=self.freqtrade.config.get('loglevel') + ).get_logger() + + self.registered_modules = [] + self.telegram = None + self._init() + + def _init(self) -> None: + """ + Init RPC modules + :return: + """ + if self.freqtrade.config['telegram'].get('enabled', False): + self.logger.info('Enabling rpc.telegram ...') + self.registered_modules.append('telegram') + self.telegram = Telegram(self.freqtrade) + + def cleanup(self) -> None: + """ + Stops all enabled rpc modules + :return: None + """ + if 'telegram' in self.registered_modules: + self.logger.info('Cleaning up rpc.telegram ...') + self.registered_modules.remove('telegram') + self.telegram.cleanup() + + def send_msg(self, msg: str) -> None: + """ + Send given markdown message to all registered rpc modules + :param msg: message + :return: None + """ + self.logger.info(msg) + if 'telegram' in self.registered_modules: + self.telegram.send_msg(msg) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 676eba8d7..fce7a81f9 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1,4 +1,9 @@ -import logging +# pragma pylint: disable=unused-argument, unused-variable, protected-access, invalid-name + +""" +This module manage Telegram communication +""" + from typing import Any, Callable from tabulate import tabulate @@ -6,89 +11,8 @@ from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater -from freqtrade.rpc.__init__ import (rpc_status_table, - rpc_trade_status, - rpc_daily_profit, - rpc_trade_statistics, - rpc_balance, - rpc_start, - rpc_stop, - rpc_forcesell, - rpc_performance, - rpc_count, - ) - -from freqtrade import __version__ - - -# Remove noisy log messages -logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) -logging.getLogger('telegram').setLevel(logging.INFO) -logger = logging.getLogger(__name__) - -_UPDATER: Updater = None -_CONF = {} - - -def init(config: dict) -> None: - """ - Initializes this module with the given config, - registers all known command handlers - and starts polling for message updates - :param config: config to use - :return: None - """ - global _UPDATER - - _CONF.update(config) - if not is_enabled(): - return - - _UPDATER = Updater(token=config['telegram']['token'], workers=0) - - # Register command handler and start telegram message polling - handles = [ - CommandHandler('status', _status), - CommandHandler('profit', _profit), - CommandHandler('balance', _balance), - CommandHandler('start', _start), - CommandHandler('stop', _stop), - CommandHandler('forcesell', _forcesell), - CommandHandler('performance', _performance), - CommandHandler('daily', _daily), - CommandHandler('count', _count), - CommandHandler('help', _help), - CommandHandler('version', _version), - ] - for handle in handles: - _UPDATER.dispatcher.add_handler(handle) - _UPDATER.start_polling( - clean=True, - bootstrap_retries=-1, - timeout=30, - read_latency=60, - ) - logger.info( - 'rpc.telegram is listening for following commands: %s', - [h.command for h in handles] - ) - - -def cleanup() -> None: - """ - Stops all running telegram threads. - :return: None - """ - if not is_enabled(): - return - _UPDATER.stop() - - -def is_enabled() -> bool: - """ - Returns True if the telegram module is activated, False otherwise - """ - return bool(_CONF['telegram'].get('enabled', False)) +from freqtrade.__init__ import __version__ +from freqtrade.rpc.rpc import RPC def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[..., Any]: @@ -97,340 +21,425 @@ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[ :param command_handler: Telegram CommandHandler :return: decorated function """ - def wrapper(*args, **kwargs): + def wrapper(self, *args, **kwargs): + """ + Decorator logic + """ update = kwargs.get('update') or args[1] # Reject unauthorized messages - chat_id = int(_CONF['telegram']['chat_id']) + chat_id = int(self._config['telegram']['chat_id']) + if int(update.message.chat_id) != chat_id: - logger.info('Rejected unauthorized message from: %s', update.message.chat_id) + self.logger.info( + 'Rejected unauthorized message from: %s', + update.message.chat_id + ) return wrapper - logger.info('Executing handler: %s for chat_id: %s', command_handler.__name__, chat_id) + self.logger.info( + 'Executing handler: %s for chat_id: %s', + command_handler.__name__, + chat_id + ) try: - return command_handler(*args, **kwargs) + return command_handler(self, *args, **kwargs) except BaseException: - logger.exception('Exception occurred within Telegram module') + self.logger.exception('Exception occurred within Telegram module') + return wrapper -@authorized_only -def _status(bot: Bot, update: Update) -> None: +class Telegram(RPC): """ - Handler for /status. - Returns the current TradeThread status - :param bot: telegram bot - :param update: message update - :return: None + Telegram, this class send messages to Telegram """ + def __init__(self, freqtrade) -> None: + """ + Init the Telegram call, and init the super class RPC + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + super().__init__(freqtrade) - # Check if additional parameters are passed - params = update.message.text.replace('/status', '').split(' ') \ - if update.message.text else [] - if 'table' in params: - _status_table(bot, update) - return + self._updater = None + self._config = freqtrade.config + self._init() - # Fetch open trade - (error, trades) = rpc_trade_status() - if error: - send_msg(trades, bot=bot) - else: - for trademsg in trades: - send_msg(trademsg, bot=bot) + def _init(self) -> None: + """ + Initializes this module with the given config, + registers all known command handlers + and starts polling for message updates + :param config: config to use + :return: None + """ + if not self.is_enabled(): + return + self._updater = Updater(token=self._config['telegram']['token'], workers=0) -@authorized_only -def _status_table(bot: Bot, update: Update) -> None: - """ - Handler for /status table. - Returns the current TradeThread status in table format - :param bot: telegram bot - :param update: message update - :return: None - """ - # Fetch open trade - (err, df_statuses) = rpc_status_table() - if err: - send_msg(df_statuses, bot=bot) - else: - message = tabulate(df_statuses, headers='keys', tablefmt='simple') - message = "
{}
".format(message) + # Register command handler and start telegram message polling + handles = [ + CommandHandler('status', self._status), + CommandHandler('profit', self._profit), + CommandHandler('balance', self._balance), + CommandHandler('start', self._start), + CommandHandler('stop', self._stop), + CommandHandler('forcesell', self._forcesell), + CommandHandler('performance', self._performance), + CommandHandler('daily', self._daily), + CommandHandler('count', self._count), + CommandHandler('help', self._help), + CommandHandler('version', self._version), + ] + for handle in handles: + self._updater.dispatcher.add_handler(handle) + self._updater.start_polling( + clean=True, + bootstrap_retries=-1, + timeout=30, + read_latency=60, + ) + self.logger.info( + 'rpc.telegram is listening for following commands: %s', + [h.command for h in handles] + ) - send_msg(message, parse_mode=ParseMode.HTML) + def cleanup(self) -> None: + """ + Stops all running telegram threads. + :return: None + """ + if not self.is_enabled(): + return + self._updater.stop() -@authorized_only -def _daily(bot: Bot, update: Update) -> None: - """ - Handler for /daily - Returns a daily profit (in BTC) over the last n days. - :param bot: telegram bot - :param update: message update - :return: None - """ - try: - timescale = int(update.message.text.replace('/daily', '').strip()) - except (TypeError, ValueError): - timescale = 7 - (error, stats) = rpc_daily_profit(timescale, - _CONF['stake_currency'], - _CONF['fiat_display_currency']) - if error: - send_msg(stats, bot=bot) - else: - stats = tabulate(stats, - headers=[ - 'Day', - 'Profit {}'.format(_CONF['stake_currency']), - 'Profit {}'.format(_CONF['fiat_display_currency']) - ], - tablefmt='simple') - message = 'Daily Profit over the last {} days:\n
{}
'.format( - timescale, stats) - send_msg(message, bot=bot, parse_mode=ParseMode.HTML) + def is_enabled(self) -> bool: + """ + Returns True if the telegram module is activated, False otherwise + """ + return bool(self._config.get('telegram', {}).get('enabled', False)) + @authorized_only + def _status(self, bot: Bot, update: Update) -> None: + """ + Handler for /status. + Returns the current TradeThread status + :param bot: telegram bot + :param update: message update + :return: None + """ -@authorized_only -def _profit(bot: Bot, update: Update) -> None: - """ - Handler for /profit. - Returns a cumulative profit statistics. - :param bot: telegram bot - :param update: message update - :return: None - """ - (error, stats) = rpc_trade_statistics(_CONF['stake_currency'], - _CONF['fiat_display_currency']) - if error: - send_msg(stats, bot=bot) - return + # Check if additional parameters are passed + params = update.message.text.replace('/status', '').split(' ') \ + if update.message.text else [] + if 'table' in params: + self._status_table(bot, update) + return - # Message to display - markdown_msg = """ -*ROI:* Close trades - ∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)` - ∙ `{profit_closed_fiat:.3f} {fiat}` -*ROI:* All trades - ∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)` - ∙ `{profit_all_fiat:.3f} {fiat}` + # Fetch open trade + (error, trades) = self.rpc_trade_status() + if error: + self.send_msg(trades, bot=bot) + else: + for trademsg in trades: + self.send_msg(trademsg, bot=bot) -*Total Trade Count:* `{trade_count}` -*First Trade opened:* `{first_trade_date}` -*Latest Trade opened:* `{latest_trade_date}` -*Avg. Duration:* `{avg_duration}` -*Best Performing:* `{best_pair}: {best_rate:.2f}%` - """.format( - coin=_CONF['stake_currency'], - fiat=_CONF['fiat_display_currency'], - profit_closed_coin=stats['profit_closed_coin'], - profit_closed_percent=stats['profit_closed_percent'], - profit_closed_fiat=stats['profit_closed_fiat'], - profit_all_coin=stats['profit_all_coin'], - profit_all_percent=stats['profit_all_percent'], - profit_all_fiat=stats['profit_all_fiat'], - trade_count=stats['trade_count'], - first_trade_date=stats['first_trade_date'], - latest_trade_date=stats['latest_trade_date'], - avg_duration=stats['avg_duration'], - best_pair=stats['best_pair'], - best_rate=stats['best_rate'] - ) - send_msg(markdown_msg, bot=bot) + @authorized_only + def _status_table(self, bot: Bot, update: Update) -> None: + """ + Handler for /status table. + Returns the current TradeThread status in table format + :param bot: telegram bot + :param update: message update + :return: None + """ + # Fetch open trade + (err, df_statuses) = self.rpc_status_table() + if err: + self.send_msg(df_statuses, bot=bot) + else: + message = tabulate(df_statuses, headers='keys', tablefmt='simple') + message = "
{}
".format(message) + self.send_msg(message, parse_mode=ParseMode.HTML) -@authorized_only -def _balance(bot: Bot, update: Update) -> None: - """ - Handler for /balance - """ - (error, result) = rpc_balance(_CONF['fiat_display_currency']) - if error: - send_msg('`All balances are zero.`') - return - - (currencys, total, symbol, value) = result - output = '' - for currency in currencys: - output += """*Currency*: {currency} -*Available*: {available} -*Balance*: {balance} -*Pending*: {pending} -*Est. BTC*: {est_btc: .8f} -""".format(**currency) - - output += """*Estimated Value*: -*BTC*: {0: .8f} -*{1}*: {2: .2f} -""".format(total, symbol, value) - send_msg(output) - - -@authorized_only -def _start(bot: Bot, update: Update) -> None: - """ - Handler for /start. - Starts TradeThread - :param bot: telegram bot - :param update: message update - :return: None - """ - (error, msg) = rpc_start() - if error: - send_msg(msg, bot=bot) - - -@authorized_only -def _stop(bot: Bot, update: Update) -> None: - """ - Handler for /stop. - Stops TradeThread - :param bot: telegram bot - :param update: message update - :return: None - """ - (error, msg) = rpc_stop() - send_msg(msg, bot=bot) - - -# FIX: no test for this!!!! -@authorized_only -def _forcesell(bot: Bot, update: Update) -> None: - """ - Handler for /forcesell . - Sells the given trade at current price - :param bot: telegram bot - :param update: message update - :return: None - """ - - trade_id = update.message.text.replace('/forcesell', '').strip() - (error, message) = rpc_forcesell(trade_id) - if error: - send_msg(message, bot=bot) - return - - -@authorized_only -def _performance(bot: Bot, update: Update) -> None: - """ - Handler for /performance. - Shows a performance statistic from finished trades - :param bot: telegram bot - :param update: message update - :return: None - """ - (error, trades) = rpc_performance() - if error: - send_msg(trades, bot=bot) - return - - stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( - index=i + 1, - pair=trade['pair'], - profit=trade['profit'], - count=trade['count'] - ) for i, trade in enumerate(trades)) - message = 'Performance:\n{}'.format(stats) - send_msg(message, parse_mode=ParseMode.HTML) - - -@authorized_only -def _count(bot: Bot, update: Update) -> None: - """ - Handler for /count. - Returns the number of trades running - :param bot: telegram bot - :param update: message update - :return: None - """ - (error, trades) = rpc_count() - if error: - send_msg(trades, bot=bot) - return - - message = tabulate({ - 'current': [len(trades)], - 'max': [_CONF['max_open_trades']] - }, headers=['current', 'max'], tablefmt='simple') - message = "
{}
".format(message) - logger.debug(message) - send_msg(message, parse_mode=ParseMode.HTML) - - -@authorized_only -def _help(bot: Bot, update: Update) -> None: - """ - Handler for /help. - Show commands of the bot - :param bot: telegram bot - :param update: message update - :return: None - """ - message = """ -*/start:* `Starts the trader` -*/stop:* `Stops the trader` -*/status [table]:* `Lists all open trades` - *table :* `will display trades in a table` -*/profit:* `Lists cumulative profit from all finished trades` -*/forcesell |all:* `Instantly sells the given trade or all trades, regardless of profit` -*/performance:* `Show performance of each finished trade grouped by pair` -*/daily :* `Shows profit or loss per day, over the last n days` -*/count:* `Show number of trades running compared to allowed number of trades` -*/balance:* `Show account balance per currency` -*/help:* `This help message` -*/version:* `Show version` - """ - send_msg(message, bot=bot) - - -@authorized_only -def _version(bot: Bot, update: Update) -> None: - """ - Handler for /version. - Show version information - :param bot: telegram bot - :param update: message update - :return: None - """ - send_msg('*Version:* `{}`'.format(__version__), bot=bot) - - -def send_msg(msg: str, bot: Bot = None, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: - """ - Send given markdown message - :param msg: message - :param bot: alternative bot - :param parse_mode: telegram parse mode - :return: None - """ - if not is_enabled(): - return - - bot = bot or _UPDATER.bot - - keyboard = [['/daily', '/profit', '/balance'], - ['/status', '/status table', '/performance'], - ['/count', '/start', '/stop', '/help']] - - reply_markup = ReplyKeyboardMarkup(keyboard) - - try: + @authorized_only + def _daily(self, bot: Bot, update: Update) -> None: + """ + Handler for /daily + Returns a daily profit (in BTC) over the last n days. + :param bot: telegram bot + :param update: message update + :return: None + """ try: - bot.send_message( - _CONF['telegram']['chat_id'], msg, - parse_mode=parse_mode, reply_markup=reply_markup + timescale = int(update.message.text.replace('/daily', '').strip()) + except (TypeError, ValueError): + timescale = 7 + (error, stats) = self.rpc_daily_profit( + timescale, + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) + if error: + self.send_msg(stats, bot=bot) + else: + stats = tabulate(stats, + headers=[ + 'Day', + 'Profit {}'.format(self._config['stake_currency']), + 'Profit {}'.format(self._config['fiat_display_currency']) + ], + tablefmt='simple') + message = 'Daily Profit over the last {} days:\n
{}
'\ + .format( + timescale, + stats + ) + self.send_msg(message, bot=bot, parse_mode=ParseMode.HTML) + + @authorized_only + def _profit(self, bot: Bot, update: Update) -> None: + """ + Handler for /profit. + Returns a cumulative profit statistics. + :param bot: telegram bot + :param update: message update + :return: None + """ + (error, stats) = self.rpc_trade_statistics( + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) + if error: + self.send_msg(stats, bot=bot) + return + + # Message to display + markdown_msg = "*ROI:* Close trades\n" \ + "∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)`\n" \ + "∙ `{profit_closed_fiat:.3f} {fiat}`\n" \ + "*ROI:* All trades\n" \ + "∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)`\n" \ + "∙ `{profit_all_fiat:.3f} {fiat}`\n" \ + "*Total Trade Count:* `{trade_count}`\n" \ + "*First Trade opened:* `{first_trade_date}`\n" \ + "*Latest Trade opened:* `{latest_trade_date}`\n" \ + "*Avg. Duration:* `{avg_duration}`\n" \ + "*Best Performing:* `{best_pair}: {best_rate:.2f}%`"\ + .format( + coin=self._config['stake_currency'], + fiat=self._config['fiat_display_currency'], + profit_closed_coin=stats['profit_closed_coin'], + profit_closed_percent=stats['profit_closed_percent'], + profit_closed_fiat=stats['profit_closed_fiat'], + profit_all_coin=stats['profit_all_coin'], + profit_all_percent=stats['profit_all_percent'], + profit_all_fiat=stats['profit_all_fiat'], + trade_count=stats['trade_count'], + first_trade_date=stats['first_trade_date'], + latest_trade_date=stats['latest_trade_date'], + avg_duration=stats['avg_duration'], + best_pair=stats['best_pair'], + best_rate=stats['best_rate'] + ) + self.send_msg(markdown_msg, bot=bot) + + @authorized_only + def _balance(self, bot: Bot, update: Update) -> None: + """ + Handler for /balance + """ + (error, result) = self.rpc_balance(self._config['fiat_display_currency']) + if error: + self.send_msg('`All balances are zero.`') + return + + (currencys, total, symbol, value) = result + output = '' + for currency in currencys: + output += """*Currency*: {currency} + *Available*: {available} + *Balance*: {balance} + *Pending*: {pending} + *Est. BTC*: {est_btc: .8f} + """.format(**currency) + + output += """*Estimated Value*: + *BTC*: {0: .8f} + *{1}*: {2: .2f} + """.format(total, symbol, value) + self.send_msg(output) + + @authorized_only + def _start(self, bot: Bot, update: Update) -> None: + """ + Handler for /start. + Starts TradeThread + :param bot: telegram bot + :param update: message update + :return: None + """ + (error, msg) = self.rpc_start() + if error: + self.send_msg(msg, bot=bot) + + @authorized_only + def _stop(self, bot: Bot, update: Update) -> None: + """ + Handler for /stop. + Stops TradeThread + :param bot: telegram bot + :param update: message update + :return: None + """ + (error, msg) = self.rpc_stop() + self.send_msg(msg, bot=bot) + + @authorized_only + def _forcesell(self, bot: Bot, update: Update) -> None: + """ + Handler for /forcesell . + Sells the given trade at current price + :param bot: telegram bot + :param update: message update + :return: None + """ + + trade_id = update.message.text.replace('/forcesell', '').strip() + (error, message) = self.rpc_forcesell(trade_id) + if error: + self.send_msg(message, bot=bot) + return + + @authorized_only + def _performance(self, bot: Bot, update: Update) -> None: + """ + Handler for /performance. + Shows a performance statistic from finished trades + :param bot: telegram bot + :param update: message update + :return: None + """ + (error, trades) = self.rpc_performance() + if error: + self.send_msg(trades, bot=bot) + return + + stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( + index=i + 1, + pair=trade['pair'], + profit=trade['profit'], + count=trade['count'] + ) for i, trade in enumerate(trades)) + message = 'Performance:\n{}'.format(stats) + self.send_msg(message, parse_mode=ParseMode.HTML) + + @authorized_only + def _count(self, bot: Bot, update: Update) -> None: + """ + Handler for /count. + Returns the number of trades running + :param bot: telegram bot + :param update: message update + :return: None + """ + (error, trades) = self.rpc_count() + if error: + self.send_msg(trades, bot=bot) + return + + message = tabulate({ + 'current': [len(trades)], + 'max': [self._config['max_open_trades']] + }, headers=['current', 'max'], tablefmt='simple') + message = "
{}
".format(message) + self.logger.debug(message) + self.send_msg(message, parse_mode=ParseMode.HTML) + + @authorized_only + def _help(self, bot: Bot, update: Update) -> None: + """ + Handler for /help. + Show commands of the bot + :param bot: telegram bot + :param update: message update + :return: None + """ + message = "*/start:* `Starts the trader`\n" \ + "*/stop:* `Stops the trader`\n" \ + "*/status [table]:* `Lists all open trades`\n" \ + " *table :* `will display trades in a table`\n" \ + "*/profit:* `Lists cumulative profit from all finished trades`\n" \ + "*/forcesell |all:* `Instantly sells the given trade or all trades, " \ + "regardless of profit`\n" \ + "*/performance:* `Show performance of each finished trade grouped by pair`\n" \ + "*/daily :* `Shows profit or loss per day, over the last n days`\n" \ + "*/count:* `Show number of trades running compared to allowed number of trades`" \ + "\n" \ + "*/balance:* `Show account balance per currency`\n" \ + "*/help:* `This help message`\n" \ + "*/version:* `Show version`" + + self.send_msg(message, bot=bot) + + @authorized_only + def _version(self, bot: Bot, update: Update) -> None: + """ + Handler for /version. + Show version information + :param bot: telegram bot + :param update: message update + :return: None + """ + self.send_msg('*Version:* `{}`'.format(__version__), bot=bot) + + def send_msg(self, msg: str, bot: Bot = None, + parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: + """ + Send given markdown message + :param msg: message + :param bot: alternative bot + :param parse_mode: telegram parse mode + :return: None + """ + if not self.is_enabled(): + return + + bot = bot or self._updater.bot + + keyboard = [['/daily', '/profit', '/balance'], + ['/status', '/status table', '/performance'], + ['/count', '/start', '/stop', '/help']] + + reply_markup = ReplyKeyboardMarkup(keyboard) + + try: + try: + bot.send_message( + self._config['telegram']['chat_id'], + text=msg, + parse_mode=parse_mode, + reply_markup=reply_markup + ) + except NetworkError as network_err: + # Sometimes the telegram server resets the current connection, + # if this is the case we send the message again. + self.logger.warning( + 'Telegram NetworkError: %s! Trying one more time.', + network_err.message + ) + bot.send_message( + self._config['telegram']['chat_id'], + text=msg, + parse_mode=parse_mode, + reply_markup=reply_markup + ) + except TelegramError as telegram_err: + self.logger.warning( + 'TelegramError: %s! Giving up on that message.', + telegram_err.message ) - except NetworkError as network_err: - # Sometimes the telegram server resets the current connection, - # if this is the case we send the message again. - logger.warning( - 'Telegram NetworkError: %s! Trying one more time.', - network_err.message - ) - bot.send_message( - _CONF['telegram']['chat_id'], msg, - parse_mode=parse_mode, reply_markup=reply_markup - ) - except TelegramError as telegram_err: - logger.warning('TelegramError: %s! Giving up on that message.', telegram_err.message) diff --git a/freqtrade/state.py b/freqtrade/state.py new file mode 100644 index 000000000..aaa5b4765 --- /dev/null +++ b/freqtrade/state.py @@ -0,0 +1,14 @@ +# pragma pylint: disable=too-few-public-methods + +""" +Bot state constant +""" +import enum + + +class State(enum.Enum): + """ + Bot running states + """ + RUNNING = 0 + STOPPED = 1 diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 2247ecf27..ea37735b7 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -2,9 +2,10 @@ import talib.abstract as ta from pandas import DataFrame + import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.strategy.interface import IStrategy from freqtrade.indicator_helpers import fishers_inverse +from freqtrade.strategy.interface import IStrategy class_name = 'DefaultStrategy' diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dc9f33244..4eb73fb2e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -4,6 +4,7 @@ This module defines the interface to apply for strategies """ from abc import ABC, abstractmethod + from pandas import DataFrame diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index 27f334d5c..d7a89d1de 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -3,15 +3,16 @@ """ This module load custom strategies """ +import importlib import os import sys -import logging -import importlib from collections import OrderedDict from pandas import DataFrame -from freqtrade.strategy.interface import IStrategy +from freqtrade.constants import Constants +from freqtrade.logger import Logger +from freqtrade.strategy.interface import IStrategy sys.path.insert(0, r'../../user_data/strategies') @@ -20,32 +21,19 @@ class Strategy(object): """ This class contains all the logic to load custom strategy class """ - __instance = None - - DEFAULT_STRATEGY = 'default_strategy' - - def __new__(cls) -> object: - """ - Used to create the Singleton - :return: Strategy object - """ - if Strategy.__instance is None: - Strategy.__instance = object.__new__(cls) - return Strategy.__instance - - def init(self, config: dict) -> None: + def __init__(self, config: dict = {}) -> None: """ Load the custom class from config parameter :param config: :return: """ - self.logger = logging.getLogger(__name__) + self.logger = Logger(name=__name__).get_logger() # Verify the strategy is in the configuration, otherwise fallback to the default strategy if 'strategy' in config: strategy = config['strategy'] else: - strategy = self.DEFAULT_STRATEGY + strategy = Constants.DEFAULT_STRATEGY # Load the strategy self._load_strategy(strategy) @@ -72,12 +60,12 @@ class Strategy(object): # Minimal ROI designed for the strategy self.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.custom_strategy.minimal_roi.items()}.items(), - key=lambda tuple: tuple[0])) # sort after converting to number + key=lambda t: t[0])) # sort after converting to number # Optimal stoploss designed for the strategy self.stoploss = float(self.custom_strategy.stoploss) - self.ticker_interval = self.custom_strategy.ticker_interval + self.ticker_interval = int(self.custom_strategy.ticker_interval) def _load_strategy(self, strategy_name: str) -> None: """ diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index edeb89a59..07dc45a3e 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -1,17 +1,21 @@ # pragma pylint: disable=missing-docstring -from datetime import datetime -from unittest.mock import MagicMock -from functools import reduce - import json +import logging +from datetime import datetime +from functools import reduce +from unittest.mock import MagicMock + import arrow import pytest from jsonschema import validate +from sqlalchemy import create_engine from telegram import Chat, Message, Update -from freqtrade.analyze import parse_ticker_dataframe -from freqtrade.strategy.strategy import Strategy -from freqtrade.misc import CONF_SCHEMA +from freqtrade.analyze import Analyze +from freqtrade.constants import Constants +from freqtrade.freqtradebot import FreqtradeBot + +logging.getLogger('').setLevel(logging.INFO) def log_has(line, logs): @@ -22,6 +26,26 @@ def log_has(line, logs): False) +# Functions for recurrent object patching +def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: + """ + This function patch _init_modules() to not call dependencies + :param mocker: a Mocker object to apply patches + :param config: Config to pass to the bot + :return: None + """ + mocker.patch('freqtrade.fiat_convert.Market', {'price_usd': 12345.0}) + mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) + mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) + mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) + + return FreqtradeBot(config, create_engine('sqlite://')) + + @pytest.fixture(scope="module") def default_conf(): """ Returns validated configuration suitable for most tests """ @@ -61,9 +85,10 @@ def default_conf(): "token": "token", "chat_id": "0" }, - "initial_state": "running" + "initial_state": "running", + "loglevel": logging.DEBUG } - validate(configuration, CONF_SCHEMA) + validate(configuration, Constants.CONF_SCHEMA) return configuration @@ -265,14 +290,7 @@ def ticker_history_without_bv(): @pytest.fixture def result(): with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) - - -@pytest.fixture -def default_strategy(): - strategy = Strategy() - strategy.init({'strategy': 'default_strategy'}) - return strategy + return Analyze.parse_ticker_dataframe(json.load(data_file)) # FIX: @@ -280,3 +298,134 @@ def default_strategy(): # that inserts a trade of some type and open-status # return the open-order-id # See tests in rpc/main that could use this + + +@pytest.fixture +def get_market_summaries_data(): + """ + This fixture is a real result from exchange.get_market_summaries() but reduced to only + 8 entries. 4 BTC, 4 USTD + :return: JSON market summaries + """ + return [ + { + 'Ask': 1.316e-05, + 'BaseVolume': 5.72599471, + 'Bid': 1.3e-05, + 'Created': '2014-04-14T00:00:00', + 'High': 1.414e-05, + 'Last': 1.298e-05, + 'Low': 1.282e-05, + 'MarketName': 'BTC-XWC', + 'OpenBuyOrders': 2000, + 'OpenSellOrders': 1484, + 'PrevDay': 1.376e-05, + 'TimeStamp': '2018-02-05T01:32:40.493', + 'Volume': 424041.21418375 + }, + { + 'Ask': 0.00627051, + 'BaseVolume': 93.23302388, + 'Bid': 0.00618192, + 'Created': '2016-10-20T04:48:30.387', + 'High': 0.00669897, + 'Last': 0.00618192, + 'Low': 0.006, + 'MarketName': 'BTC-XZC', + 'OpenBuyOrders': 343, + 'OpenSellOrders': 2037, + 'PrevDay': 0.00668229, + 'TimeStamp': '2018-02-05T01:32:43.383', + 'Volume': 14863.60730702 + }, + { + 'Ask': 0.01137247, + 'BaseVolume': 383.55922657, + 'Bid': 0.01136006, + 'Created': '2016-11-15T20:29:59.73', + 'High': 0.012, + 'Last': 0.01137247, + 'Low': 0.01119883, + 'MarketName': 'BTC-ZCL', + 'OpenBuyOrders': 1332, + 'OpenSellOrders': 5317, + 'PrevDay': 0.01179603, + 'TimeStamp': '2018-02-05T01:32:42.773', + 'Volume': 33308.07358285 + }, + { + 'Ask': 0.04155821, + 'BaseVolume': 274.75369074, + 'Bid': 0.04130002, + 'Created': '2016-10-28T17:13:10.833', + 'High': 0.04354429, + 'Last': 0.041585, + 'Low': 0.0413, + 'MarketName': 'BTC-ZEC', + 'OpenBuyOrders': 863, + 'OpenSellOrders': 5579, + 'PrevDay': 0.0429, + 'TimeStamp': '2018-02-05T01:32:43.21', + 'Volume': 6479.84033259 + }, + { + 'Ask': 210.99999999, + 'BaseVolume': 615132.70989532, + 'Bid': 210.05503736, + 'Created': '2017-07-21T01:08:49.397', + 'High': 257.396, + 'Last': 211.0, + 'Low': 209.05333589, + 'MarketName': 'USDT-XMR', + 'OpenBuyOrders': 180, + 'OpenSellOrders': 1203, + 'PrevDay': 247.93528899, + 'TimeStamp': '2018-02-05T01:32:43.117', + 'Volume': 2688.17410793 + }, + { + 'Ask': 0.79589979, + 'BaseVolume': 9349557.01853031, + 'Bid': 0.789226, + 'Created': '2017-07-14T17:10:10.737', + 'High': 0.977, + 'Last': 0.79589979, + 'Low': 0.781, + 'MarketName': 'USDT-XRP', + 'OpenBuyOrders': 1075, + 'OpenSellOrders': 6508, + 'PrevDay': 0.93300218, + 'TimeStamp': '2018-02-05T01:32:42.383', + 'Volume': 10801663.00788851 + }, + { + 'Ask': 0.05154982, + 'BaseVolume': 2311087.71232136, + 'Bid': 0.05040107, + 'Created': '2017-12-29T19:29:18.357', + 'High': 0.06668561, + 'Last': 0.0508, + 'Low': 0.05006731, + 'MarketName': 'USDT-XVG', + 'OpenBuyOrders': 655, + 'OpenSellOrders': 5544, + 'PrevDay': 0.0627, + 'TimeStamp': '2018-02-05T01:32:41.507', + 'Volume': 40031424.2152716 + }, + { + 'Ask': 332.65500022, + 'BaseVolume': 562911.87455665, + 'Bid': 330.00000001, + 'Created': '2017-07-14T17:10:10.673', + 'High': 401.59999999, + 'Last': 332.65500019, + 'Low': 330.0, + 'MarketName': 'USDT-ZEC', + 'OpenBuyOrders': 161, + 'OpenSellOrders': 1731, + 'PrevDay': 391.42, + 'TimeStamp': '2018-02-05T01:32:42.947', + 'Volume': 1571.09647946 + } + ] diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 4f6eab083..f2874f2da 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1,15 +1,16 @@ # pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement # pragma pylint: disable=protected-access -from unittest.mock import MagicMock -from random import randint import logging -from requests.exceptions import RequestException -import pytest +from random import randint +from unittest.mock import MagicMock +import pytest +from requests.exceptions import RequestException + +import freqtrade.exchange as exchange from freqtrade import OperationalException from freqtrade.exchange import init, validate_pairs, buy, sell, get_balance, get_balances, \ get_ticker, get_ticker_history, cancel_order, get_name, get_fee -import freqtrade.exchange as exchange from freqtrade.tests.conftest import log_has API_INIT = False diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index 058c25de1..2c66215c2 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -1,10 +1,12 @@ # pragma pylint: disable=missing-docstring, C0103, protected-access, unused-argument from unittest.mock import MagicMock + import pytest from requests.exceptions import ContentDecodingError -from freqtrade.exchange.bittrex import Bittrex + import freqtrade.exchange.bittrex as btx +from freqtrade.exchange.bittrex import Bittrex # Eat this flake8 diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 36e14f00e..021474d5c 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,16 +1,28 @@ -# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103 -import random -import logging +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument + +import json import math +import random +from copy import deepcopy +from typing import List from unittest.mock import MagicMock -import pandas as pd + import numpy as np -from freqtrade import exchange, optimize -from freqtrade.exchange import Bittrex -from freqtrade.optimize import preprocess -from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe -import freqtrade.optimize.backtesting as backtesting -from freqtrade.tests.conftest import log_has +import pandas as pd +from arrow import Arrow + +from freqtrade import optimize +from freqtrade.analyze import Analyze +from freqtrade.arguments import Arguments +from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration +from freqtrade.tests.conftest import default_conf, log_has + +# Avoid to reinit the same object again and again +_BACKTESTING = Backtesting(default_conf()) + + +def get_args(args) -> List[str]: + return Arguments(args, '').get_parsed_arg() def trim_dictlist(dict_list, num): @@ -20,124 +32,6 @@ def trim_dictlist(dict_list, num): return new -# use for mock freqtrade.exchange.get_ticker_history' -def _load_pair_as_ticks(pair, tickfreq): - ticks = optimize.load_data(None, ticker_interval=8, pairs=[pair]) - ticks = trim_dictlist(ticks, -200) - return ticks[pair] - - -# FIX: fixturize this? -def _make_backtest_conf(conf=None, - pair='BTC_UNITEST', - record=None): - data = optimize.load_data(None, ticker_interval=8, pairs=[pair]) - data = trim_dictlist(data, -200) - return {'stake_amount': conf['stake_amount'], - 'processed': optimize.preprocess(data), - 'max_open_trades': 10, - 'realistic': True, - 'record': record} - - -def _trend(signals, buy_value, sell_value): - n = len(signals['low']) - buy = np.zeros(n) - sell = np.zeros(n) - for i in range(0, len(signals['buy'])): - if random.random() > 0.5: # Both buy and sell signals at same timeframe - buy[i] = buy_value - sell[i] = sell_value - signals['buy'] = buy - signals['sell'] = sell - return signals - - -def _trend_alternate(dataframe=None): - signals = dataframe - low = signals['low'] - n = len(low) - buy = np.zeros(n) - sell = np.zeros(n) - for i in range(0, len(buy)): - if i % 2 == 0: - buy[i] = 1 - else: - sell[i] = 1 - signals['buy'] = buy - signals['sell'] = sell - return dataframe - - -def _run_backtest_1(strategy, fun, backtest_conf): - # strategy is a global (hidden as a singleton), so we - # emulate strategy being pure, by override/restore here - # if we dont do this, the override in strategy will carry over - # to other tests - old_buy = strategy.populate_buy_trend - old_sell = strategy.populate_sell_trend - strategy.populate_buy_trend = fun # Override - strategy.populate_sell_trend = fun # Override - results = backtest(backtest_conf) - strategy.populate_buy_trend = old_buy # restore override - strategy.populate_sell_trend = old_sell # restore override - return results - - -def test_generate_text_table(): - results = pd.DataFrame( - { - 'currency': ['BTC_ETH', 'BTC_ETH'], - 'profit_percent': [0.1, 0.2], - 'profit_BTC': [0.2, 0.4], - 'duration': [10, 30], - 'profit': [2, 0], - 'loss': [0, 0] - } - ) - print(generate_text_table({'BTC_ETH': {}}, results, 'BTC')) - assert generate_text_table({'BTC_ETH': {}}, results, 'BTC') == ( - 'pair buy count avg profit % total profit BTC avg duration profit loss\n' # noqa - '------- ----------- -------------- ------------------ -------------- -------- ------\n' # noqa - 'BTC_ETH 2 15.00 0.60000000 20.0 2 0\n' # noqa - 'TOTAL 2 15.00 0.60000000 20.0 2 0') # noqa - - -def test_get_timeframe(default_strategy): - data = preprocess(optimize.load_data( - None, ticker_interval=1, pairs=['BTC_UNITEST'])) - min_date, max_date = get_timeframe(data) - assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' - assert max_date.isoformat() == '2017-11-14T22:59:00+00:00' - - -def test_backtest(default_strategy, default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) - - data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH']) - data = trim_dictlist(data, -200) - results = backtest({'stake_amount': default_conf['stake_amount'], - 'processed': optimize.preprocess(data), - 'max_open_trades': 10, - 'realistic': True}) - assert not results.empty - - -def test_backtest_1min_ticker_interval(default_strategy, default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) - - # Run a backtesting for an exiting 5min ticker_interval - data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST']) - data = trim_dictlist(data, -200) - results = backtest({'stake_amount': default_conf['stake_amount'], - 'processed': optimize.preprocess(data), - 'max_open_trades': 1, - 'realistic': True}) - assert not results.empty - - def load_data_test(what): timerange = ((None, 'line'), None, -100) data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST'], timerange=timerange) @@ -181,34 +75,429 @@ def load_data_test(what): return data -def simple_backtest(config, contour, num_results): +def simple_backtest(config, contour, num_results) -> None: + backtesting = _BACKTESTING + data = load_data_test(contour) - processed = optimize.preprocess(data) + processed = backtesting.tickerdata_to_dataframe(data) assert isinstance(processed, dict) - results = backtest({'stake_amount': config['stake_amount'], - 'processed': processed, - 'max_open_trades': 1, - 'realistic': True}) + results = backtesting.backtest( + { + 'stake_amount': config['stake_amount'], + 'processed': processed, + 'max_open_trades': 1, + 'realistic': True + } + ) # results :: assert len(results) == num_results +def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False, timerange=None): + tickerdata = optimize.load_tickerdata_file(datadir, 'BTC_UNITEST', 1, timerange=timerange) + pairdata = {'BTC_UNITEST': tickerdata} + return pairdata + + +# use for mock freqtrade.exchange.get_ticker_history' +def _load_pair_as_ticks(pair, tickfreq): + ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair]) + ticks = trim_dictlist(ticks, -200) + return ticks[pair] + + +# FIX: fixturize this? +def _make_backtest_conf(conf=None, pair='BTC_UNITEST', record=None): + data = optimize.load_data(None, ticker_interval=8, pairs=[pair]) + data = trim_dictlist(data, -200) + return { + 'stake_amount': conf['stake_amount'], + 'processed': _BACKTESTING.tickerdata_to_dataframe(data), + 'max_open_trades': 10, + 'realistic': True, + 'record': record + } + + +def _trend(signals, buy_value, sell_value): + n = len(signals['low']) + buy = np.zeros(n) + sell = np.zeros(n) + for i in range(0, len(signals['buy'])): + if random.random() > 0.5: # Both buy and sell signals at same timeframe + buy[i] = buy_value + sell[i] = sell_value + signals['buy'] = buy + signals['sell'] = sell + return signals + + +def _trend_alternate(dataframe=None): + signals = dataframe + low = signals['low'] + n = len(low) + buy = np.zeros(n) + sell = np.zeros(n) + for i in range(0, len(buy)): + if i % 2 == 0: + buy[i] = 1 + else: + sell[i] = 1 + signals['buy'] = buy + signals['sell'] = sell + return dataframe + + +def _run_backtest_1(fun, backtest_conf): + # strategy is a global (hidden as a singleton), so we + # emulate strategy being pure, by override/restore here + # if we dont do this, the override in strategy will carry over + # to other tests + old_buy = _BACKTESTING.populate_buy_trend + old_sell = _BACKTESTING.populate_sell_trend + _BACKTESTING.populate_buy_trend = fun # Override + _BACKTESTING.populate_sell_trend = fun # Override + results = _BACKTESTING.backtest(backtest_conf) + _BACKTESTING.populate_buy_trend = old_buy # restore override + _BACKTESTING.populate_sell_trend = old_sell # restore override + return results + + +# Unit tests +def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + 'backtesting' + ] + + config = setup_configuration(get_args(args)) + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert log_has( + 'Parameter --datadir detected: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert not log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + + assert 'live' not in config + assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples) + + assert 'realistic_simulation' not in config + assert not log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + + assert 'refresh_pairs' not in config + assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) + + assert 'timerange' not in config + assert 'export' not in config + + +def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + '--datadir', '/foo/bar', + 'backtesting', + '--ticker-interval', '1', + '--live', + '--realistic-simulation', + '--refresh-pairs-cached', + '--timerange', ':100', + '--export', '/bar/foo' + ] + + config = setup_configuration(get_args(args)) + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert log_has( + 'Parameter --datadir detected: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + assert log_has( + 'Using ticker_interval: 1 ...', + caplog.record_tuples + ) + + assert 'live' in config + assert log_has('Parameter -l/--live detected ...', caplog.record_tuples) + + assert 'realistic_simulation'in config + assert log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert log_has('Using max_open_trades: 1 ...', caplog.record_tuples) + + assert 'refresh_pairs'in config + assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) + assert 'timerange' in config + assert log_has( + 'Parameter --timerange detected: {} ...'.format(config['timerange']), + caplog.record_tuples + ) + + assert 'export' in config + assert log_has( + 'Parameter --export detected: {} ...'.format(config['export']), + caplog.record_tuples + ) + + +def test_start(mocker, default_conf, caplog) -> None: + """ + Test start() function + """ + start_mock = MagicMock() + mocker.patch('freqtrade.optimize.backtesting.Backtesting.start', start_mock) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + 'backtesting' + ] + args = get_args(args) + start(args) + assert log_has( + 'Starting freqtrade in Backtesting mode', + caplog.record_tuples + ) + assert start_mock.call_count == 1 + + +def test_backtesting__init__(mocker, default_conf) -> None: + """ + Test Backtesting.__init__() method + """ + init_mock = MagicMock() + mocker.patch('freqtrade.optimize.backtesting.Backtesting._init', init_mock) + + backtesting = Backtesting(default_conf) + assert backtesting.config == default_conf + assert backtesting.analyze is None + assert backtesting.ticker_interval is None + assert backtesting.tickerdata_to_dataframe is None + assert backtesting.populate_buy_trend is None + assert backtesting.populate_sell_trend is None + assert init_mock.call_count == 1 + + +def test_backtesting_init(default_conf) -> None: + """ + Test Backtesting._init() method + """ + backtesting = Backtesting(default_conf) + assert backtesting.config == default_conf + assert isinstance(backtesting.analyze, Analyze) + assert backtesting.ticker_interval == 5 + assert callable(backtesting.tickerdata_to_dataframe) + assert callable(backtesting.populate_buy_trend) + assert callable(backtesting.populate_sell_trend) + + +def test_tickerdata_to_dataframe(default_conf) -> None: + """ + Test Backtesting.tickerdata_to_dataframe() method + """ + + timerange = ((None, 'line'), None, -100) + tick = optimize.load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) + tickerlist = {'BTC_UNITEST': tick} + + backtesting = _BACKTESTING + data = backtesting.tickerdata_to_dataframe(tickerlist) + assert len(data['BTC_UNITEST']) == 100 + + # Load Analyze to compare the result between Backtesting function and Analyze are the same + analyze = Analyze(default_conf) + data2 = analyze.tickerdata_to_dataframe(tickerlist) + assert data['BTC_UNITEST'].equals(data2['BTC_UNITEST']) + + +def test_get_timeframe() -> None: + """ + Test Backtesting.get_timeframe() method + """ + backtesting = _BACKTESTING + + data = backtesting.tickerdata_to_dataframe( + optimize.load_data( + None, + ticker_interval=1, + pairs=['BTC_UNITEST'] + ) + ) + min_date, max_date = backtesting.get_timeframe(data) + assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' + assert max_date.isoformat() == '2017-11-14T22:59:00+00:00' + + +def test_generate_text_table(): + """ + Test Backtesting.generate_text_table() method + """ + backtesting = _BACKTESTING + + results = pd.DataFrame( + { + 'currency': ['BTC_ETH', 'BTC_ETH'], + 'profit_percent': [0.1, 0.2], + 'profit_BTC': [0.2, 0.4], + 'duration': [10, 30], + 'profit': [2, 0], + 'loss': [0, 0] + } + ) + + result_str = ( + 'pair buy count avg profit % ' + 'total profit BTC avg duration profit loss\n' + '------- ----------- -------------- ' + '------------------ -------------- -------- ------\n' + 'BTC_ETH 2 15.00 ' + '0.60000000 20.0 2 0\n' + 'TOTAL 2 15.00 ' + '0.60000000 20.0 2 0' + ) + + assert backtesting._generate_text_table(data={'BTC_ETH': {}}, results=results) == result_str + + +def test_backtesting_start(default_conf, mocker, caplog) -> None: + """ + Test Backtesting.start() method + """ + def get_timeframe(input1, input2): + return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) + + mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) + mocker.patch('freqtrade.optimize.load_data', mocked_load_data) + mocker.patch('freqtrade.exchange.get_ticker_history') + mocker.patch.multiple( + 'freqtrade.optimize.backtesting.Backtesting', + backtest=MagicMock(), + _generate_text_table=MagicMock(return_value='1'), + get_timeframe=get_timeframe, + ) + + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'] = ['BTC_UNITEST'] + conf['ticker_interval'] = 1 + conf['live'] = False + conf['datadir'] = None + conf['export'] = None + conf['timerange'] = '-100' + + backtesting = Backtesting(conf) + backtesting.start() + # check the logs, that will contain the backtest result + exists = [ + 'Using local backtesting data (using whitelist in given config) ...', + 'Using stake_currency: BTC ...', + 'Using stake_amount: 0.001 ...', + 'Measuring data from 2017-11-14T21:17:00+00:00 ' + 'up to 2017-11-14T22:59:00+00:00 (0 days)..' + ] + for line in exists: + assert log_has(line, caplog.record_tuples) + + +def test_backtest(default_conf) -> None: + """ + Test Backtesting.backtest() method + """ + backtesting = _BACKTESTING + + data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH']) + data = trim_dictlist(data, -200) + results = backtesting.backtest( + { + 'stake_amount': default_conf['stake_amount'], + 'processed': backtesting.tickerdata_to_dataframe(data), + 'max_open_trades': 10, + 'realistic': True + } + ) + assert not results.empty + + +def test_backtest_1min_ticker_interval(default_conf) -> None: + """ + Test Backtesting.backtest() method with 1 min ticker + """ + backtesting = _BACKTESTING + + # Run a backtesting for an exiting 5min ticker_interval + data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST']) + data = trim_dictlist(data, -200) + results = backtesting.backtest( + { + 'stake_amount': default_conf['stake_amount'], + 'processed': backtesting.tickerdata_to_dataframe(data), + 'max_open_trades': 1, + 'realistic': True + } + ) + assert not results.empty + + +def test_processed() -> None: + """ + Test Backtesting.backtest() method with offline data + """ + backtesting = _BACKTESTING + + dict_of_tickerrows = load_data_test('raise') + dataframes = backtesting.tickerdata_to_dataframe(dict_of_tickerrows) + dataframe = dataframes['BTC_UNITEST'] + cols = dataframe.columns + # assert the dataframe got some of the indicator columns + for col in ['close', 'high', 'low', 'open', 'date', + 'ema50', 'ao', 'macd', 'plus_dm']: + assert col in cols + + +def test_backtest_pricecontours(default_conf) -> None: + tests = [['raise', 17], ['lower', 0], ['sine', 17]] + for [contour, numres] in tests: + simple_backtest(default_conf, contour, numres) + + # Test backtest using offline data (testdata directory) - - -def test_backtest_ticks(default_conf, mocker, default_strategy): - mocker.patch.dict('freqtrade.main._CONF', default_conf) +def test_backtest_ticks(default_conf): ticks = [1, 5] - fun = default_strategy.populate_buy_trend + fun = _BACKTESTING.populate_buy_trend for tick in ticks: backtest_conf = _make_backtest_conf(conf=default_conf) - results = _run_backtest_1(default_strategy, fun, backtest_conf) + results = _run_backtest_1(fun, backtest_conf) assert not results.empty -def test_backtest_clash_buy_sell(default_conf, mocker, default_strategy): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - +def test_backtest_clash_buy_sell(default_conf): # Override the default buy trend function in our default_strategy def fun(dataframe=None): buy_value = 1 @@ -216,13 +505,11 @@ def test_backtest_clash_buy_sell(default_conf, mocker, default_strategy): return _trend(dataframe, buy_value, sell_value) backtest_conf = _make_backtest_conf(conf=default_conf) - results = _run_backtest_1(default_strategy, fun, backtest_conf) + results = _run_backtest_1(fun, backtest_conf) assert results.empty -def test_backtest_only_sell(default_conf, mocker, default_strategy): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - +def test_backtest_only_sell(default_conf): # Override the default buy trend function in our default_strategy def fun(dataframe=None): buy_value = 0 @@ -230,31 +517,29 @@ def test_backtest_only_sell(default_conf, mocker, default_strategy): return _trend(dataframe, buy_value, sell_value) backtest_conf = _make_backtest_conf(conf=default_conf) - results = _run_backtest_1(default_strategy, fun, backtest_conf) + results = _run_backtest_1(fun, backtest_conf) assert results.empty -def test_backtest_alternate_buy_sell(default_conf, mocker, default_strategy): - mocker.patch.dict('freqtrade.main._CONF', default_conf) +def test_backtest_alternate_buy_sell(default_conf): backtest_conf = _make_backtest_conf(conf=default_conf, pair='BTC_UNITEST') - results = _run_backtest_1(default_strategy, _trend_alternate, - backtest_conf) + results = _run_backtest_1(_trend_alternate, backtest_conf) assert len(results) == 3 -def test_backtest_record(default_conf, mocker, default_strategy): +def test_backtest_record(default_conf, mocker): names = [] records = [] - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.misc.file_dump_json', - new=lambda n, r: (names.append(n), records.append(r))) + mocker.patch( + 'freqtrade.optimize.backtesting.file_dump_json', + new=lambda n, r: (names.append(n), records.append(r)) + ) backtest_conf = _make_backtest_conf( conf=default_conf, pair='BTC_UNITEST', record="trades" ) - results = _run_backtest_1(default_strategy, _trend_alternate, - backtest_conf) + results = _run_backtest_1(_trend_alternate, backtest_conf) assert len(results) == 3 # Assert file_dump_json was only called once assert names == ['backtest-result.json'] @@ -277,74 +562,48 @@ def test_backtest_record(default_conf, mocker, default_strategy): assert dur > 0 -def test_processed(default_conf, mocker, default_strategy): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - dict_of_tickerrows = load_data_test('raise') - dataframes = optimize.preprocess(dict_of_tickerrows) - dataframe = dataframes['BTC_UNITEST'] - cols = dataframe.columns - # assert the dataframe got some of the indicator columns - for col in ['close', 'high', 'low', 'open', 'date', - 'ema50', 'ao', 'macd', 'plus_dm']: - assert col in cols - - -def test_backtest_pricecontours(default_conf, mocker, default_strategy): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - tests = [['raise', 17], ['lower', 0], ['sine', 17]] - for [contour, numres] in tests: - simple_backtest(default_conf, contour, numres) - - -def mocked_load_data(datadir, pairs=[], ticker_interval=0, refresh_pairs=False, timerange=None): - tickerdata = optimize.load_tickerdata_file(datadir, 'BTC_UNITEST', 1, timerange=timerange) - pairdata = {'BTC_UNITEST': tickerdata} - return pairdata - - -def test_backtest_start(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - default_conf['exchange']['pair_whitelist'] = ['BTC_UNITEST'] - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.misc.load_config', new=lambda s: default_conf) - mocker.patch.multiple('freqtrade.optimize', - load_data=mocked_load_data) - args = MagicMock() - args.ticker_interval = 1 - args.level = 10 - args.live = False - args.datadir = None - args.export = None - args.timerange = '-100' # needed due to MagicMock malleability - backtesting.start(args) - # check the logs, that will contain the backtest result - exists = ['Using max_open_trades: 1 ...', - 'Using stake_amount: 0.001 ...', - 'Measuring data from 2017-11-14T21:17:00+00:00 ' - 'up to 2017-11-14T22:59:00+00:00 (0 days)..'] - for line in exists: - assert log_has(line, caplog.record_tuples) - - -def test_backtest_start_live(default_strategy, default_conf, mocker, caplog): - caplog.set_level(logging.INFO) +def test_backtest_start_live(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['BTC_UNITEST'] mocker.patch('freqtrade.exchange.get_ticker_history', new=lambda n, i: _load_pair_as_ticks(n, i)) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.misc.load_config', new=lambda s: default_conf) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock()) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + args = MagicMock() args.ticker_interval = 1 args.level = 10 args.live = True args.datadir = None args.export = None + args.strategy = 'default_strategy' args.timerange = '-100' # needed due to MagicMock malleability - backtesting.start(args) + + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + 'backtesting', + '--ticker-interval', '1', + '--live', + '--timerange', '-100' + ] + args = get_args(args) + start(args) # check the logs, that will contain the backtest result - exists = ['Using max_open_trades: 1 ...', - 'Using stake_amount: 0.001 ...', - 'Measuring data from 2017-11-14T19:32:00+00:00 ' - 'up to 2017-11-14T22:59:00+00:00 (0 days)..'] + exists = [ + 'Parameter -i/--ticker-interval detected ...', + 'Using ticker_interval: 1 ...', + 'Parameter -l/--live detected ...', + 'Using max_open_trades: 1 ...', + 'Parameter --timerange detected: -100 ..', + 'Parameter --datadir detected: freqtrade/tests/testdata ...', + 'Using stake_currency: BTC ...', + 'Using stake_amount: 0.001 ...', + 'Downloading data for all pairs in whitelist ...', + 'Measuring data from 2017-11-14T19:32:00+00:00 up to 2017-11-14T22:59:00+00:00 (0 days)..' + ] + for line in exists: - assert log_has(line, caplog.record_tuples) + log_has(line, caplog.record_tuples) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 13b924f1d..6d376471a 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,125 +1,141 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import logging - +import json +import os +from copy import deepcopy from unittest.mock import MagicMock import pandas as pd -from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ - log_results, save_trials, read_trials, generate_roi_table, has_space +from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.optimize.hyperopt import Hyperopt, start from freqtrade.strategy.strategy import Strategy -import freqtrade.optimize.hyperopt as hyperopt +from freqtrade.tests.conftest import default_conf, log_has +from freqtrade.tests.optimize.test_backtesting import get_args -def test_loss_calculation_prefer_correct_trade_count(): - correct = calculate_loss(1, TARGET_TRADES, 20) - over = calculate_loss(1, TARGET_TRADES + 100, 20) - under = calculate_loss(1, TARGET_TRADES - 100, 20) - assert over > correct - assert under > correct +# Avoid to reinit the same object again and again +_HYPEROPT = Hyperopt(default_conf()) -def test_loss_calculation_prefer_shorter_trades(): - shorter = calculate_loss(1, 100, 20) - longer = calculate_loss(1, 100, 30) - assert shorter < longer - - -def test_loss_calculation_has_limited_profit(): - correct = calculate_loss(EXPECTED_MAX_PROFIT, TARGET_TRADES, 20) - over = calculate_loss(EXPECTED_MAX_PROFIT * 2, TARGET_TRADES, 20) - under = calculate_loss(EXPECTED_MAX_PROFIT / 2, TARGET_TRADES, 20) - assert over == correct - assert under > correct - - -def create_trials(mocker): +# Functions for recurrent object patching +def create_trials(mocker) -> None: """ When creating trials, mock the hyperopt Trials so that *by default* - we don't create any pickle'd files in the filesystem - we might have a pickle'd file so make sure that we return false when looking for it """ - mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', - return_value='freqtrade/tests/optimize/ut_trials.pickle') - mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', - return_value=False) - mocker.patch('freqtrade.optimize.hyperopt.save_trials', - return_value=None) - mocker.patch('freqtrade.optimize.hyperopt.read_trials', - return_value=None) - mocker.patch('freqtrade.optimize.hyperopt.os.remove', - return_value=True) + _HYPEROPT.trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') + + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False) + mocker.patch('freqtrade.optimize.hyperopt.os.path.getsize', return_value=1) + mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + return mocker.Mock( - results=[{ - 'loss': 1, - 'result': 'foo', - 'status': 'ok' - }], + results=[ + { + 'loss': 1, + 'result': 'foo', + 'status': 'ok' + } + ], best_trial={'misc': {'vals': {'adx': 999}}} ) -def test_start_calls_fmin(mocker): - trials = create_trials(mocker) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials) - mocker.patch('freqtrade.optimize.hyperopt.sorted', - return_value=trials.results) - mocker.patch('freqtrade.optimize.preprocess') - mocker.patch('freqtrade.optimize.load_data') - mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - - args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False, - timerange=None, spaces='all') - Strategy().init({'strategy': 'default_strategy'}) +# Unit tests +def test_start(mocker, default_conf, caplog) -> None: + """ + Test start() function + """ + start_mock = MagicMock() + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + 'hyperopt', + '--epochs', '5' + ] + args = get_args(args) + Strategy({'strategy': 'default_strategy'}) start(args) - mock_fmin.assert_called_once() + import pprint + pprint.pprint(caplog.record_tuples) + + assert log_has( + 'Starting freqtrade in Hyperopt mode', + caplog.record_tuples + ) + assert start_mock.call_count == 1 -def test_start_uses_mongotrials(mocker): - mock_mongotrials = mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', - return_value=create_trials(mocker)) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.load_data') - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) +def test_loss_calculation_prefer_correct_trade_count() -> None: + """ + Test Hyperopt.calculate_loss() + """ + hyperopt = _HYPEROPT + Strategy({'strategy': 'default_strategy'}) - args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True, - timerange=None, spaces='all') - Strategy().init({'strategy': 'default_strategy'}) - start(args) - - mock_mongotrials.assert_called_once() + correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20) + over = hyperopt.calculate_loss(1, hyperopt.target_trades + 100, 20) + under = hyperopt.calculate_loss(1, hyperopt.target_trades - 100, 20) + assert over > correct + assert under > correct -def test_log_results_if_loss_improves(mocker): - logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info') - global CURRENT_BEST_LOSS - CURRENT_BEST_LOSS = 2 - log_results({ - 'loss': 1, - 'current_tries': 1, - 'total_tries': 2, - 'result': 'foo' - }) +def test_loss_calculation_prefer_shorter_trades() -> None: + """ + Test Hyperopt.calculate_loss() + """ + hyperopt = _HYPEROPT - logger.assert_called_once() + shorter = hyperopt.calculate_loss(1, 100, 20) + longer = hyperopt.calculate_loss(1, 100, 30) + assert shorter < longer -def test_no_log_if_loss_does_not_improve(mocker): - logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info') - global CURRENT_BEST_LOSS - CURRENT_BEST_LOSS = 2 - log_results({ - 'loss': 3, - }) +def test_loss_calculation_has_limited_profit() -> None: + hyperopt = _HYPEROPT - assert not logger.called + correct = hyperopt.calculate_loss(hyperopt.expected_max_profit, hyperopt.target_trades, 20) + over = hyperopt.calculate_loss(hyperopt.expected_max_profit * 2, hyperopt.target_trades, 20) + under = hyperopt.calculate_loss(hyperopt.expected_max_profit / 2, hyperopt.target_trades, 20) + assert over == correct + assert under > correct -def test_fmin_best_results(mocker, caplog): - caplog.set_level(logging.INFO) +def test_log_results_if_loss_improves(caplog) -> None: + hyperopt = _HYPEROPT + hyperopt.current_best_loss = 2 + hyperopt.log_results( + { + 'loss': 1, + 'current_tries': 1, + 'total_tries': 2, + 'result': 'foo' + } + ) + assert log_has(' 1/2: foo. Loss 1.00000', caplog.record_tuples) + + +def test_no_log_if_loss_does_not_improve(caplog) -> None: + hyperopt = _HYPEROPT + hyperopt.current_best_loss = 2 + hyperopt.log_results( + { + 'loss': 3, + } + ) + assert caplog.record_tuples == [] + + +def test_fmin_best_results(mocker, default_conf, caplog) -> None: fmin_result = { "macd_below_zero": 0, "adx": 1, @@ -144,41 +160,67 @@ def test_fmin_best_results(mocker, caplog): "roi_p3": 3, } - mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', return_value=create_trials(mocker)) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.load_data') - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'timerange': None}) + conf.update({'spaces': 'all'}) - args = mocker.Mock(epochs=1, config='config.json.example', - timerange=None, spaces='all') - Strategy().init({'strategy': 'default_strategy'}) - start(args) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + + Strategy({'strategy': 'default_strategy'}) + hyperopt = Hyperopt(conf) + hyperopt.trials = create_trials(mocker) + hyperopt.tickerdata_to_dataframe = MagicMock() + hyperopt.start() exists = [ - 'Best parameters', + 'Best parameters:', '"adx": {\n "enabled": true,\n "value": 15.0\n },', + '"fastd": {\n "enabled": true,\n "value": 40.0\n },', '"green_candle": {\n "enabled": true\n },', + '"macd_below_zero": {\n "enabled": false\n },', '"mfi": {\n "enabled": false\n },', + '"over_sar": {\n "enabled": false\n },', + '"roi_p1": 1.0,', + '"roi_p2": 2.0,', + '"roi_p3": 3.0,', + '"roi_t1": 1.0,', + '"roi_t2": 2.0,', + '"roi_t3": 3.0,', + '"rsi": {\n "enabled": true,\n "value": 37.0\n },', + '"stoploss": -0.1,', '"trigger": {\n "type": "faststoch10"\n },', - '"stoploss": -0.1', + '"uptrend_long_ema": {\n "enabled": true\n },', + '"uptrend_short_ema": {\n "enabled": false\n },', + '"uptrend_sma": {\n "enabled": false\n }', + 'ROI table:\n{0: 6.0, 3.0: 3.0, 5.0: 1.0, 6.0: 0}', + 'Best Result:\nfoo' ] - for line in exists: assert line in caplog.text -def test_fmin_throw_value_error(mocker, caplog): - caplog.set_level(logging.INFO) - Strategy().init({'strategy': 'default_strategy'}) - mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', return_value=create_trials(mocker)) - mocker.patch('freqtrade.optimize.tickerdata_to_dataframe') - mocker.patch('freqtrade.optimize.load_data') +def test_fmin_throw_value_error(mocker, default_conf, caplog) -> None: + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError()) - args = mocker.Mock(epochs=1, config='config.json.example', - timerange=None, spaces='all') - Strategy().init({'strategy': 'default_strategy'}) - start(args) + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'timerange': None}) + conf.update({'spaces': 'all'}) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + Strategy({'strategy': 'default_strategy'}) + hyperopt = Hyperopt(conf) + hyperopt.trials = create_trials(mocker) + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() exists = [ 'Best Result:', @@ -190,69 +232,82 @@ def test_fmin_throw_value_error(mocker, caplog): assert line in caplog.text -def test_resuming_previous_hyperopt_results_succeeds(mocker): - import freqtrade.optimize.hyperopt as hyperopt +def test_resuming_previous_hyperopt_results_succeeds(mocker, default_conf) -> None: trials = create_trials(mocker) - mocker.patch('freqtrade.optimize.hyperopt.TRIALS', - return_value=trials) - mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', - return_value=True) - mocker.patch('freqtrade.optimize.hyperopt.len', - return_value=len(trials.results)) - mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials', - return_value=trials) - mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials', - return_value=None) - mocker.patch('freqtrade.optimize.hyperopt.sorted', - return_value=trials.results) - mocker.patch('freqtrade.optimize.preprocess') - mocker.patch('freqtrade.optimize.load_data') - mocker.patch('freqtrade.optimize.hyperopt.fmin', - return_value={}) - args = mocker.Mock(epochs=1, - config='config.json.example', - mongodb=False, - timerange=None, - spaces='all') - Strategy().init({'strategy': 'default_strategy'}) - start(args) + + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'mongodb': False}) + conf.update({'timerange': None}) + conf.update({'spaces': 'all'}) + + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.len', return_value=len(trials.results)) + mock_read = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.read_trials', + return_value=trials + ) + mock_save = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.save_trials', + return_value=None + ) + mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) + + Strategy({'strategy': 'default_strategy'}) + hyperopt = Hyperopt(conf) + hyperopt.trials = trials + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() mock_read.assert_called_once() mock_save.assert_called_once() - current_tries = hyperopt._CURRENT_TRIES - total_tries = hyperopt.TOTAL_TRIES + current_tries = hyperopt.current_tries + total_tries = hyperopt.total_tries assert current_tries == len(trials.results) assert total_tries == (current_tries + len(trials.results)) -def test_save_trials_saves_trials(mocker): +def test_save_trials_saves_trials(mocker, caplog) -> None: + create_trials(mocker) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + + hyperopt = _HYPEROPT + mocker.patch('freqtrade.optimize.hyperopt.open', return_value=hyperopt.trials_file) + + hyperopt.save_trials() + + assert log_has( + 'Saving Trials to \'freqtrade/tests/optimize/ut_trials.pickle\'', + caplog.record_tuples + ) + mock_dump.assert_called_once() + + +def test_read_trials_returns_trials_file(mocker, caplog) -> None: trials = create_trials(mocker) - mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', - return_value=None) - trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', - return_value='ut_trials.pickle') - mocker.patch('freqtrade.optimize.hyperopt.open', - return_value=trials_path) - save_trials(trials, trials_path) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', return_value=trials) + mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', return_value=mock_load) - mock_dump.assert_called_once_with(trials, trials_path) - - -def test_read_trials_returns_trials_file(mocker): - trials = create_trials(mocker) - mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', - return_value=trials) - mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', - return_value=mock_load) - - assert read_trials() == trials + hyperopt = _HYPEROPT + hyperopt_trial = hyperopt.read_trials() + assert log_has( + 'Reading Trials from \'freqtrade/tests/optimize/ut_trials.pickle\'', + caplog.record_tuples + ) + assert hyperopt_trial == trials mock_open.assert_called_once() mock_load.assert_called_once() -def test_roi_table_generation(): +def test_roi_table_generation() -> None: params = { 'roi_t1': 5, 'roi_t2': 10, @@ -261,7 +316,54 @@ def test_roi_table_generation(): 'roi_p2': 2, 'roi_p3': 3, } - assert generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} + + hyperopt = _HYPEROPT + assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} + + +def test_start_calls_fmin(mocker, default_conf) -> None: + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'mongodb': False}) + conf.update({'timerange': None}) + conf.update({'spaces': 'all'}) + + hyperopt = Hyperopt(conf) + hyperopt.trials = trials + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() + mock_fmin.assert_called_once() + + +def test_start_uses_mongotrials(mocker, default_conf) -> None: + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) + mock_mongotrials = mocker.patch( + 'freqtrade.optimize.hyperopt.MongoTrials', + return_value=create_trials(mocker) + ) + + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'epochs': 1}) + conf.update({'mongodb': True}) + conf.update({'timerange': None}) + conf.update({'spaces': 'all'}) + mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) + + hyperopt = Hyperopt(conf) + hyperopt.tickerdata_to_dataframe = MagicMock() + + hyperopt.start() + mock_mongotrials.assert_called_once() + mock_fmin.assert_called_once() # test log_trials_result @@ -269,26 +371,167 @@ def test_roi_table_generation(): # test optimizer if 'ro_t1' in params def test_format_results(): - trades = [('BTC_ETH', 2, 2, 123), - ('BTC_LTC', 1, 1, 123), - ('BTC_XRP', -1, -2, -246)] + """ + Test Hyperopt.format_results() + """ + trades = [ + ('BTC_ETH', 2, 2, 123), + ('BTC_LTC', 1, 1, 123), + ('BTC_XRP', -1, -2, -246) + ] labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] df = pd.DataFrame.from_records(trades, columns=labels) - x = hyperopt.format_results(df) + x = Hyperopt.format_results(df) assert x.find(' 66.67%') def test_signal_handler(mocker): + """ + Test Hyperopt.signal_handler() + """ m = MagicMock() mocker.patch('sys.exit', m) - mocker.patch('freqtrade.optimize.hyperopt.save_trials', m) - mocker.patch('freqtrade.optimize.hyperopt.log_trials_result', m) + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.save_trials', m) + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.log_trials_result', m) + + hyperopt = _HYPEROPT hyperopt.signal_handler(9, None) assert m.call_count == 3 def test_has_space(): - assert has_space(['buy', 'roi'], 'roi') - assert has_space(['buy', 'roi'], 'buy') - assert not has_space(['buy', 'roi'], 'stoploss') - assert has_space(['all'], 'buy') + """ + Test Hyperopt.has_space() method + """ + _HYPEROPT.config.update({'spaces': ['buy', 'roi']}) + assert _HYPEROPT.has_space('roi') + assert _HYPEROPT.has_space('buy') + assert not _HYPEROPT.has_space('stoploss') + + _HYPEROPT.config.update({'spaces': ['all']}) + assert _HYPEROPT.has_space('buy') + + +def test_populate_indicators() -> None: + """ + Test Hyperopt.populate_indicators() + """ + tick = load_tickerdata_file(None, 'BTC_UNITEST', 1) + tickerlist = {'BTC_UNITEST': tick} + dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) + dataframe = _HYPEROPT.populate_indicators(dataframes['BTC_UNITEST']) + + # Check if some indicators are generated. We will not test all of them + assert 'adx' in dataframe + assert 'ao' in dataframe + assert 'cci' in dataframe + + +def test_buy_strategy_generator() -> None: + """ + Test Hyperopt.buy_strategy_generator() + """ + tick = load_tickerdata_file(None, 'BTC_UNITEST', 1) + tickerlist = {'BTC_UNITEST': tick} + dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) + dataframe = _HYPEROPT.populate_indicators(dataframes['BTC_UNITEST']) + + populate_buy_trend = _HYPEROPT.buy_strategy_generator( + { + 'uptrend_long_ema': { + 'enabled': True + }, + 'macd_below_zero': { + 'enabled': True + }, + 'uptrend_short_ema': { + 'enabled': True + }, + 'mfi': { + 'enabled': True, + 'value': 20 + }, + 'fastd': { + 'enabled': True, + 'value': 20 + }, + 'adx': { + 'enabled': True, + 'value': 20 + }, + 'rsi': { + 'enabled': True, + 'value': 20 + }, + 'over_sar': { + 'enabled': True, + }, + 'green_candle': { + 'enabled': True, + }, + 'uptrend_sma': { + 'enabled': True, + }, + + 'trigger': { + 'type': 'lower_bb' + } + } + ) + result = populate_buy_trend(dataframe) + # Check if some indicators are generated. We will not test all of them + assert 'buy' in result + assert 1 in result['buy'] + + +def test_generate_optimizer(mocker, default_conf) -> None: + """ + Test Hyperopt.generate_optimizer() function + """ + conf = deepcopy(default_conf) + conf.update({'config': 'config.json.example'}) + conf.update({'timerange': None}) + conf.update({'spaces': 'all'}) + + trades = [ + ('BTC_POWR', 0.023117, 0.000233, 100) + ] + labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + backtest_result = pd.DataFrame.from_records(trades, columns=labels) + + mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.backtest', + MagicMock(return_value=backtest_result) + ) + + optimizer_param = { + 'adx': {'enabled': False}, + 'fastd': {'enabled': True, 'value': 35.0}, + 'green_candle': {'enabled': True}, + 'macd_below_zero': {'enabled': True}, + 'mfi': {'enabled': False}, + 'over_sar': {'enabled': False}, + 'roi_p1': 0.01, + 'roi_p2': 0.01, + 'roi_p3': 0.1, + 'roi_t1': 60.0, + 'roi_t2': 30.0, + 'roi_t3': 20.0, + 'rsi': {'enabled': False}, + 'stoploss': -0.4, + 'trigger': {'type': 'macd_cross_signal'}, + 'uptrend_long_ema': {'enabled': False}, + 'uptrend_short_ema': {'enabled': True}, + 'uptrend_sma': {'enabled': True} + } + + response_expected = { + 'loss': 1.9840569076926293, + 'result': ' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' + '(0.0231Σ%). Avg duration 100.0 mins.', + 'status': 'ok' + } + + hyperopt = Hyperopt(conf) + generate_optimizer_value = hyperopt.generate_optimizer(optimizer_param) + assert generate_optimizer_value == response_expected diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index ed43f0f4e..e26d30534 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -1,16 +1,15 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 -import os import json -import logging +import os import uuid from shutil import copyfile -from freqtrade import exchange, optimize -from freqtrade.exchange import Bittrex -from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ - download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, file_dump_json + +from freqtrade import optimize +from freqtrade.misc import file_dump_json +from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \ + download_backtesting_testdata, load_tickerdata_file, trim_tickerlist from freqtrade.tests.conftest import log_has -from freqtrade.strategy.strategy import Strategy # Change this if modifying BTC_UNITEST testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -47,12 +46,11 @@ def _clean_test_file(file: str) -> None: os.rename(file_swp, file) -def test_load_data_30min_ticker(default_conf, ticker_history, mocker, caplog): - caplog.set_level(logging.INFO) +def test_load_data_30min_ticker(ticker_history, mocker, caplog) -> None: + """ + Test load_data() with 30 min ticker + """ mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) file = 'freqtrade/tests/testdata/BTC_UNITTEST-30.json' _backup_file(file, copy_file=True) @@ -62,12 +60,11 @@ def test_load_data_30min_ticker(default_conf, ticker_history, mocker, caplog): _clean_test_file(file) -def test_load_data_5min_ticker(default_conf, ticker_history, mocker, caplog): - caplog.set_level(logging.INFO) +def test_load_data_5min_ticker(ticker_history, mocker, caplog) -> None: + """ + Test load_data() with 5 min ticker + """ mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) file = 'freqtrade/tests/testdata/BTC_ETH-5.json' _backup_file(file, copy_file=True) @@ -77,12 +74,11 @@ def test_load_data_5min_ticker(default_conf, ticker_history, mocker, caplog): _clean_test_file(file) -def test_load_data_1min_ticker(default_conf, ticker_history, mocker, caplog): - caplog.set_level(logging.INFO) +def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: + """ + Test load_data() with 1 min ticker + """ mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) file = 'freqtrade/tests/testdata/BTC_ETH-1.json' _backup_file(file, copy_file=True) @@ -92,12 +88,11 @@ def test_load_data_1min_ticker(default_conf, ticker_history, mocker, caplog): _clean_test_file(file) -def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, caplog): - caplog.set_level(logging.INFO) +def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: + """ + Test load_data() with 1 min ticker + """ mocker.patch('freqtrade.optimize.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - exchange._API = Bittrex({'key': '', 'secret': ''}) file = 'freqtrade/tests/testdata/BTC_MEME-1.json' _backup_file(file) @@ -107,14 +102,12 @@ def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, capl _clean_test_file(file) -def test_testdata_path(): +def test_testdata_path() -> None: assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None) -def test_download_pairs(default_conf, ticker_history, mocker): +def test_download_pairs(ticker_history, mocker) -> None: mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json' file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json' @@ -151,13 +144,10 @@ def test_download_pairs(default_conf, ticker_history, mocker): _clean_test_file(file2_5) -def test_download_pairs_exception(default_conf, ticker_history, mocker, caplog): - caplog.set_level(logging.INFO) +def test_download_pairs_exception(ticker_history, mocker, caplog) -> None: mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', side_effect=BaseException('File Error')) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) file1_1 = 'freqtrade/tests/testdata/BTC_MEME-1.json' file1_5 = 'freqtrade/tests/testdata/BTC_MEME-5.json' @@ -171,10 +161,8 @@ def test_download_pairs_exception(default_conf, ticker_history, mocker, caplog): assert log_has('Failed to download the pair: "BTC-MEME", Interval: 1 min', caplog.record_tuples) -def test_download_backtesting_testdata(default_conf, ticker_history, mocker): +def test_download_backtesting_testdata(ticker_history, mocker) -> None: mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=ticker_history) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) # Download a 1 min ticker file file1 = 'freqtrade/tests/testdata/BTC_XEL-1.json' @@ -192,7 +180,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): _clean_test_file(file2) -def test_download_backtesting_testdata2(mocker): +def test_download_backtesting_testdata2(mocker) -> None: tick = [{'T': 'bar'}, {'T': 'foo'}] mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick) @@ -200,7 +188,7 @@ def test_download_backtesting_testdata2(mocker): assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3) -def test_load_tickerdata_file(): +def test_load_tickerdata_file() -> None: # 7 does not exist in either format. assert not load_tickerdata_file(None, 'BTC_UNITEST', 7) # 1 exists only as a .json @@ -211,23 +199,18 @@ def test_load_tickerdata_file(): assert _BTC_UNITTEST_LENGTH == len(tickerdata) -def test_init(default_conf, mocker): +def test_init(default_conf, mocker) -> None: conf = {'exchange': {'pair_whitelist': []}} mocker.patch('freqtrade.optimize.hyperopt_optimize_conf', return_value=conf) - assert {} == optimize.load_data('', pairs=[], refresh_pairs=True, - ticker_interval=int(default_conf['ticker_interval'])) + assert {} == optimize.load_data( + '', + pairs=[], + refresh_pairs=True, + ticker_interval=int(default_conf['ticker_interval']) + ) -def test_tickerdata_to_dataframe(): - Strategy().init({'strategy': 'default_strategy'}) - timerange = ((None, 'line'), None, -100) - tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) - tickerlist = {'BTC_UNITEST': tick} - data = optimize.tickerdata_to_dataframe(tickerlist) - assert len(data['BTC_UNITEST']) == 100 - - -def test_trim_tickerlist(): +def test_trim_tickerlist() -> None: with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: ticker_list = json.load(data_file) ticker_list_len = len(ticker_list) @@ -272,7 +255,11 @@ def test_trim_tickerlist(): assert ticker_list_len == ticker_len -def test_file_dump_json(): +def test_file_dump_json() -> None: + """ + Test file_dump_json() + :return: None + """ file = 'freqtrade/tests/testdata/test_{id}.json'.format(id=str(uuid.uuid4())) data = {'bar': 'foo'} diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 8102e8885..50943b1bc 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -1,203 +1,133 @@ -# pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 +# pragma pylint: disable=invalid-sequence-index, invalid-name, too-many-arguments + +""" +Unit test file for rpc/rpc.py +""" + from datetime import datetime -from copy import deepcopy from unittest.mock import MagicMock + from sqlalchemy import create_engine -from freqtrade.rpc import init, cleanup, send_msg +from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade -import freqtrade.main as main -import freqtrade.misc as misc -import freqtrade.rpc as rpc +from freqtrade.rpc.rpc import RPC +from freqtrade.state import State +from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap -def prec_satoshi(a, b): +# Functions for recurrent object patching +def prec_satoshi(a, b) -> float: """ :return: True if A and B differs less than one satoshi. """ return abs(a - b) < 0.00000001 -def test_init_telegram_enabled(default_conf, mocker): - module_list = [] - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', module_list) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.init', MagicMock()) +# Unit tests +def test_rpc_trade_status(default_conf, ticker, mocker) -> None: + """ + Test rpc_trade_status() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) - init(default_conf) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) - assert telegram_mock.call_count == 1 - assert 'telegram' in module_list - - -def test_init_telegram_disabled(default_conf, mocker): - module_list = [] - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', module_list) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.init', MagicMock()) - - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - init(conf) - - assert telegram_mock.call_count == 0 - assert 'telegram' not in module_list - - -def test_cleanup_telegram_enabled(mocker): - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', ['telegram']) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.cleanup', MagicMock()) - cleanup() - assert telegram_mock.call_count == 1 - - -def test_cleanup_telegram_disabled(mocker): - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', []) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.cleanup', MagicMock()) - cleanup() - assert telegram_mock.call_count == 0 - - -def test_send_msg_telegram_enabled(mocker): - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', ['telegram']) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.send_msg', MagicMock()) - send_msg('test') - assert telegram_mock.call_count == 1 - - -def test_send_msg_telegram_disabled(mocker): - mocker.patch('freqtrade.rpc.REGISTERED_MODULES', []) - telegram_mock = mocker.patch('freqtrade.rpc.telegram.send_msg', MagicMock()) - send_msg('test') - assert telegram_mock.call_count == 0 - - -def test_rpc_forcesell(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - cancel_order_mock = MagicMock() - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - cancel_order=cancel_order_mock, - get_order=MagicMock(return_value={ - 'closed': True, - 'type': 'LIMIT_BUY', - })) - main.init(default_conf, create_engine('sqlite://')) - - misc.update_state(misc.State.STOPPED) - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == '`trader is not running`' - misc.update_state(misc.State.RUNNING) - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == 'Invalid argument.' - - (error, res) = rpc.rpc_forcesell('all') - assert not error - assert res == '' - - main.create_trade(0.001, 5) - (error, res) = rpc.rpc_forcesell('all') - assert not error - assert res == '' - - (error, res) = rpc.rpc_forcesell('1') - assert not error - assert res == '' - - misc.update_state(misc.State.STOPPED) - - (error, res) = rpc.rpc_forcesell(None) - assert error - assert res == '`trader is not running`' - - (error, res) = rpc.rpc_forcesell('all') - assert error - assert res == '`trader is not running`' - - misc.update_state(misc.State.RUNNING) - - assert cancel_order_mock.call_count == 0 - # make an limit-buy open trade - mocker.patch.multiple('freqtrade.exchange', - get_order=MagicMock(return_value={ - 'closed': None, - 'type': 'LIMIT_BUY' - })) - # check that the trade is called, which is done - # by ensuring exchange.cancel_order is called - (error, res) = rpc.rpc_forcesell('1') - assert not error - assert res == '' - assert cancel_order_mock.call_count == 1 - - main.create_trade(0.001, 5) - # make an limit-sell open trade - mocker.patch.multiple('freqtrade.exchange', - get_order=MagicMock(return_value={ - 'closed': None, - 'type': 'LIMIT_SELL' - })) - (error, res) = rpc.rpc_forcesell('2') - assert not error - assert res == '' - # status quo, no exchange calls - assert cancel_order_mock.call_count == 1 - - -def test_rpc_trade_status(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - main.init(default_conf, create_engine('sqlite://')) - - misc.update_state(misc.State.STOPPED) + freqtradebot.update_state(State.STOPPED) (error, result) = rpc.rpc_trade_status() assert error - assert result.find('trader is not running') >= 0 + assert 'trader is not running' in result - misc.update_state(misc.State.RUNNING) + freqtradebot.update_state(State.RUNNING) (error, result) = rpc.rpc_trade_status() assert error - assert result.find('no active trade') >= 0 + assert 'no active trade' in result - main.create_trade(0.001, 5) + freqtradebot.create_trade() (error, result) = rpc.rpc_trade_status() assert not error trade = result[0] + + result_message = [ + '*Trade ID:* `1`\n' + '*Current Pair:* ' + '[BTC_ETH](https://www.bittrex.com/Market/Index?MarketName=BTC-ETH)\n' + '*Open Since:* `just now`\n' + '*Amount:* `90.99181074`\n' + '*Open Rate:* `0.00001099`\n' + '*Close Rate:* `None`\n' + '*Current Rate:* `0.00001098`\n' + '*Close Profit:* `None`\n' + '*Current Profit:* `-0.59%`\n' + '*Open Order:* `(LIMIT_BUY rem=0.00000000)`' + ] + assert result == result_message assert trade.find('[BTC_ETH]') >= 0 -def test_rpc_daily_profit(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0})) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - main.init(default_conf, create_engine('sqlite://')) +def test_rpc_status_table(default_conf, ticker, mocker) -> None: + """ + Test rpc_status_table() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + (error, result) = rpc.rpc_status_table() + assert error + assert '*Status:* `trader is not running`' in result + + freqtradebot.update_state(State.RUNNING) + (error, result) = rpc.rpc_status_table() + assert error + assert '*Status:* `no active order`' in result + + freqtradebot.create_trade() + (error, result) = rpc.rpc_status_table() + assert 'just now' in result['Since'].all() + assert 'BTC_ETH' in result['Pair'].all() + assert '-0.59%' in result['Profit'].all() + + +def test_rpc_daily_profit(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker)\ + -> None: + """ + Test rpc_daily_profit() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] + rpc = RPC(freqtradebot) + # Create some test data - main.create_trade(0.001, 5) + freqtradebot.create_trade() trade = Trade.query.first() assert trade @@ -209,8 +139,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, limit_buy_order, limit_s # Try valid data update.message.text = '/daily 2' - (error, days) = rpc.rpc_daily_profit(7, stake_currency, - fiat_display_currency) + (error, days) = rpc.rpc_daily_profit(7, stake_currency, fiat_display_currency) assert not error assert len(days) == 7 for day in days: @@ -224,50 +153,56 @@ def test_rpc_daily_profit(default_conf, update, ticker, limit_buy_order, limit_s assert str(days[0][0]) == str(datetime.utcnow().date()) # Try invalid data - (error, days) = rpc.rpc_daily_profit(0, stake_currency, - fiat_display_currency) + (error, days) = rpc.rpc_daily_profit(0, stake_currency, fiat_display_currency) assert error assert days.find('must be an integer greater than 0') >= 0 def test_rpc_trade_statistics( - default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0})) + default_conf, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker) -> None: + """ + Test rpc_trade_statistics() method + """ + patch_get_signal(mocker, (True, False)) + mocker.patch.multiple( + 'freqtrade.fiat_convert.Market', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - main.init(default_conf, create_engine('sqlite://')) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] - (error, stats) = rpc.rpc_trade_statistics(stake_currency, - fiat_display_currency) + rpc = RPC(freqtradebot) + + (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) assert error assert stats.find('no closed trade') >= 0 # Create some test data - main.create_trade(0.001, 5) + freqtradebot.create_trade() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) + # Update the ticker with a market going up - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up + ) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() trade.is_open = False - (error, stats) = rpc.rpc_trade_statistics(stake_currency, - fiat_display_currency) + (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) assert not error assert prec_satoshi(stats['profit_closed_coin'], 6.217e-05) assert prec_satoshi(stats['profit_closed_percent'], 6.2) @@ -285,33 +220,41 @@ def test_rpc_trade_statistics( # Test that rpc_trade_statistics can handle trades that lacks # trade.open_rate (it is set to None) -def test_rpc_trade_statistics_closed( - default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0})) +def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, ticker_sell_up, limit_buy_order, + limit_sell_order): + """ + Test rpc_trade_statistics() method + """ + patch_get_signal(mocker, (True, False)) + mocker.patch.multiple( + 'freqtrade.fiat_convert.Market', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - main.init(default_conf, create_engine('sqlite://')) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] + rpc = RPC(freqtradebot) + # Create some test data - main.create_trade(0.001, 5) + freqtradebot.create_trade() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) # Update the ticker with a market going up - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up + ) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() trade.is_open = False @@ -319,8 +262,7 @@ def test_rpc_trade_statistics_closed( for trade in Trade.query.order_by(Trade.id).all(): trade.open_rate = None - (error, stats) = rpc.rpc_trade_statistics(stake_currency, - fiat_display_currency) + (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) assert not error assert prec_satoshi(stats['profit_closed_coin'], 0) assert prec_satoshi(stats['profit_closed_percent'], 0) @@ -336,57 +278,223 @@ def test_rpc_trade_statistics_closed( assert prec_satoshi(stats['best_rate'], 6.2) -def test_rpc_balance_handle(default_conf, update, mocker): - mock_balance = [{ - 'Currency': 'BTC', - 'Balance': 10.0, - 'Available': 12.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }, { - 'Currency': 'ETH', - 'Balance': 0.0, - 'Available': 0.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }] - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.main.exchange', - get_balances=MagicMock(return_value=mock_balance)) - mocker.patch.multiple('freqtrade.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0})) +def test_rpc_balance_handle(default_conf, mocker): + """ + Test rpc_balance() method + """ + mock_balance = [ + { + 'Currency': 'BTC', + 'Balance': 10.0, + 'Available': 12.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + }, + { + 'Currency': 'ETH', + 'Balance': 0.0, + 'Available': 0.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + } + ] + + patch_get_signal(mocker, (True, False)) + mocker.patch.multiple( + 'freqtrade.fiat_convert.Market', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + ) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_balances=MagicMock(return_value=mock_balance) + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) (error, res) = rpc.rpc_balance(default_conf['fiat_display_currency']) assert not error (trade, x, y, z) = res assert prec_satoshi(x, 10) assert prec_satoshi(z, 150000) - assert y == 'USD' + assert 'USD' in y assert len(trade) == 1 - assert trade[0]['currency'] == 'BTC' + assert 'BTC' in trade[0]['currency'] assert prec_satoshi(trade[0]['available'], 12) assert prec_satoshi(trade[0]['balance'], 10) assert prec_satoshi(trade[0]['pending'], 0) assert prec_satoshi(trade[0]['est_btc'], 10) -def test_performance_handle( - default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - main.init(default_conf, create_engine('sqlite://')) +def test_rpc_start(mocker, default_conf) -> None: + """ + Test rpc_start() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock() + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) + freqtradebot.update_state(State.STOPPED) + + (error, result) = rpc.rpc_start() + assert not error + assert '`Starting trader ...`' in result + assert freqtradebot.get_state() == State.RUNNING + + (error, result) = rpc.rpc_start() + assert error + assert '*Status:* `already running`' in result + assert freqtradebot.get_state() == State.RUNNING + + +def test_rpc_stop(mocker, default_conf) -> None: + """ + Test rpc_stop() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock() + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) + freqtradebot.update_state(State.RUNNING) + + (error, result) = rpc.rpc_stop() + assert not error + assert '`Stopping trader ...`' in result + assert freqtradebot.get_state() == State.STOPPED + + (error, result) = rpc.rpc_stop() + assert error + assert '*Status:* `already stopped`' in result + assert freqtradebot.get_state() == State.STOPPED + + +def test_rpc_forcesell(default_conf, ticker, mocker) -> None: + """ + Test rpc_forcesell() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + cancel_order=cancel_order_mock, + get_order=MagicMock( + return_value={ + 'closed': True, + 'type': 'LIMIT_BUY', + } + ) + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + (error, res) = rpc.rpc_forcesell(None) + assert error + assert res == '`trader is not running`' + + freqtradebot.update_state(State.RUNNING) + (error, res) = rpc.rpc_forcesell(None) + assert error + assert res == 'Invalid argument.' + + (error, res) = rpc.rpc_forcesell('all') + assert not error + assert res == '' + + freqtradebot.create_trade() + (error, res) = rpc.rpc_forcesell('all') + assert not error + assert res == '' + + (error, res) = rpc.rpc_forcesell('1') + assert not error + assert res == '' + + freqtradebot.update_state(State.STOPPED) + (error, res) = rpc.rpc_forcesell(None) + assert error + assert res == '`trader is not running`' + + (error, res) = rpc.rpc_forcesell('all') + assert error + assert res == '`trader is not running`' + + freqtradebot.update_state(State.RUNNING) + assert cancel_order_mock.call_count == 0 + # make an limit-buy open trade + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_order', + return_value={ + 'closed': None, + 'type': 'LIMIT_BUY' + } + ) + # check that the trade is called, which is done + # by ensuring exchange.cancel_order is called + (error, res) = rpc.rpc_forcesell('1') + assert not error + assert res == '' + assert cancel_order_mock.call_count == 1 + + freqtradebot.create_trade() + # make an limit-sell open trade + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_order', + return_value={ + 'closed': None, + 'type': 'LIMIT_SELL' + } + ) + (error, res) = rpc.rpc_forcesell('2') + assert not error + assert res == '' + # status quo, no exchange calls + assert cancel_order_mock.call_count == 1 + + +def test_performance_handle(default_conf, ticker, limit_buy_order, + limit_sell_order, mocker) -> None: + """ + Test rpc_performance() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_balances=MagicMock(return_value=ticker), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) # Create some test data - main.create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() trade = Trade.query.first() assert trade @@ -404,3 +512,33 @@ def test_performance_handle( assert res[0]['pair'] == 'BTC_ETH' assert res[0]['count'] == 1 assert prec_satoshi(res[0]['profit'], 6.2) + + +def test_rpc_count(mocker, default_conf, ticker) -> None: + """ + Test rpc_count() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.rpc.rpc_manager.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_balances=MagicMock(return_value=ticker), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + rpc = RPC(freqtradebot) + + (error, trades) = rpc.rpc_count() + nb_trades = len(trades) + assert not error + assert nb_trades == 0 + + # Create some test data + freqtradebot.create_trade() + (error, trades) = rpc.rpc_count() + nb_trades = len(trades) + assert not error + assert nb_trades == 1 diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py new file mode 100644 index 000000000..1d56dea3a --- /dev/null +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -0,0 +1,139 @@ +""" +Unit test file for rpc/rpc_manager.py +""" + +import logging +from copy import deepcopy +from unittest.mock import MagicMock + +from freqtrade.rpc.rpc_manager import RPCManager +from freqtrade.rpc.telegram import Telegram +from freqtrade.tests.conftest import log_has, get_patched_freqtradebot + + +def test_rpc_manager_object() -> None: + """ + Test the Arguments object has the mandatory methods + :return: None + """ + assert hasattr(RPCManager, '_init') + assert hasattr(RPCManager, 'send_msg') + assert hasattr(RPCManager, 'cleanup') + + +def test__init__(mocker, default_conf) -> None: + """ + Test __init__() method + """ + init_mock = mocker.patch('freqtrade.rpc.rpc_manager.RPCManager._init', MagicMock()) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + + rpc_manager = RPCManager(freqtradebot) + assert rpc_manager.freqtrade == freqtradebot + assert rpc_manager.registered_modules == [] + assert rpc_manager.telegram is None + assert init_mock.call_count == 1 + + +def test_init_telegram_disabled(mocker, default_conf, caplog) -> None: + """ + Test _init() method with Telegram disabled + """ + caplog.set_level(logging.DEBUG) + + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + + freqtradebot = get_patched_freqtradebot(mocker, conf) + rpc_manager = RPCManager(freqtradebot) + + assert not log_has('Enabling rpc.telegram ...', caplog.record_tuples) + assert rpc_manager.registered_modules == [] + assert rpc_manager.telegram is None + + +def test_init_telegram_enabled(mocker, default_conf, caplog) -> None: + """ + Test _init() method with Telegram enabled + """ + caplog.set_level(logging.DEBUG) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + rpc_manager = RPCManager(freqtradebot) + + assert log_has('Enabling rpc.telegram ...', caplog.record_tuples) + len_modules = len(rpc_manager.registered_modules) + assert len_modules == 1 + assert 'telegram' in rpc_manager.registered_modules + assert isinstance(rpc_manager.telegram, Telegram) + + +def test_cleanup_telegram_disabled(mocker, default_conf, caplog) -> None: + """ + Test cleanup() method with Telegram disabled + """ + caplog.set_level(logging.DEBUG) + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.cleanup', MagicMock()) + + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + + freqtradebot = get_patched_freqtradebot(mocker, conf) + rpc_manager = RPCManager(freqtradebot) + rpc_manager.cleanup() + + assert not log_has('Cleaning up rpc.telegram ...', caplog.record_tuples) + assert telegram_mock.call_count == 0 + + +def test_cleanup_telegram_enabled(mocker, default_conf, caplog) -> None: + """ + Test cleanup() method with Telegram enabled + """ + caplog.set_level(logging.DEBUG) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.cleanup', MagicMock()) + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + rpc_manager = RPCManager(freqtradebot) + + # Check we have Telegram as a registered modules + assert 'telegram' in rpc_manager.registered_modules + + rpc_manager.cleanup() + assert log_has('Cleaning up rpc.telegram ...', caplog.record_tuples) + assert 'telegram' not in rpc_manager.registered_modules + assert telegram_mock.call_count == 1 + + +def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: + """ + Test send_msg() method with Telegram disabled + """ + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + + freqtradebot = get_patched_freqtradebot(mocker, conf) + rpc_manager = RPCManager(freqtradebot) + rpc_manager.send_msg('test') + + assert log_has('test', caplog.record_tuples) + assert telegram_mock.call_count == 0 + + +def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: + """ + Test send_msg() method with Telegram disabled + """ + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + rpc_manager = RPCManager(freqtradebot) + rpc_manager.send_msg('test') + + assert log_has('test', caplog.record_tuples) + assert telegram_mock.call_count == 1 diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 82d6dd865..4796b077e 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1,6 +1,12 @@ -# pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 -# pragma pylint: disable=unused-argument +# pragma pylint: disable=protected-access, unused-argument, invalid-name +# pragma pylint: disable=too-many-lines, too-many-arguments + +""" +Unit test file for rpc/telegram.py +""" + import re +from copy import deepcopy from datetime import datetime from random import randint from unittest.mock import MagicMock @@ -10,139 +16,353 @@ from telegram import Update, Message, Chat from telegram.error import NetworkError from freqtrade import __version__ -from freqtrade.main import init, create_trade -from freqtrade.misc import update_state, State, get_state +from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade -from freqtrade.rpc import telegram -from freqtrade.rpc.telegram import authorized_only, is_enabled, send_msg, _status, _status_table, \ - _profit, _forcesell, _performance, _daily, _count, _start, _stop, _balance, _version, _help - -import freqtrade.rpc.telegram as tg +from freqtrade.rpc.telegram import Telegram +from freqtrade.rpc.telegram import authorized_only +from freqtrade.state import State +from freqtrade.tests.conftest import get_patched_freqtradebot, log_has +from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap -def test_is_enabled(default_conf, mocker): - mocker.patch.dict('freqtrade.rpc.telegram._CONF', default_conf) - default_conf['telegram']['enabled'] = False - assert is_enabled() is False +class DummyCls(Telegram): + """ + Dummy class for testing the Telegram @authorized_only decorator + """ + def __init__(self, freqtrade) -> None: + super().__init__(freqtrade) + self.state = {'called': False} + + @authorized_only + def dummy_handler(self, *args, **kwargs) -> None: + """ + Fake method that only change the state of the object + """ + self.state['called'] = True + + @authorized_only + def dummy_exception(self, *args, **kwargs) -> None: + """ + Fake method that throw an exception + """ + raise Exception('test') -def test_init_disabled(default_conf, mocker): - mocker.patch.dict('freqtrade.rpc.telegram._CONF', default_conf) - default_conf['telegram']['enabled'] = False - telegram.init(default_conf) +def test__init__(default_conf, mocker) -> None: + """ + Test __init__() method + """ + mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + + telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) + assert telegram._updater is None + assert telegram._config == default_conf -def test_authorized_only(default_conf, mocker): - mocker.patch.dict('freqtrade.rpc.telegram._CONF', default_conf) +def test_init(default_conf, mocker, caplog) -> None: + """ + Test _init() method + """ + start_polling = MagicMock() + mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock(return_value=start_polling)) + + Telegram(get_patched_freqtradebot(mocker, default_conf)) + assert start_polling.call_count == 0 + + # number of handles registered + assert start_polling.dispatcher.add_handler.call_count == 11 + assert start_polling.start_polling.call_count == 1 + + message_str = "rpc.telegram is listening for following commands: [['status'], ['profit'], " \ + "['balance'], ['start'], ['stop'], ['forcesell'], ['performance'], ['daily'], " \ + "['count'], ['help'], ['version']]" + + assert log_has(message_str, caplog.record_tuples) + + +def test_init_disabled(default_conf, mocker, caplog) -> None: + """ + Test _init() method when Telegram is disabled + """ + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + Telegram(get_patched_freqtradebot(mocker, conf)) + + message_str = "rpc.telegram is listening for following commands: [['status'], ['profit'], " \ + "['balance'], ['start'], ['stop'], ['forcesell'], ['performance'], ['daily'], " \ + "['count'], ['help'], ['version']]" + + assert not log_has(message_str, caplog.record_tuples) + + +def test_cleanup(default_conf, mocker) -> None: + """ + Test cleanup() method + """ + updater_mock = MagicMock() + updater_mock.stop = MagicMock() + mocker.patch('freqtrade.rpc.telegram.Updater', updater_mock) + + # not enabled + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + telegram = Telegram(get_patched_freqtradebot(mocker, conf)) + telegram.cleanup() + assert telegram._updater is None + assert updater_mock.call_count == 0 + assert not hasattr(telegram._updater, 'stop') + assert updater_mock.stop.call_count == 0 + + # enabled + conf['telegram']['enabled'] = True + telegram = Telegram(get_patched_freqtradebot(mocker, conf)) + telegram.cleanup() + assert telegram._updater.stop.call_count == 1 + + +def test_is_enabled(default_conf, mocker) -> None: + """ + Test is_enabled() method + """ + mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) + + telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) + assert telegram.is_enabled() + + +def test_is_not_enabled(default_conf, mocker) -> None: + """ + Test is_enabled() method + """ + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + telegram = Telegram(get_patched_freqtradebot(mocker, conf)) + + assert not telegram.is_enabled() + + +def test_authorized_only(default_conf, mocker, caplog) -> None: + """ + Test authorized_only() method when we are authorized + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) chat = Chat(0, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) - state = {'called': False} - @authorized_only - def dummy_handler(*args, **kwargs) -> None: - state['called'] = True - - dummy_handler(MagicMock(), update) - assert state['called'] is True + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy.dummy_handler(bot=MagicMock(), update=update) + assert dummy.state['called'] is True + assert log_has( + 'Executing handler: dummy_handler for chat_id: 0', + caplog.record_tuples + ) + assert not log_has( + 'Rejected unauthorized message from: 0', + caplog.record_tuples + ) + assert not log_has( + 'Exception occurred within Telegram module', + caplog.record_tuples + ) -def test_authorized_only_unauthorized(default_conf, mocker): - mocker.patch.dict('freqtrade.rpc.telegram._CONF', default_conf) +def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: + """ + Test authorized_only() method when we are unauthorized + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) chat = Chat(0xdeadbeef, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) - state = {'called': False} - @authorized_only - def dummy_handler(*args, **kwargs) -> None: - state['called'] = True - - dummy_handler(MagicMock(), update) - assert state['called'] is False + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy.dummy_handler(bot=MagicMock(), update=update) + assert dummy.state['called'] is False + assert not log_has( + 'Executing handler: dummy_handler for chat_id: 3735928559', + caplog.record_tuples + ) + assert log_has( + 'Rejected unauthorized message from: 3735928559', + caplog.record_tuples + ) + assert not log_has( + 'Exception occurred within Telegram module', + caplog.record_tuples + ) -def test_authorized_only_exception(default_conf, mocker): - mocker.patch.dict('freqtrade.rpc.telegram._CONF', default_conf) +def test_authorized_only_exception(default_conf, mocker, caplog) -> None: + """ + Test authorized_only() method when an exception is thrown + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), Chat(0, 0)) - @authorized_only - def dummy_handler(*args, **kwargs) -> None: - raise Exception('test') - - dummy_handler(MagicMock(), update) + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy.dummy_exception(bot=MagicMock(), update=update) + assert dummy.state['called'] is False + assert not log_has( + 'Executing handler: dummy_handler for chat_id: 0', + caplog.record_tuples + ) + assert not log_has( + 'Rejected unauthorized message from: 0', + caplog.record_tuples + ) + assert log_has( + 'Exception occurred within Telegram module', + caplog.record_tuples + ) -def test_status_handle(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) +def test_status(default_conf, update, mocker, ticker) -> None: + """ + Test _status() method + """ + update.message.chat.id = 123 + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + conf['telegram']['chat_id'] = 123 + + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - init(default_conf, create_engine('sqlite://')) + status_table = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + rpc_trade_status=MagicMock(return_value=(False, [1, 2, 3])), + _status_table=status_table, + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - update_state(State.STOPPED) - _status(bot=MagicMock(), update=update) + freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Create some test data + for _ in range(3): + freqtradebot.create_trade() + + telegram._status(bot=MagicMock(), update=update) + assert msg_mock.call_count == 3 + + update.message.text = MagicMock() + update.message.text.replace = MagicMock(return_value='table 2 3') + telegram._status(bot=MagicMock(), update=update) + assert status_table.call_count == 1 + + +def test_status_handle(default_conf, update, ticker, mocker) -> None: + """ + Test _status() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + msg_mock = MagicMock() + status_table = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _status_table=status_table, + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'trader is not running' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() - update_state(State.RUNNING) - _status(bot=MagicMock(), update=update) + freqtradebot.update_state(State.RUNNING) + telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'no active trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() # Trigger status while we have a fulfilled order for the open trade - _status(bot=MagicMock(), update=update) + telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert '[BTC_ETH]' in msg_mock.call_args_list[0][0][0] -def test_status_table_handle(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) +def test_status_table_handle(default_conf, update, ticker, mocker) -> None: + """ + Test _status_table() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) mocker.patch.multiple( - 'freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_order_id')) - init(default_conf, create_engine('sqlite://')) - update_state(State.STOPPED) - _status_table(bot=MagicMock(), update=update) + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_order_id') + ) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + conf = deepcopy(default_conf) + conf['stake_amount'] = 15.0 + freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + telegram._status_table(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'trader is not running' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() - update_state(State.RUNNING) - _status_table(bot=MagicMock(), update=update) + freqtradebot.update_state(State.RUNNING) + telegram._status_table(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'no active order' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() # Create some test data - create_trade(15.0, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() - _status_table(bot=MagicMock(), update=update) + telegram._status_table(bot=MagicMock(), update=update) text = re.sub('', '', msg_mock.call_args_list[-1][0][0]) line = text.split("\n") @@ -153,254 +373,35 @@ def test_status_table_handle(default_conf, update, ticker, mocker): assert msg_mock.call_count == 1 -def test_profit_handle( - default_conf, update, ticker, ticker_sell_up, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) +def test_daily_handle(default_conf, update, ticker, limit_buy_order, + limit_sell_order, mocker) -> None: + """ + Test _daily() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch( + 'freqtrade.fiat_convert.CryptoToFiatConverter._find_price', + return_value=15000.0 + ) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0})) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - _profit(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'no closed trade' in msg_mock.call_args_list[0][0][0] - msg_mock.reset_mock() + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - trade = Trade.query.first() - - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) - - _profit(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'no closed trade' in msg_mock.call_args_list[-1][0][0] - msg_mock.reset_mock() - - # Update the ticker with a market going up - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) - trade.update(limit_sell_order) - - trade.close_date = datetime.utcnow() - trade.is_open = False - - _profit(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert '*ROI:* Close trades' in msg_mock.call_args_list[-1][0][0] - assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] - assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] - assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] - assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] - assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] - - assert '*Best Performing:* `BTC_ETH: 6.20%`' in msg_mock.call_args_list[-1][0][0] - - -def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - # Increase the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) - - update.message.text = '/forcesell 1' - _forcesell(bot=MagicMock(), update=update) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] - assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] - - -def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - # Decrease the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_down) - - trade = Trade.query.first() - assert trade - - update.message.text = '/forcesell 1' - _forcesell(bot=MagicMock(), update=update) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] - assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] - - -def test_forcesell_all_handle(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - for _ in range(4): - create_trade(0.001, int(default_conf['ticker_interval'])) - rpc_mock.reset_mock() - - update.message.text = '/forcesell all' - _forcesell(bot=MagicMock(), update=update) - - assert rpc_mock.call_count == 4 - for args in rpc_mock.call_args_list: - assert '0.00001098' in args[0][0] - assert 'loss: -0.59%, -0.00000591 BTC' in args[0][0] - assert '-0.089 USD' in args[0][0] - - -def test_forcesell_handle_invalid(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, True)) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock()) - init(default_conf, create_engine('sqlite://')) - - # Trader is not running - update_state(State.STOPPED) - update.message.text = '/forcesell 1' - _forcesell(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'not running' in msg_mock.call_args_list[0][0][0] - - # No argument - msg_mock.reset_mock() - update_state(State.RUNNING) - update.message.text = '/forcesell' - _forcesell(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'Invalid argument' in msg_mock.call_args_list[0][0][0] - - # Invalid argument - msg_mock.reset_mock() - update_state(State.RUNNING) - update.message.text = '/forcesell 123456' - _forcesell(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'Invalid argument.' in msg_mock.call_args_list[0][0][0] - - -def test_performance_handle( - default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - trade = Trade.query.first() - assert trade - - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) - - # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) - - trade.close_date = datetime.utcnow() - trade.is_open = False - _performance(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'Performance' in msg_mock.call_args_list[0][0][0] - assert 'BTC_ETH\t6.20% (1)' in msg_mock.call_args_list[0][0][0] - - -def test_daily_handle(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0})) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() trade = Trade.query.first() assert trade @@ -415,7 +416,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, limit_sell_ # Try valid data update.message.text = '/daily 2' - _daily(bot=MagicMock(), update=update) + telegram._daily(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'Daily' in msg_mock.call_args_list[0][0][0] assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] @@ -427,8 +428,8 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, limit_sell_ # Reset msg_mock msg_mock.reset_mock() # Add two other trades - create_trade(0.001, int(default_conf['ticker_interval'])) - create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() + freqtradebot.create_trade() trades = Trade.query.all() for trade in trades: @@ -439,225 +440,179 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, limit_sell_ update.message.text = '/daily 1' - _daily(bot=MagicMock(), update=update) + telegram._daily(bot=MagicMock(), update=update) assert str(' 0.00018651 BTC') in msg_mock.call_args_list[0][0][0] assert str(' 2.798 USD') in msg_mock.call_args_list[0][0][0] assert str(' 3 trades') in msg_mock.call_args_list[0][0][0] -def test_daily_wrong_input(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) +def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: + """ + Test _daily() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) msg_mock = MagicMock() - mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch.multiple('freqtrade.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0})) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) # Try invalid data msg_mock.reset_mock() - update_state(State.RUNNING) + freqtradebot.update_state(State.RUNNING) update.message.text = '/daily -2' - _daily(bot=MagicMock(), update=update) + telegram._daily(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'must be an integer greater than 0' in msg_mock.call_args_list[0][0][0] # Try invalid data msg_mock.reset_mock() - update_state(State.RUNNING) + freqtradebot.update_state(State.RUNNING) update.message.text = '/daily today' - _daily(bot=MagicMock(), update=update) + telegram._daily(bot=MagicMock(), update=update) assert str('Daily Profit over the last 7 days') in msg_mock.call_args_list[0][0][0] -def test_count_handle(default_conf, update, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) +def test_profit_handle(default_conf, update, ticker, ticker_sell_up, + limit_buy_order, limit_sell_order, mocker) -> None: + """ + Test _profit() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) msg_mock = MagicMock() mocker.patch.multiple( - 'freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_order_id')) - init(default_conf, create_engine('sqlite://')) - update_state(State.STOPPED) - _count(bot=MagicMock(), update=update) + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._profit(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert 'not running' in msg_mock.call_args_list[0][0][0] + assert 'no closed trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() - update_state(State.RUNNING) # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) + freqtradebot.create_trade() + trade = Trade.query.first() + + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + telegram._profit(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'no closed trade' in msg_mock.call_args_list[-1][0][0] msg_mock.reset_mock() - _count(bot=MagicMock(), update=update) - msg = '
  current    max\n---------  -----\n        1      {}
'.format( - default_conf['max_open_trades'] - ) - assert msg in msg_mock.call_args_list[0][0][0] + # Update the ticker with a market going up + mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', ticker_sell_up) + trade.update(limit_sell_order) + trade.close_date = datetime.utcnow() + trade.is_open = False -def test_performance_handle_invalid(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, True)) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock()) - init(default_conf, create_engine('sqlite://')) - - # Trader is not running - update_state(State.STOPPED) - _performance(bot=MagicMock(), update=update) + telegram._profit(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert 'not running' in msg_mock.call_args_list[0][0][0] + assert '*ROI:* Close trades' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] + assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.00006217 BTC (6.20%)`' in msg_mock.call_args_list[-1][0][0] + assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] + + assert '*Best Performing:* `BTC_ETH: 6.20%`' in msg_mock.call_args_list[-1][0][0] -def test_start_handle(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - _CONF=default_conf, - init=MagicMock()) - init(default_conf, create_engine('sqlite://')) - update_state(State.STOPPED) - assert get_state() == State.STOPPED - _start(bot=MagicMock(), update=update) - assert get_state() == State.RUNNING - assert msg_mock.call_count == 0 - - -def test_start_handle_already_running(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - _CONF=default_conf, - init=MagicMock()) - init(default_conf, create_engine('sqlite://')) - update_state(State.RUNNING) - assert get_state() == State.RUNNING - _start(bot=MagicMock(), update=update) - assert get_state() == State.RUNNING - assert msg_mock.call_count == 1 - assert 'already running' in msg_mock.call_args_list[0][0][0] - - -def test_stop_handle(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - _CONF=default_conf, - init=MagicMock()) - init(default_conf, create_engine('sqlite://')) - update_state(State.RUNNING) - assert get_state() == State.RUNNING - _stop(bot=MagicMock(), update=update) - assert get_state() == State.STOPPED - assert msg_mock.call_count == 1 - assert 'Stopping trader' in msg_mock.call_args_list[0][0][0] - - -def test_stop_handle_already_stopped(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - _CONF=default_conf, - init=MagicMock()) - init(default_conf, create_engine('sqlite://')) - update_state(State.STOPPED) - assert get_state() == State.STOPPED - _stop(bot=MagicMock(), update=update) - assert get_state() == State.STOPPED - assert msg_mock.call_count == 1 - assert 'already stopped' in msg_mock.call_args_list[0][0][0] - - -def test_telegram_balance_handle(default_conf, update, mocker): - mock_balance = [{ - 'Currency': 'BTC', - 'Balance': 10.0, - 'Available': 12.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }, { - 'Currency': 'ETH', - 'Balance': 0.0, - 'Available': 0.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }, { - 'Currency': 'USDT', - 'Balance': 10000.0, - 'Available': 0.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }, { - 'Currency': 'LTC', - 'Balance': 10.0, - 'Available': 10.0, - 'Pending': 0.0, - 'CryptoAddress': 'XXXX', - }] +def test_telegram_balance_handle(default_conf, update, mocker) -> None: + """ + Test _balance() method + """ + mock_balance = [ + { + 'Currency': 'BTC', + 'Balance': 10.0, + 'Available': 12.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + }, + { + 'Currency': 'ETH', + 'Balance': 0.0, + 'Available': 0.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + }, + { + 'Currency': 'USDT', + 'Balance': 10000.0, + 'Available': 0.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + }, + { + 'Currency': 'LTC', + 'Balance': 10.0, + 'Available': 10.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + } + ] def mock_ticker(symbol, refresh): + """ + Mock Bittrex.get_ticker() response + """ if symbol == 'USDT_BTC': return { 'bid': 10000.00, 'ask': 10000.00, 'last': 10000.00, } - else: - return { - 'bid': 0.1, - 'ask': 0.1, - 'last': 0.1, - } - mocker.patch.dict('freqtrade.main._CONF', default_conf) + return { + 'bid': 0.1, + 'ask': 0.1, + 'last': 0.1, + } + + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value=mock_balance) + mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', side_effect=mock_ticker) + msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - get_balances=MagicMock(return_value=mock_balance)) - mocker.patch.multiple('freqtrade.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0})) - mocker.patch('freqtrade.main.exchange.get_ticker', side_effect=mock_ticker) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) - _balance(bot=MagicMock(), update=update) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._balance(bot=MagicMock(), update=update) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 assert '*Currency*: BTC' in result @@ -668,135 +623,471 @@ def test_telegram_balance_handle(default_conf, update, mocker): assert '*BTC*: 12.00000000' in result -def test_zero_balance_handle(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) +def test_zero_balance_handle(default_conf, update, mocker) -> None: + """ + Test _balance() method when the Exchange platform returns nothing + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value=[]) + msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) - mocker.patch.multiple('freqtrade.main.exchange', - get_balances=MagicMock(return_value=[])) - _balance(bot=MagicMock(), update=update) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._balance(bot=MagicMock(), update=update) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 assert '`All balances are zero.`' in result -def test_help_handle(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) +def test_start_handle(default_conf, update, mocker) -> None: + """ + Test _start() method + """ + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - _help(bot=MagicMock(), update=update) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + assert freqtradebot.get_state() == State.STOPPED + telegram._start(bot=MagicMock(), update=update) + assert freqtradebot.get_state() == State.RUNNING + assert msg_mock.call_count == 0 + + +def test_start_handle_already_running(default_conf, update, mocker) -> None: + """ + Test _start() method + """ + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.RUNNING) + assert freqtradebot.get_state() == State.RUNNING + telegram._start(bot=MagicMock(), update=update) + assert freqtradebot.get_state() == State.RUNNING + assert msg_mock.call_count == 1 + assert 'already running' in msg_mock.call_args_list[0][0][0] + + +def test_stop_handle(default_conf, update, mocker) -> None: + """ + Test _stop() method + """ + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.RUNNING) + assert freqtradebot.get_state() == State.RUNNING + telegram._stop(bot=MagicMock(), update=update) + assert freqtradebot.get_state() == State.STOPPED + assert msg_mock.call_count == 1 + assert 'Stopping trader' in msg_mock.call_args_list[0][0][0] + + +def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: + """ + Test _stop() method + """ + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + assert freqtradebot.get_state() == State.STOPPED + telegram._stop(bot=MagicMock(), update=update) + assert freqtradebot.get_state() == State.STOPPED + assert msg_mock.call_count == 1 + assert 'already stopped' in msg_mock.call_args_list[0][0][0] + + +def test_forcesell_handle(default_conf, update, ticker, ticker_sell_up, mocker) -> None: + """ + Test _forcesell() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Create some test data + freqtradebot.create_trade() + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch('freqtrade.freqtradebot.exchange.get_ticker', ticker_sell_up) + + update.message.text = '/forcesell 1' + telegram._forcesell(bot=MagicMock(), update=update) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert 'Amount' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] + assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] + assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] + + +def test_forcesell_down_handle(default_conf, update, ticker, ticker_sell_down, mocker) -> None: + """ + Test _forcesell() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Create some test data + freqtradebot.create_trade() + + # Decrease the price and sell it + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_down + ) + + trade = Trade.query.first() + assert trade + + update.message.text = '/forcesell 1' + telegram._forcesell(bot=MagicMock(), update=update) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert 'Amount' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] + assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] + + +def test_forcesell_all_handle(default_conf, update, ticker, mocker) -> None: + """ + Test _forcesell() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Create some test data + for _ in range(4): + freqtradebot.create_trade() + rpc_mock.reset_mock() + + update.message.text = '/forcesell all' + telegram._forcesell(bot=MagicMock(), update=update) + + assert rpc_mock.call_count == 4 + for args in rpc_mock.call_args_list: + assert '0.00001098' in args[0][0] + assert 'loss: -0.59%, -0.00000591 BTC' in args[0][0] + assert '-0.089 USD' in args[0][0] + + +def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: + """ + Test _forcesell() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Trader is not running + freqtradebot.update_state(State.STOPPED) + update.message.text = '/forcesell 1' + telegram._forcesell(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'not running' in msg_mock.call_args_list[0][0][0] + + # No argument + msg_mock.reset_mock() + freqtradebot.update_state(State.RUNNING) + update.message.text = '/forcesell' + telegram._forcesell(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'Invalid argument' in msg_mock.call_args_list[0][0][0] + + # Invalid argument + msg_mock.reset_mock() + freqtradebot.update_state(State.RUNNING) + update.message.text = '/forcesell 123456' + telegram._forcesell(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'Invalid argument.' in msg_mock.call_args_list[0][0][0] + + +def test_performance_handle(default_conf, update, ticker, limit_buy_order, + limit_sell_order, mocker) -> None: + """ + Test _performance() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Create some test data + freqtradebot.create_trade() + trade = Trade.query.first() + assert trade + + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + # Simulate fulfilled LIMIT_SELL order for trade + trade.update(limit_sell_order) + + trade.close_date = datetime.utcnow() + trade.is_open = False + telegram._performance(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'Performance' in msg_mock.call_args_list[0][0][0] + assert 'BTC_ETH\t6.20% (1)' in msg_mock.call_args_list[0][0][0] + + +def test_performance_handle_invalid(default_conf, update, mocker) -> None: + """ + Test _performance() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + # Trader is not running + freqtradebot.update_state(State.STOPPED) + telegram._performance(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'not running' in msg_mock.call_args_list[0][0][0] + + +def test_count_handle(default_conf, update, ticker, mocker) -> None: + """ + Test _count() method + """ + patch_get_signal(mocker, (True, False)) + patch_coinmarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_order_id') + ) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + freqtradebot.update_state(State.STOPPED) + telegram._count(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert 'not running' in msg_mock.call_args_list[0][0][0] + msg_mock.reset_mock() + freqtradebot.update_state(State.RUNNING) + + # Create some test data + freqtradebot.create_trade() + msg_mock.reset_mock() + telegram._count(bot=MagicMock(), update=update) + + msg = '
  current    max\n---------  -----\n        1      {}
'.format( + default_conf['max_open_trades'] + ) + assert msg in msg_mock.call_args_list[0][0][0] + + +def test_help_handle(default_conf, update, mocker) -> None: + """ + Test _help() method + """ + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._help(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert '*/help:* `This help message`' in msg_mock.call_args_list[0][0][0] -def test_version_handle(default_conf, update, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) +def test_version_handle(default_conf, update, mocker) -> None: + """ + Test _version() method + """ + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) msg_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=msg_mock) + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + send_msg=msg_mock + ) + freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) - _version(bot=MagicMock(), update=update) + telegram._version(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert '*Version:* `{}`'.format(__version__) in msg_mock.call_args_list[0][0][0] -def test_send_msg(default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) +def test_send_msg(default_conf, mocker) -> None: + """ + Test send_msg() method + """ + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + conf = deepcopy(default_conf) bot = MagicMock() - send_msg('test', bot) + freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._config['telegram']['enabled'] = False + telegram.send_msg('test', bot) assert not bot.method_calls bot.reset_mock() - default_conf['telegram']['enabled'] = True - send_msg('test', bot) + telegram._config['telegram']['enabled'] = True + telegram.send_msg('test', bot) assert len(bot.method_calls) == 1 -def test_send_msg_network_error(default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - default_conf['telegram']['enabled'] = True +def test_send_msg_network_error(default_conf, mocker, caplog) -> None: + """ + Test send_msg() method + """ + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + conf = deepcopy(default_conf) bot = MagicMock() bot.send_message = MagicMock(side_effect=NetworkError('Oh snap')) - send_msg('test', bot) + freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + telegram = Telegram(freqtradebot) + + telegram._config['telegram']['enabled'] = True + telegram.send_msg('test', bot) # Bot should've tried to send it twice assert len(bot.method_calls) == 2 - - -def test_init(default_conf, mocker): - start_polling = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - # mock telegram.ext.Updater - Updater=MagicMock(return_value=start_polling)) - # not enabled - tg.init(default_conf) - assert start_polling.call_count == 0 - # number of handles registered - assert start_polling.dispatcher.add_handler.call_count == 11 - assert start_polling.start_polling.call_count == 1 - - # enabled - default_conf['telegram'] = {} - default_conf['telegram']['enabled'] = True - default_conf['telegram']['token'] = '' - tg.init(default_conf) - - -def test_cleanup(default_conf, mocker): - default_conf['telegram'] = {} - default_conf['telegram']['enabled'] = False - updater_mock = MagicMock() - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - _UPDATER=updater_mock) - # not enabled - tg.cleanup() - assert updater_mock.stop.call_count == 0 - - # enabled - default_conf['telegram']['enabled'] = True - tg.cleanup() - assert updater_mock.stop.call_count == 1 - - -def test_status(default_conf, update, mocker): - update.message.chat.id = 123 - default_conf['telegram'] = {} - default_conf['telegram']['chat_id'] = 123 - mocker.patch('telegram.update', MagicMock()) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock()) - msg_mock = MagicMock() - status_table = MagicMock() - mocker.patch.multiple('freqtrade.rpc', - send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - rpc_trade_status=MagicMock(return_value=(False, [1, 2, 3])), - _status_table=status_table, - send_msg=msg_mock) - _status(bot=MagicMock(), update=update) - assert msg_mock.call_count == 3 - update.message.text = MagicMock() - update.message.text.replace = MagicMock(return_value='table 2 3') - _status(bot=MagicMock(), update=update) - assert status_table.call_count == 1 + assert log_has( + 'Telegram NetworkError: Oh snap! Trying one more time.', + caplog.record_tuples + ) diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index f23c1fa48..2b91fbec5 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -1,14 +1,16 @@ import json + import pytest from pandas import DataFrame + +from freqtrade.analyze import Analyze from freqtrade.strategy.default_strategy import DefaultStrategy, class_name -from freqtrade.analyze import parse_ticker_dataframe @pytest.fixture def result(): with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) + return Analyze.parse_ticker_dataframe(json.load(data_file)) def test_default_strategy_class_name(): diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index b0ce88e98..7ce9ae0ef 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging + from freqtrade.strategy.strategy import Strategy @@ -21,7 +22,6 @@ def test_search_strategy(): def test_strategy_structure(): - assert hasattr(Strategy, 'init') assert hasattr(Strategy, 'populate_indicators') assert hasattr(Strategy, 'populate_buy_trend') assert hasattr(Strategy, 'populate_sell_trend') @@ -53,8 +53,7 @@ def test_load_not_found_strategy(caplog): def test_strategy(result): - strategy = Strategy() - strategy.init({'strategy': 'default_strategy'}) + strategy = Strategy({'strategy': 'default_strategy'}) assert hasattr(strategy.custom_strategy, 'minimal_roi') assert strategy.minimal_roi[0] == 0.04 @@ -82,8 +81,7 @@ def test_strategy_override_minimal_roi(caplog): "0": 0.5 } } - strategy = Strategy() - strategy.init(config) + strategy = Strategy(config) assert hasattr(strategy.custom_strategy, 'minimal_roi') assert strategy.minimal_roi[0] == 0.5 @@ -99,8 +97,7 @@ def test_strategy_override_stoploss(caplog): 'strategy': 'default_strategy', 'stoploss': -0.5 } - strategy = Strategy() - strategy.init(config) + strategy = Strategy(config) assert hasattr(strategy.custom_strategy, 'stoploss') assert strategy.stoploss == -0.5 @@ -117,8 +114,7 @@ def test_strategy_override_ticker_interval(caplog): 'strategy': 'default_strategy', 'ticker_interval': 60 } - strategy = Strategy() - strategy.init(config) + strategy = Strategy(config) assert hasattr(strategy.custom_strategy, 'ticker_interval') assert strategy.ticker_interval == 60 @@ -138,8 +134,7 @@ def test_strategy_fallback_default_strategy(): def test_strategy_singleton(): - strategy1 = Strategy() - strategy1.init({'strategy': 'default_strategy'}) + strategy1 = Strategy({'strategy': 'default_strategy'}) assert hasattr(strategy1.custom_strategy, 'minimal_roi') assert strategy1.minimal_roi[0] == 0.04 diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index b70596091..b5f52774d 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -1,6 +1,6 @@ -# pragma pylint: disable=missing-docstring,C0103 +# pragma pylint: disable=missing-docstring,C0103,protected-access -from freqtrade.main import refresh_whitelist, gen_pair_whitelist +import freqtrade.tests.conftest as tt # test tools # whitelist, blacklist, filtering, all of that will # eventually become some rules to run on a generic ACL engine @@ -8,21 +8,22 @@ from freqtrade.main import refresh_whitelist, gen_pair_whitelist def whitelist_conf(): - return { - 'stake_currency': 'BTC', - 'exchange': { - 'pair_whitelist': [ - 'BTC_ETH', - 'BTC_TKN', - 'BTC_TRST', - 'BTC_SWT', - 'BTC_BCC' - ], - 'pair_blacklist': [ - 'BTC_BLK' - ], - }, - } + config = tt.default_conf() + + config['stake_currency'] = 'BTC' + config['exchange']['pair_whitelist'] = [ + 'BTC_ETH', + 'BTC_TKN', + 'BTC_TRST', + 'BTC_SWT', + 'BTC_BCC' + ] + + config['exchange']['pair_blacklist'] = [ + 'BTC_BLK' + ] + + return config def get_market_summaries(): @@ -86,11 +87,13 @@ def get_health_empty(): def test_refresh_market_pair_not_in_whitelist(mocker): conf = whitelist_conf() - mocker.patch.dict('freqtrade.main._CONF', conf) - mocker.patch.multiple('freqtrade.main.exchange', - get_wallet_health=get_health) - refreshedwhitelist = refresh_whitelist( - conf['exchange']['pair_whitelist'] + ['BTC_XXX']) + + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + + mocker.patch('freqtrade.freqtradebot.exchange.get_wallet_health', get_health) + refreshedwhitelist = freqtradebot._refresh_whitelist( + conf['exchange']['pair_whitelist'] + ['BTC_XXX'] + ) # List ordered by BaseVolume whitelist = ['BTC_ETH', 'BTC_TKN'] # Ensure all except those in whitelist are removed @@ -99,10 +102,11 @@ def test_refresh_market_pair_not_in_whitelist(mocker): def test_refresh_whitelist(mocker): conf = whitelist_conf() - mocker.patch.dict('freqtrade.main._CONF', conf) - mocker.patch.multiple('freqtrade.main.exchange', - get_wallet_health=get_health) - refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist']) + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + + mocker.patch('freqtrade.freqtradebot.exchange.get_wallet_health', get_health) + refreshedwhitelist = freqtradebot._refresh_whitelist(conf['exchange']['pair_whitelist']) + # List ordered by BaseVolume whitelist = ['BTC_ETH', 'BTC_TKN'] # Ensure all except those in whitelist are removed @@ -111,26 +115,32 @@ def test_refresh_whitelist(mocker): def test_refresh_whitelist_dynamic(mocker): conf = whitelist_conf() - mocker.patch.dict('freqtrade.main._CONF', conf) - mocker.patch.multiple('freqtrade.main.exchange', - get_wallet_health=get_health) - mocker.patch.multiple('freqtrade.main.exchange', - get_market_summaries=get_market_summaries) + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + get_wallet_health=get_health, + get_market_summaries=get_market_summaries + ) + # argument: use the whitelist dynamically by exchange-volume whitelist = ['BTC_TKN', 'BTC_ETH'] - refreshedwhitelist = refresh_whitelist( - gen_pair_whitelist(conf['stake_currency'])) + + refreshedwhitelist = freqtradebot._refresh_whitelist( + freqtradebot._gen_pair_whitelist(conf['stake_currency']) + ) + assert whitelist == refreshedwhitelist def test_refresh_whitelist_dynamic_empty(mocker): conf = whitelist_conf() - mocker.patch.dict('freqtrade.main._CONF', conf) - mocker.patch.multiple('freqtrade.main.exchange', - get_wallet_health=get_health_empty) + freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + mocker.patch('freqtrade.freqtradebot.exchange.get_wallet_health', get_health_empty) + # argument: use the whitelist dynamically by exchange-volume whitelist = [] conf['exchange']['pair_whitelist'] = [] - refresh_whitelist(whitelist) + freqtradebot._refresh_whitelist(whitelist) pairslist = conf['exchange']['pair_whitelist'] + assert set(whitelist) == set(pairslist) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 7e8f74b7e..558ea7ee5 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -1,16 +1,51 @@ # pragma pylint: disable=missing-docstring, C0103 + +""" +Unit test file for analyse.py +""" + import datetime +import logging from unittest.mock import MagicMock import arrow -import logging from pandas import DataFrame +from freqtrade.analyze import Analyze, SignalType +from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import log_has -from freqtrade.analyze import (get_signal, parse_ticker_dataframe, - populate_buy_trend, populate_indicators, - populate_sell_trend) -from freqtrade.strategy.strategy import Strategy + +# Avoid to reinit the same object again and again +_ANALYZE = Analyze({'strategy': 'default_strategy'}) + + +def test_signaltype_object() -> None: + """ + Test the SignalType object has the mandatory Constants + :return: None + """ + assert hasattr(SignalType, 'BUY') + assert hasattr(SignalType, 'SELL') + + +def test_analyze_object() -> None: + """ + Test the Analyze object has the mandatory methods + :return: None + """ + assert hasattr(Analyze, 'parse_ticker_dataframe') + assert hasattr(Analyze, 'populate_indicators') + assert hasattr(Analyze, 'populate_buy_trend') + assert hasattr(Analyze, 'populate_sell_trend') + assert hasattr(Analyze, 'analyze_ticker') + assert hasattr(Analyze, 'get_signal') + assert hasattr(Analyze, 'should_sell') + assert hasattr(Analyze, 'min_roi_reached') + + +def test_dataframe_correct_length(result): + dataframe = Analyze.parse_ticker_dataframe(result) + assert len(result.index) == len(dataframe.index) def test_dataframe_correct_columns(result): @@ -18,78 +53,88 @@ def test_dataframe_correct_columns(result): ['close', 'high', 'low', 'open', 'date', 'volume'] -def test_dataframe_correct_length(result): - dataframe = parse_ticker_dataframe(result) - assert len(result.index) == len(dataframe.index) - - def test_populates_buy_trend(result): # Load the default strategy for the unit test, because this logic is done in main.py - Strategy().init({'strategy': 'default_strategy'}) - - dataframe = populate_buy_trend(populate_indicators(result)) + dataframe = _ANALYZE.populate_buy_trend(_ANALYZE.populate_indicators(result)) assert 'buy' in dataframe.columns def test_populates_sell_trend(result): # Load the default strategy for the unit test, because this logic is done in main.py - Strategy().init({'strategy': 'default_strategy'}) - - dataframe = populate_sell_trend(populate_indicators(result)) + dataframe = _ANALYZE.populate_sell_trend(_ANALYZE.populate_indicators(result)) assert 'sell' in dataframe.columns def test_returns_latest_buy_signal(mocker): mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock()) - mocker.patch( - 'freqtrade.analyze.analyze_ticker', - return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) - ) - assert get_signal('BTC-ETH', 5) == (True, False) - mocker.patch( - 'freqtrade.analyze.analyze_ticker', - return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) + ) ) - assert get_signal('BTC-ETH', 5) == (False, True) + assert _ANALYZE.get_signal('BTC-ETH', 5) == (True, False) + + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) + ) + ) + assert _ANALYZE.get_signal('BTC-ETH', 5) == (False, True) def test_returns_latest_sell_signal(mocker): mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock()) - mocker.patch( - 'freqtrade.analyze.analyze_ticker', - return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) + ) ) - assert get_signal('BTC-ETH', 5) == (False, True) - mocker.patch( - 'freqtrade.analyze.analyze_ticker', - return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) + assert _ANALYZE.get_signal('BTC-ETH', 5) == (False, True) + + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) + ) ) - assert get_signal('BTC-ETH', 5) == (True, False) + assert _ANALYZE.get_signal('BTC-ETH', 5) == (True, False) def test_get_signal_empty(default_conf, mocker, caplog): caplog.set_level(logging.INFO) mocker.patch('freqtrade.analyze.get_ticker_history', return_value=None) - assert (False, False) == get_signal('foo', int(default_conf['ticker_interval'])) + assert (False, False) == _ANALYZE.get_signal('foo', int(default_conf['ticker_interval'])) assert log_has('Empty ticker history for pair foo', caplog.record_tuples) def test_get_signal_exception_valueerror(default_conf, mocker, caplog): caplog.set_level(logging.INFO) mocker.patch('freqtrade.analyze.get_ticker_history', return_value=1) - mocker.patch('freqtrade.analyze.analyze_ticker', - side_effect=ValueError('xyz')) - assert (False, False) == get_signal('foo', int(default_conf['ticker_interval'])) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + side_effect=ValueError('xyz') + ) + ) + assert (False, False) == _ANALYZE.get_signal('foo', int(default_conf['ticker_interval'])) assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) def test_get_signal_empty_dataframe(default_conf, mocker, caplog): caplog.set_level(logging.INFO) mocker.patch('freqtrade.analyze.get_ticker_history', return_value=1) - mocker.patch('freqtrade.analyze.analyze_ticker', return_value=DataFrame([])) - assert (False, False) == get_signal('xyz', int(default_conf['ticker_interval'])) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame([]) + ) + ) + assert (False, False) == _ANALYZE.get_signal('xyz', int(default_conf['ticker_interval'])) assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) @@ -99,27 +144,51 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): # FIX: The get_signal function has hardcoded 10, which we must inturn hardcode oldtime = arrow.utcnow() - datetime.timedelta(minutes=11) ticks = DataFrame([{'buy': 1, 'date': oldtime}]) - mocker.patch('freqtrade.analyze.analyze_ticker', return_value=DataFrame(ticks)) - assert (False, False) == get_signal('xyz', int(default_conf['ticker_interval'])) - assert log_has('Outdated history for pair xyz. Last tick is 11 minutes old', - caplog.record_tuples) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + return_value=DataFrame(ticks) + ) + ) + assert (False, False) == _ANALYZE.get_signal('xyz', int(default_conf['ticker_interval'])) + assert log_has( + 'Outdated history for pair xyz. Last tick is 11 minutes old', + caplog.record_tuples + ) def test_get_signal_handles_exceptions(mocker): mocker.patch('freqtrade.analyze.get_ticker_history', return_value=MagicMock()) - mocker.patch('freqtrade.analyze.analyze_ticker', - side_effect=Exception('invalid ticker history ')) + mocker.patch.multiple( + 'freqtrade.analyze.Analyze', + analyze_ticker=MagicMock( + side_effect=Exception('invalid ticker history ') + ) + ) - assert get_signal('BTC-ETH', 5) == (False, False) + assert _ANALYZE.get_signal('BTC-ETH', 5) == (False, False) def test_parse_ticker_dataframe(ticker_history, ticker_history_without_bv): columns = ['close', 'high', 'low', 'open', 'date', 'volume'] # Test file with BV data - dataframe = parse_ticker_dataframe(ticker_history) + dataframe = Analyze.parse_ticker_dataframe(ticker_history) assert dataframe.columns.tolist() == columns # Test file without BV data - dataframe = parse_ticker_dataframe(ticker_history_without_bv) + dataframe = Analyze.parse_ticker_dataframe(ticker_history_without_bv) assert dataframe.columns.tolist() == columns + + +def test_tickerdata_to_dataframe(default_conf) -> None: + """ + Test Analyze.tickerdata_to_dataframe() method + """ + analyze = Analyze(default_conf) + + timerange = ((None, 'line'), None, -100) + tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) + tickerlist = {'BTC_UNITEST': tick} + data = analyze.tickerdata_to_dataframe(tickerlist) + assert len(data['BTC_UNITEST']) == 100 diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py new file mode 100644 index 000000000..3e0639304 --- /dev/null +++ b/freqtrade/tests/test_arguments.py @@ -0,0 +1,134 @@ +# pragma pylint: disable=missing-docstring, C0103 + +""" +Unit test file for arguments.py +""" + +import argparse +import logging + +import pytest + +from freqtrade.arguments import Arguments + + +def test_arguments_object() -> None: + """ + Test the Arguments object has the mandatory methods + :return: None + """ + assert hasattr(Arguments, 'get_parsed_arg') + assert hasattr(Arguments, 'parse_args') + assert hasattr(Arguments, 'parse_timerange') + assert hasattr(Arguments, 'scripts_options') + + +# Parse common command-line-arguments. Used for all tools +def test_parse_args_none() -> None: + arguments = Arguments([], '') + assert isinstance(arguments, Arguments) + assert isinstance(arguments.parser, argparse.ArgumentParser) + assert isinstance(arguments.parser, argparse.ArgumentParser) + + +def test_parse_args_defaults() -> None: + args = Arguments([], '').get_parsed_arg() + assert args.config == 'config.json' + assert args.dynamic_whitelist is None + assert args.loglevel == logging.INFO + + +def test_parse_args_config() -> None: + args = Arguments(['-c', '/dev/null'], '').get_parsed_arg() + assert args.config == '/dev/null' + + args = Arguments(['--config', '/dev/null'], '').get_parsed_arg() + assert args.config == '/dev/null' + + +def test_parse_args_verbose() -> None: + args = Arguments(['-v'], '').get_parsed_arg() + assert args.loglevel == logging.DEBUG + + args = Arguments(['--verbose'], '').get_parsed_arg() + assert args.loglevel == logging.DEBUG + + +def test_scripts_options() -> None: + arguments = Arguments(['-p', 'BTC_ETH'], '') + arguments.scripts_options() + args = arguments.get_parsed_arg() + assert args.pair == 'BTC_ETH' + + +def test_parse_args_version() -> None: + with pytest.raises(SystemExit, match=r'0'): + Arguments(['--version'], '').get_parsed_arg() + + +def test_parse_args_invalid() -> None: + with pytest.raises(SystemExit, match=r'2'): + Arguments(['-c'], '').get_parsed_arg() + + +def test_parse_args_dynamic_whitelist() -> None: + args = Arguments(['--dynamic-whitelist'], '').get_parsed_arg() + assert args.dynamic_whitelist == 20 + + +def test_parse_args_dynamic_whitelist_10() -> None: + args = Arguments(['--dynamic-whitelist', '10'], '').get_parsed_arg() + assert args.dynamic_whitelist == 10 + + +def test_parse_args_dynamic_whitelist_invalid_values() -> None: + with pytest.raises(SystemExit, match=r'2'): + Arguments(['--dynamic-whitelist', 'abc'], '').get_parsed_arg() + + +def test_parse_timerange_incorrect() -> None: + assert ((None, 'line'), None, -200) == Arguments.parse_timerange('-200') + assert (('line', None), 200, None) == Arguments.parse_timerange('200-') + with pytest.raises(Exception, match=r'Incorrect syntax.*'): + Arguments.parse_timerange('-') + + +def test_parse_args_backtesting_invalid() -> None: + with pytest.raises(SystemExit, match=r'2'): + Arguments(['backtesting --ticker-interval'], '').get_parsed_arg() + + with pytest.raises(SystemExit, match=r'2'): + Arguments(['backtesting --ticker-interval', 'abc'], '').get_parsed_arg() + + +def test_parse_args_backtesting_custom() -> None: + args = [ + '-c', 'test_conf.json', + 'backtesting', + '--live', + '--ticker-interval', '1', + '--refresh-pairs-cached'] + call_args = Arguments(args, '').get_parsed_arg() + assert call_args.config == 'test_conf.json' + assert call_args.live is True + assert call_args.loglevel == logging.INFO + assert call_args.subparser == 'backtesting' + assert call_args.func is not None + assert call_args.ticker_interval == 1 + assert call_args.refresh_pairs is True + + +def test_parse_args_hyperopt_custom() -> None: + args = [ + '-c', 'test_conf.json', + 'hyperopt', + '--epochs', '20', + '--spaces', 'buy' + ] + call_args = Arguments(args, '').get_parsed_arg() + assert call_args.config == 'test_conf.json' + assert call_args.epochs == 20 + assert call_args.loglevel == logging.INFO + assert call_args.subparser == 'hyperopt' + assert call_args.spaces == ['buy'] + assert call_args.func is not None diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py new file mode 100644 index 000000000..002eac722 --- /dev/null +++ b/freqtrade/tests/test_configuration.py @@ -0,0 +1,316 @@ +# pragma pylint: disable=protected-access, invalid-name + +""" +Unit test file for configuration.py +""" +import json +from copy import deepcopy +from unittest.mock import MagicMock + +import pytest +from jsonschema import ValidationError + +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.tests.conftest import log_has + + +def test_configuration_object() -> None: + """ + Test the Constants object has the mandatory Constants + """ + assert hasattr(Configuration, 'load_config') + assert hasattr(Configuration, '_load_config_file') + assert hasattr(Configuration, '_validate_config') + assert hasattr(Configuration, '_load_common_config') + assert hasattr(Configuration, '_load_backtesting_config') + assert hasattr(Configuration, '_load_hyperopt_config') + assert hasattr(Configuration, 'get_config') + + +def test_load_config_invalid_pair(default_conf, mocker) -> None: + """ + Test the configuration validator with an invalid PAIR format + """ + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'].append('BTC-ETH') + + with pytest.raises(ValidationError, match=r'.*does not match.*'): + configuration = Configuration([]) + configuration._validate_config(conf) + + +def test_load_config_missing_attributes(default_conf, mocker) -> None: + """ + Test the configuration validator with a missing attribute + """ + conf = deepcopy(default_conf) + conf.pop('exchange') + + with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): + configuration = Configuration([]) + configuration._validate_config(conf) + + +def test_load_config_file(default_conf, mocker, caplog) -> None: + """ + Test Configuration._load_config_file() method + """ + file_mock = mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + configuration = Configuration([]) + validated_conf = configuration._load_config_file('somefile') + assert file_mock.call_count == 1 + assert validated_conf.items() >= default_conf.items() + assert 'internals' in validated_conf + assert log_has('Validating configuration ...', caplog.record_tuples) + + +def test_load_config_file_exception(mocker, caplog) -> None: + """ + Test Configuration._load_config_file() method + """ + mocker.patch( + 'freqtrade.configuration.open', + MagicMock(side_effect=FileNotFoundError('File not found')) + ) + configuration = Configuration([]) + + with pytest.raises(SystemExit): + configuration._load_config_file('somefile') + assert log_has( + 'Config file "somefile" not found. Please create your config file', + caplog.record_tuples + ) + + +def test_load_config(default_conf, mocker) -> None: + """ + Test Configuration.load_config() without any cli params + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = Arguments([], '').get_parsed_arg() + configuration = Configuration(args) + validated_conf = configuration.load_config() + + assert 'strategy' in validated_conf + assert validated_conf['strategy'] == 'default_strategy' + assert 'dynamic_whitelist' not in validated_conf + assert 'dry_run_db' not in validated_conf + + +def test_load_config_with_params(default_conf, mocker) -> None: + """ + Test Configuration.load_config() with cli params used + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--dynamic-whitelist', '10', + '--strategy', 'test_strategy', + '--dry-run-db' + ] + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + validated_conf = configuration.load_config() + + assert 'dynamic_whitelist' in validated_conf + assert validated_conf['dynamic_whitelist'] == 10 + assert 'strategy' in validated_conf + assert validated_conf['strategy'] == 'test_strategy' + assert 'dry_run_db' in validated_conf + assert validated_conf['dry_run_db'] is True + + +def test_show_info(default_conf, mocker, caplog) -> None: + """ + Test Configuration.show_info() + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--dynamic-whitelist', '10', + '--strategy', 'test_strategy', + '--dry-run-db' + ] + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + configuration.get_config() + + assert log_has( + 'Parameter --dynamic-whitelist detected. ' + 'Using dynamically generated whitelist. ' + '(not applicable with Backtesting and Hyperopt)', + caplog.record_tuples + ) + + assert log_has( + 'Parameter --dry-run-db detected ...', + caplog.record_tuples + ) + + assert log_has( + 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite"', + caplog.record_tuples + ) + + # Test the Dry run condition + configuration.config.update({'dry_run': False}) + configuration._load_common_config(configuration.config) + assert log_has( + 'Dry run is disabled. (--dry_run_db ignored)', + caplog.record_tuples + ) + + +def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + 'backtesting' + ] + + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert log_has( + 'Parameter --datadir detected: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert not log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + + assert 'live' not in config + assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples) + + assert 'realistic_simulation' not in config + assert not log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + + assert 'refresh_pairs' not in config + assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) + + assert 'timerange' not in config + assert 'export' not in config + + +def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'default_strategy', + '--datadir', '/foo/bar', + 'backtesting', + '--ticker-interval', '1', + '--live', + '--realistic-simulation', + '--refresh-pairs-cached', + '--timerange', ':100', + '--export', '/bar/foo' + ] + + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert log_has( + 'Parameter --datadir detected: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + assert log_has( + 'Using ticker_interval: 1 ...', + caplog.record_tuples + ) + + assert 'live' in config + assert log_has('Parameter -l/--live detected ...', caplog.record_tuples) + + assert 'realistic_simulation'in config + assert log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert log_has('Using max_open_trades: 1 ...', caplog.record_tuples) + + assert 'refresh_pairs'in config + assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) + assert 'timerange' in config + assert log_has( + 'Parameter --timerange detected: {} ...'.format(config['timerange']), + caplog.record_tuples + ) + + assert 'export' in config + assert log_has( + 'Parameter --export detected: {} ...'.format(config['export']), + caplog.record_tuples + ) + + +def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + 'hyperopt', + '--epochs', '10', + '--use-mongodb', + '--spaces', 'all', + ] + + args = Arguments(args, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert 'epochs' in config + assert int(config['epochs']) == 10 + assert log_has('Parameter --epochs detected ...', caplog.record_tuples) + assert log_has('Will run Hyperopt with for 10 epochs ...', caplog.record_tuples) + + assert 'mongodb' in config + assert config['mongodb'] is True + assert log_has('Parameter --use-mongodb detected ...', caplog.record_tuples) + + assert 'spaces' in config + assert config['spaces'] == ['all'] + assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) diff --git a/freqtrade/tests/test_constants.py b/freqtrade/tests/test_constants.py new file mode 100644 index 000000000..6d544502f --- /dev/null +++ b/freqtrade/tests/test_constants.py @@ -0,0 +1,26 @@ +""" +Unit test file for constants.py +""" + +from freqtrade.constants import Constants + + +def test_constant_object() -> None: + """ + Test the Constants object has the mandatory Constants + """ + assert hasattr(Constants, 'CONF_SCHEMA') + assert hasattr(Constants, 'DYNAMIC_WHITELIST') + assert hasattr(Constants, 'PROCESS_THROTTLE_SECS') + assert hasattr(Constants, 'TICKER_INTERVAL') + assert hasattr(Constants, 'HYPEROPT_EPOCH') + assert hasattr(Constants, 'RETRY_TIMEOUT') + assert hasattr(Constants, 'DEFAULT_STRATEGY') + + +def test_conf_schema() -> None: + """ + Test the CONF_SCHEMA is from the right type + """ + constant = Constants() + assert isinstance(constant.CONF_SCHEMA, dict) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index 44b2a7e8c..1f69a7d32 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -1,30 +1,33 @@ # pragma pylint: disable=missing-docstring, C0103 import pandas -import freqtrade.optimize -from freqtrade import analyze + +from freqtrade.analyze import Analyze +from freqtrade.optimize import load_data from freqtrade.strategy.strategy import Strategy _pairs = ['BTC_ETH'] def load_dataframe_pair(pairs): - ld = freqtrade.optimize.load_data(None, ticker_interval=5, pairs=pairs) + ld = load_data(None, ticker_interval=5, pairs=pairs) assert isinstance(ld, dict) assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] + + analyze = Analyze({'strategy': 'default_strategy'}) dataframe = analyze.analyze_ticker(dataframe) return dataframe def test_dataframe_load(): - Strategy().init({'strategy': 'default_strategy'}) + Strategy({'strategy': 'default_strategy'}) dataframe = load_dataframe_pair(_pairs) assert isinstance(dataframe, pandas.core.frame.DataFrame) def test_dataframe_columns_exists(): - Strategy().init({'strategy': 'default_strategy'}) + Strategy({'strategy': 'default_strategy'}) dataframe = load_dataframe_pair(_pairs) assert 'high' in dataframe.columns assert 'low' in dataframe.columns diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py new file mode 100644 index 000000000..d58b428da --- /dev/null +++ b/freqtrade/tests/test_freqtradebot.py @@ -0,0 +1,1278 @@ +# pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments + +""" +Unit test file for freqtradebot.py +""" + +import logging +import re +import time +from copy import deepcopy +from typing import Dict, Optional +from unittest.mock import MagicMock + +import arrow +import pytest +import requests +from sqlalchemy import create_engine + +from freqtrade import DependencyException, OperationalException +from freqtrade.exchange import Exchanges +from freqtrade.freqtradebot import FreqtradeBot +from freqtrade.persistence import Trade +from freqtrade.state import State +from freqtrade.tests.conftest import log_has + + +# Functions for recurrent object patching +def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: + """ + This function patch _init_modules() to not call dependencies + :param mocker: a Mocker object to apply patches + :param config: Config to pass to the bot + :return: None + """ + mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) + mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) + mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) + mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) + patch_coinmarketcap(mocker) + + return FreqtradeBot(config, create_engine('sqlite://')) + + +def patch_get_signal(mocker, value=(True, False)) -> None: + """ + + :param mocker: mocker to patch Analyze class + :param value: which value Analyze.get_signal() must return + :return: None + """ + mocker.patch( + 'freqtrade.freqtradebot.Analyze.get_signal', + side_effect=lambda s, t: value + ) + + +def patch_RPCManager(mocker) -> MagicMock: + """ + This function mock RPC manager to avoid repeating this code in almost every tests + :param mocker: mocker to patch RPCManager class + :return: RPCManager.send_msg MagicMock to track if this method is called + """ + mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) + rpc_mock = mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) + return rpc_mock + + +def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None: + """ + Mocker to coinmarketcap to speed up tests + :param mocker: mocker to patch coinmarketcap class + :return: None + """ + mock = MagicMock() + + if value: + mock.ticker = {'price_usd': 12345.0} + + mocker.patch('freqtrade.fiat_convert.Market', mock) + + +# Unit tests +def test_freqtradebot_object() -> None: + """ + Test the FreqtradeBot object has the mandatory public methods + """ + assert hasattr(FreqtradeBot, 'worker') + assert hasattr(FreqtradeBot, 'get_state') + assert hasattr(FreqtradeBot, 'update_state') + assert hasattr(FreqtradeBot, 'clean') + assert hasattr(FreqtradeBot, 'create_trade') + assert hasattr(FreqtradeBot, 'get_target_bid') + assert hasattr(FreqtradeBot, 'process_maybe_execute_buy') + assert hasattr(FreqtradeBot, 'process_maybe_execute_sell') + assert hasattr(FreqtradeBot, 'handle_trade') + assert hasattr(FreqtradeBot, 'check_handle_timedout') + assert hasattr(FreqtradeBot, 'handle_timedout_limit_buy') + assert hasattr(FreqtradeBot, 'handle_timedout_limit_sell') + assert hasattr(FreqtradeBot, 'execute_sell') + + +def test_freqtradebot(mocker, default_conf) -> None: + """ + Test __init__, _init_modules, update_state, and get_state methods + """ + freqtrade = get_patched_freqtradebot(mocker, default_conf) + assert freqtrade.get_state() is State.RUNNING + + conf = deepcopy(default_conf) + conf.pop('initial_state') + freqtrade = FreqtradeBot(conf) + assert freqtrade.get_state() is State.STOPPED + + +def test_clean(mocker, default_conf, caplog) -> None: + """ + Test clean() method + """ + mock_cleanup = MagicMock() + mocker.patch('freqtrade.persistence.cleanup', mock_cleanup) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + assert freqtrade.get_state() == State.RUNNING + + assert freqtrade.clean() + assert freqtrade.get_state() == State.STOPPED + assert log_has('Stopping trader and cleaning up modules...', caplog.record_tuples) + assert mock_cleanup.call_count == 1 + + +def test_worker_running(mocker, default_conf, caplog) -> None: + """ + Test worker() method. Test when we start the bot + """ + mock_throttle = MagicMock() + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._throttle', mock_throttle) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + state = freqtrade.worker(old_state=None) + assert state is State.RUNNING + assert log_has('Changing state to: RUNNING', caplog.record_tuples) + assert mock_throttle.call_count == 1 + + +def test_worker_stopped(mocker, default_conf, caplog) -> None: + """ + Test worker() method. Test when we stop the bot + """ + mock_throttle = MagicMock() + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._throttle', mock_throttle) + mock_sleep = mocker.patch('time.sleep', return_value=None) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade.update_state(State.STOPPED) + state = freqtrade.worker(old_state=State.RUNNING) + assert state is State.STOPPED + assert log_has('Changing state to: STOPPED', caplog.record_tuples) + assert mock_throttle.call_count == 0 + assert mock_sleep.call_count == 1 + + +def test_throttle(mocker, default_conf, caplog) -> None: + """ + Test _throttle() method + """ + def func(): + """ + Test function to throttle + """ + return 42 + + caplog.set_level(logging.DEBUG) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + start = time.time() + result = freqtrade._throttle(func, min_secs=0.1) + end = time.time() + + assert result == 42 + assert end - start > 0.1 + assert log_has('Throttling func for 0.10 seconds', caplog.record_tuples) + + result = freqtrade._throttle(func, min_secs=-1) + assert result == 42 + + +def test_throttle_with_assets(mocker, default_conf) -> None: + """ + Test _throttle() method when the function passed can have parameters + """ + def func(nb_assets=-1): + """ + Test function to throttle + """ + return nb_assets + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + result = freqtrade._throttle(func, min_secs=0.1, nb_assets=666) + assert result == 666 + + result = freqtrade._throttle(func, min_secs=0.1) + assert result == -1 + + +def test_gen_pair_whitelist(mocker, default_conf, get_market_summaries_data) -> None: + """ + Test _gen_pair_whitelist() method + """ + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_market_summaries', + return_value=get_market_summaries_data + ) + + # Test to retrieved BTC sorted on BaseVolume + whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC') + assert whitelist == ['BTC_ZCL', 'BTC_ZEC', 'BTC_XZC', 'BTC_XWC'] + + # Test to retrieved BTC sorted on OpenBuyOrders + whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='OpenBuyOrders') + assert whitelist == ['BTC_XWC', 'BTC_ZCL', 'BTC_ZEC', 'BTC_XZC'] + + # Test with USDT sorted on BaseVolume + whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT') + assert whitelist == ['USDT_XRP', 'USDT_XVG', 'USDT_XMR', 'USDT_ZEC'] + + # Test with ETH (our fixture does not have ETH, but Bittrex returns them) + whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH') + assert whitelist == [] + + +@pytest.mark.skip(reason="Test not implemented") +def test_refresh_whitelist() -> None: + """ + Test _refresh_whitelist() method + """ + pass + + +def test_create_trade(default_conf, ticker, limit_buy_order, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + + # Save state of current whitelist + whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade is not None + assert trade.stake_amount == 0.001 + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == Exchanges.BITTREX.name + + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + assert trade.open_rate == 0.00001099 + assert trade.amount == 90.99181073 + + assert whitelist == default_conf['exchange']['pair_whitelist'] + + +def test_create_trade_minimal_amount(default_conf, ticker, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + buy_mock = MagicMock(return_value='mocked_limit_buy') + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=buy_mock + ) + + conf = deepcopy(default_conf) + conf['stake_amount'] = 0.0005 + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + freqtrade.create_trade() + rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] + assert rate * amount >= conf['stake_amount'] + + +def test_create_trade_no_stake_amount(default_conf, ticker, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy'), + get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5) + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + with pytest.raises(DependencyException, match=r'.*stake amount.*'): + freqtrade.create_trade() + + +def test_create_trade_no_pairs(default_conf, ticker, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'] = ["BTC_ETH"] + conf['exchange']['pair_blacklist'] = ["BTC_ETH"] + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + freqtrade.create_trade() + + with pytest.raises(DependencyException, match=r'.*No currency pairs in whitelist.*'): + freqtrade.create_trade() + + +def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'] = ["BTC_ETH"] + conf['exchange']['pair_blacklist'] = ["BTC_ETH"] + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + freqtrade.create_trade() + + with pytest.raises(DependencyException, match=r'.*No currency pairs in whitelist.*'): + freqtrade.create_trade() + + +def test_create_trade_no_signal(default_conf, mocker) -> None: + """ + Test create_trade() method + """ + conf = deepcopy(default_conf) + conf['dry_run'] = True + + patch_get_signal(mocker, value=(False, False)) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker_history=MagicMock(return_value=20), + get_balance=MagicMock(return_value=20) + ) + + conf = deepcopy(default_conf) + conf['stake_amount'] = 10 + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + Trade.query = MagicMock() + Trade.query.filter = MagicMock() + assert not freqtrade.create_trade() + + +def test_process_trade_creation(default_conf, ticker, limit_buy_order, + health, mocker, caplog) -> None: + """ + Test the trade creation in _process() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_wallet_health=health, + buy=MagicMock(return_value='mocked_limit_buy'), + get_order=MagicMock(return_value=limit_buy_order) + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + assert not trades + + result = freqtrade._process() + assert result is True + + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + assert len(trades) == 1 + trade = trades[0] + assert trade is not None + assert trade.stake_amount == default_conf['stake_amount'] + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == Exchanges.BITTREX.name + assert trade.open_rate == 0.00001099 + assert trade.amount == 90.99181073703367 + + assert log_has( + 'Checking buy signals to create a new trade with stake_amount: 0.001000 ...', + caplog.record_tuples + ) + + +def test_process_exchange_failures(default_conf, ticker, health, mocker) -> None: + """ + Test _process() method when a RequestException happens + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_wallet_health=health, + buy=MagicMock(side_effect=requests.exceptions.RequestException) + ) + sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) + + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + result = freqtrade._process() + assert result is False + assert sleep_mock.has_calls() + + +def test_process_operational_exception(default_conf, ticker, health, mocker) -> None: + """ + Test _process() method when an OperationalException happens + """ + patch_get_signal(mocker) + msg_mock = patch_RPCManager(mocker) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_wallet_health=health, + buy=MagicMock(side_effect=OperationalException) + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + assert freqtrade.get_state() == State.RUNNING + + result = freqtrade._process() + assert result is False + assert freqtrade.get_state() == State.STOPPED + assert 'OperationalException' in msg_mock.call_args_list[-1][0][0] + + +def test_process_trade_handling(default_conf, ticker, limit_buy_order, health, mocker) -> None: + """ + Test _process() + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_wallet_health=health, + buy=MagicMock(return_value='mocked_limit_buy'), + get_order=MagicMock(return_value=limit_buy_order) + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + assert not trades + result = freqtrade._process() + assert result is True + trades = Trade.query.filter(Trade.is_open.is_(True)).all() + assert len(trades) == 1 + + result = freqtrade._process() + assert result is False + + +def test_balance_fully_ask_side(mocker) -> None: + """ + Test get_target_bid() method + """ + freqtrade = get_patched_freqtradebot(mocker, {'bid_strategy': {'ask_last_balance': 0.0}}) + + assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 20 + + +def test_balance_fully_last_side(mocker) -> None: + """ + Test get_target_bid() method + """ + freqtrade = get_patched_freqtradebot(mocker, {'bid_strategy': {'ask_last_balance': 1.0}}) + + assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 10 + + +def test_balance_bigger_last_ask(mocker) -> None: + """ + Test get_target_bid() method + """ + freqtrade = get_patched_freqtradebot(mocker, {'bid_strategy': {'ask_last_balance': 1.0}}) + + assert freqtrade.get_target_bid({'ask': 5, 'last': 10}) == 5 + + +def test_process_maybe_execute_buy(mocker, default_conf) -> None: + """ + Test process_maybe_execute_buy() method + """ + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(return_value=True)) + assert freqtrade.process_maybe_execute_buy() + + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(return_value=False)) + assert not freqtrade.process_maybe_execute_buy() + + +def test_process_maybe_execute_buy_exception(mocker, default_conf, caplog) -> None: + """ + Test exception on process_maybe_execute_buy() method + """ + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + mocker.patch( + 'freqtrade.freqtradebot.FreqtradeBot.create_trade', + MagicMock(side_effect=DependencyException) + ) + freqtrade.process_maybe_execute_buy() + log_has('Unable to create trade:', caplog.record_tuples) + + +def test_process_maybe_execute_sell(mocker, default_conf) -> None: + """ + Test process_maybe_execute_sell() method + """ + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) + mocker.patch('freqtrade.freqtradebot.exchange.get_order', return_value=1) + + trade = MagicMock() + trade.open_order_id = '123' + assert not freqtrade.process_maybe_execute_sell(trade) + trade.is_open = True + trade.open_order_id = None + # Assert we call handle_trade() if trade is feasible for execution + assert freqtrade.process_maybe_execute_sell(trade) + + +def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker) -> None: + """ + Test check_handle() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=MagicMock(return_value='mocked_limit_buy'), + sell=MagicMock(return_value='mocked_limit_sell') + ) + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + trade.update(limit_buy_order) + assert trade.is_open is True + + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade) is True + assert trade.open_order_id == 'mocked_limit_sell' + + # Simulate fulfilled LIMIT_SELL order for trade + trade.update(limit_sell_order) + + assert trade.close_rate == 0.00001173 + assert trade.close_profit == 0.06201057 + assert trade.calc_profit() == 0.00006217 + assert trade.close_date is not None + + +def test_handle_overlpapping_signals(default_conf, ticker, mocker) -> None: + """ + Test check_handle() method + """ + conf = deepcopy(default_conf) + conf.update({'experimental': {'use_sell_signal': True}}) + + patch_get_signal(mocker, value=(True, True)) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + + freqtrade.create_trade() + + # Buy and Sell triggering, so doing nothing ... + trades = Trade.query.all() + nb_trades = len(trades) + assert nb_trades == 0 + + # Buy is triggering, so buying ... + patch_get_signal(mocker, value=(True, False)) + freqtrade.create_trade() + trades = Trade.query.all() + nb_trades = len(trades) + assert nb_trades == 1 + assert trades[0].is_open is True + + # Buy and Sell are not triggering, so doing nothing ... + patch_get_signal(mocker, value=(False, False)) + assert freqtrade.handle_trade(trades[0]) is False + trades = Trade.query.all() + nb_trades = len(trades) + assert nb_trades == 1 + assert trades[0].is_open is True + + # Buy and Sell are triggering, so doing nothing ... + patch_get_signal(mocker, value=(True, True)) + assert freqtrade.handle_trade(trades[0]) is False + trades = Trade.query.all() + nb_trades = len(trades) + assert nb_trades == 1 + assert trades[0].is_open is True + + # Sell is triggering, guess what : we are Selling! + patch_get_signal(mocker, value=(False, True)) + trades = Trade.query.all() + assert freqtrade.handle_trade(trades[0]) is True + + +def test_handle_trade_roi(default_conf, ticker, mocker, caplog) -> None: + """ + Test check_handle() method + """ + caplog.set_level(logging.DEBUG) + conf = deepcopy(default_conf) + conf.update({'experimental': {'use_sell_signal': True}}) + + patch_get_signal(mocker, value=(True, False)) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.is_open = True + + # FIX: sniffing logs, suggest handle_trade should not execute_sell + # instead that responsibility should be moved out of handle_trade(), + # we might just want to check if we are in a sell condition without + # executing + # if ROI is reached we must sell + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade) + assert log_has('Required profit reached. Selling..', caplog.record_tuples) + + +def test_handle_trade_experimental(default_conf, ticker, mocker, caplog) -> None: + """ + Test check_handle() method + """ + caplog.set_level(logging.DEBUG) + conf = deepcopy(default_conf) + conf.update({'experimental': {'use_sell_signal': True}}) + + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.is_open = True + + patch_get_signal(mocker, value=(False, False)) + assert not freqtrade.handle_trade(trade) + + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade) + assert log_has('Sell signal received. Selling..', caplog.record_tuples) + + +def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mocker) -> None: + """ + Test check_handle() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value='mocked_limit_buy') + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + # Create trade and sell it + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + trade.update(limit_buy_order) + trade.update(limit_sell_order) + assert trade.is_open is False + + with pytest.raises(ValueError, match=r'.*closed trade.*'): + freqtrade.handle_trade(trade) + + +def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mocker) -> None: + """ + Test check_handle_timedout() method + """ + rpc_mock = patch_RPCManager(mocker) + cancel_order_mock = MagicMock() + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(return_value=limit_buy_order_old), + cancel_order=cancel_order_mock + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trade_buy = Trade( + pair='BTC_ETH', + open_rate=0.00001099, + exchange='BITTREX', + open_order_id='123456789', + amount=90.99181073, + fee=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + Trade.session.add(trade_buy) + + # check it does cancel buy orders over the time limit + freqtrade.check_handle_timedout(600) + assert cancel_order_mock.call_count == 1 + assert rpc_mock.call_count == 1 + trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 0 + + +def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker) -> None: + """ + Test check_handle_timedout() method + """ + rpc_mock = patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(return_value=limit_sell_order_old), + cancel_order=cancel_order_mock + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trade_sell = Trade( + pair='BTC_ETH', + open_rate=0.00001099, + exchange='BITTREX', + open_order_id='123456789', + amount=90.99181073, + fee=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(hours=-5).datetime, + close_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=False + ) + + Trade.session.add(trade_sell) + + # check it does cancel sell orders over the time limit + freqtrade.check_handle_timedout(600) + assert cancel_order_mock.call_count == 1 + assert rpc_mock.call_count == 1 + assert trade_sell.is_open is True + + +def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, + mocker) -> None: + """ + Test check_handle_timedout() method + """ + rpc_mock = patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(return_value=limit_buy_order_old_partial), + cancel_order=cancel_order_mock + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trade_buy = Trade( + pair='BTC_ETH', + open_rate=0.00001099, + exchange='BITTREX', + open_order_id='123456789', + amount=90.99181073, + fee=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + Trade.session.add(trade_buy) + + # check it does cancel buy orders over the time limit + # note this is for a partially-complete buy order + freqtrade.check_handle_timedout(600) + assert cancel_order_mock.call_count == 1 + assert rpc_mock.call_count == 1 + trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() + assert len(trades) == 1 + assert trades[0].amount == 23.0 + assert trades[0].stake_amount == trade_buy.open_rate * trades[0].amount + + +def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) -> None: + """ + Test check_handle_timedout() method when get_order throw an exception + """ + patch_RPCManager(mocker) + cancel_order_mock = MagicMock() + patch_coinmarketcap(mocker) + + mocker.patch.multiple( + 'freqtrade.freqtradebot.FreqtradeBot', + handle_timedout_limit_buy=MagicMock(), + handle_timedout_limit_sell=MagicMock(), + ) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(side_effect=requests.exceptions.RequestException('Oh snap')), + cancel_order=cancel_order_mock + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trade_buy = Trade( + pair='BTC_ETH', + open_rate=0.00001099, + exchange='BITTREX', + open_order_id='123456789', + amount=90.99181073, + fee=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + Trade.session.add(trade_buy) + regexp = re.compile( + 'Cannot query order for Trade(id=1, pair=BTC_ETH, amount=90.99181073, ' + 'open_rate=0.00001099, open_since=10 hours ago) due to Traceback (most ' + 'recent call last):\n.*' + ) + + freqtrade.check_handle_timedout(600) + assert filter(regexp.match, caplog.record_tuples) + + +def test_handle_timedout_limit_buy(mocker, default_conf) -> None: + """ + Test handle_timedout_limit_buy() method + """ + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + cancel_order=cancel_order_mock + ) + + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + Trade.session = MagicMock() + trade = MagicMock() + order = {'remaining': 1, + 'amount': 1} + assert freqtrade.handle_timedout_limit_buy(trade, order) + assert cancel_order_mock.call_count == 1 + order['amount'] = 2 + assert not freqtrade.handle_timedout_limit_buy(trade, order) + assert cancel_order_mock.call_count == 2 + + +def test_handle_timedout_limit_sell(mocker, default_conf) -> None: + """ + Test handle_timedout_limit_sell() method + """ + patch_RPCManager(mocker) + cancel_order_mock = MagicMock() + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + cancel_order=cancel_order_mock + ) + + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + trade = MagicMock() + order = {'remaining': 1, + 'amount': 1} + assert freqtrade.handle_timedout_limit_sell(trade, order) + assert cancel_order_mock.call_count == 1 + order['amount'] = 2 + assert not freqtrade.handle_timedout_limit_sell(trade, order) + # Assert cancel_order was not called (callcount remains unchanged) + assert cancel_order_mock.call_count == 1 + + +def test_execute_sell_up(default_conf, ticker, ticker_sell_up, mocker) -> None: + """ + Test execute_sell() method with a ticker going UP + """ + patch_get_signal(mocker) + rpc_mock = patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + # Create some test data + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up + ) + + freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert 'Amount' in rpc_mock.call_args_list[-1][0][0] + assert 'Profit' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] + assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] + assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] + + +def test_execute_sell_down(default_conf, ticker, ticker_sell_down, mocker) -> None: + """ + Test execute_sell() method with a ticker going DOWN + """ + patch_get_signal(mocker) + rpc_mock = patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + # Create some test data + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + # Decrease the price and sell it + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_down + ) + + freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert 'Amount' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] + assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] + + +def test_execute_sell_without_conf_sell_up(default_conf, ticker, ticker_sell_up, mocker) -> None: + """ + Test execute_sell() method with a ticker going DOWN and with a bot config empty + """ + patch_get_signal(mocker) + rpc_mock = patch_RPCManager(mocker) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + # Create some test data + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up + ) + freqtrade.config = {} + + freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert 'Amount' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] + assert '(profit: 6.11%, 0.00006126)' in rpc_mock.call_args_list[-1][0][0] + assert 'USD' not in rpc_mock.call_args_list[-1][0][0] + + +def test_execute_sell_without_conf_sell_down(default_conf, ticker, + ticker_sell_down, mocker) -> None: + """ + Test execute_sell() method with a ticker going DOWN and with a bot config empty + """ + patch_get_signal(mocker) + rpc_mock = patch_RPCManager(mocker) + patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker + ) + freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + + # Create some test data + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + # Decrease the price and sell it + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_down + ) + + freqtrade.config = {} + freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid']) + + assert rpc_mock.call_count == 2 + assert 'Selling' in rpc_mock.call_args_list[-1][0][0] + assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] + assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] + assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + + +def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, mocker) -> None: + """ + Test sell_profit_only feature when enabled + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00002172, + 'ask': 0.00002173, + 'last': 0.00002172 + }), + buy=MagicMock(return_value='mocked_limit_buy') + ) + conf = deepcopy(default_conf) + conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': True, + } + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade) is True + + +def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker) -> None: + """ + Test sell_profit_only feature when disabled + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00002172, + 'ask': 0.00002173, + 'last': 0.00002172 + }), + buy=MagicMock(return_value='mocked_limit_buy') + ) + conf = deepcopy(default_conf) + conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': False, + } + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade) is True + + +def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value='mocked_limit_buy') + ) + conf = deepcopy(default_conf) + conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': True, + } + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade) is False + + +def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value='mocked_limit_buy') + ) + + conf = deepcopy(default_conf) + conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': False, + } + + freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + patch_get_signal(mocker, value=(False, True)) + assert freqtrade.handle_trade(trade) is True diff --git a/freqtrade/tests/test_indicator_helpers.py b/freqtrade/tests/test_indicator_helpers.py index 90330a6ef..87b085a0b 100644 --- a/freqtrade/tests/test_indicator_helpers.py +++ b/freqtrade/tests/test_indicator_helpers.py @@ -1,4 +1,5 @@ import pandas as pd + from freqtrade.indicator_helpers import went_up, went_down diff --git a/freqtrade/tests/test_logger.py b/freqtrade/tests/test_logger.py new file mode 100644 index 000000000..8e094b2d1 --- /dev/null +++ b/freqtrade/tests/test_logger.py @@ -0,0 +1,97 @@ +""" +Unit test file for logger.py +""" + +import logging + +from freqtrade.logger import Logger + + +def test_logger_object() -> None: + """ + Test the Constants object has the mandatory Constants + :return: None + """ + logger = Logger() + assert logger.name == '' + assert logger.level == 20 + assert logger.level is logging.INFO + assert hasattr(logger, 'get_logger') + + logger = Logger(name='Foo', level=logging.WARNING) + assert logger.name == 'Foo' + assert logger.name != '' + assert logger.level == 30 + assert logger.level is logging.WARNING + + +def test_get_logger() -> None: + """ + Test Logger.get_logger() and Logger._init_logger() + :return: None + """ + logger = Logger(name='get_logger', level=logging.WARNING) + get_logger = logger.get_logger() + assert logger.logger is get_logger + assert get_logger is not None + assert hasattr(get_logger, 'debug') + assert hasattr(get_logger, 'info') + assert hasattr(get_logger, 'warning') + assert hasattr(get_logger, 'critical') + assert hasattr(get_logger, 'exception') + + +def test_set_name() -> None: + """ + Test Logger.set_name() + :return: None + """ + logger = Logger(name='set_name') + assert logger.name == 'set_name' + + logger.set_name('set_name_new') + assert logger.name == 'set_name_new' + + +def test_set_level() -> None: + """ + Test Logger.set_name() + :return: None + """ + logger = Logger(name='Foo', level=logging.WARNING) + assert logger.level == logging.WARNING + assert logger.get_logger().level == logging.WARNING + + logger.set_level(logging.INFO) + assert logger.level == logging.INFO + assert logger.get_logger().level == logging.INFO + + +def test_sending_msg(caplog) -> None: + """ + Test send a logging message + :return: None + """ + logger = Logger(name='sending_msg', level=logging.WARNING).get_logger() + + logger.info('I am an INFO message') + assert('sending_msg', logging.INFO, 'I am an INFO message') not in caplog.record_tuples + + logger.warning('I am an WARNING message') + assert ('sending_msg', logging.WARNING, 'I am an WARNING message') in caplog.record_tuples + + +def test_set_format(caplog) -> None: + """ + Test Logger.set_format() + :return: None + """ + log = Logger(name='set_format') + logger = log.get_logger() + + logger.info('I am the first message') + assert ('set_format', logging.INFO, 'I am the first message') in caplog.record_tuples + + log.set_format(log_format='%(message)s', propagate=True) + logger.info('I am the second message') + assert ('set_format', logging.INFO, 'I am the second message') in caplog.record_tuples diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 5f490a183..71b816968 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -1,30 +1,23 @@ -# pragma pylint: disable=missing-docstring, C0103 -import copy +""" +Unit test file for main.py +""" + import logging from unittest.mock import MagicMock -import arrow import pytest -import requests -from sqlalchemy import create_engine -import freqtrade.main as main -from freqtrade import DependencyException, OperationalException -from freqtrade.exchange import Exchanges -from freqtrade.main import (_process, check_handle_timedout, create_trade, - execute_sell, get_target_bid, handle_trade, init) -from freqtrade.misc import State, get_state -from freqtrade.persistence import Trade +from freqtrade.main import main, set_loggers from freqtrade.tests.conftest import log_has -def test_parse_args_backtesting(mocker): - """ Test that main() can start backtesting or hyperopt. - and also ensure we can pass some specific arguments - further argument parsing is done in test_misc.py """ - backtesting_mock = mocker.patch( - 'freqtrade.optimize.backtesting.start', MagicMock()) - main.main(['backtesting']) +def test_parse_args_backtesting(mocker) -> None: + """ + Test that main() can start backtesting and also ensure we can pass some specific arguments + further argument parsing is done in test_arguments.py + """ + backtesting_mock = mocker.patch('freqtrade.optimize.backtesting.start', MagicMock()) + main(['backtesting']) assert backtesting_mock.call_count == 1 call_args = backtesting_mock.call_args[0][0] assert call_args.config == 'config.json' @@ -35,10 +28,12 @@ def test_parse_args_backtesting(mocker): assert call_args.ticker_interval is None -def test_main_start_hyperopt(mocker): - hyperopt_mock = mocker.patch( - 'freqtrade.optimize.hyperopt.start', MagicMock()) - main.main(['hyperopt']) +def test_main_start_hyperopt(mocker) -> None: + """ + Test that main() can start hyperopt + """ + hyperopt_mock = mocker.patch('freqtrade.optimize.hyperopt.start', MagicMock()) + main(['hyperopt']) assert hyperopt_mock.call_count == 1 call_args = hyperopt_mock.call_args[0][0] assert call_args.config == 'config.json' @@ -47,793 +42,52 @@ def test_main_start_hyperopt(mocker): assert call_args.func is not None -def test_process_maybe_execute_buy(default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.create_trade', return_value=True) - assert main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) - mocker.patch('freqtrade.main.create_trade', return_value=False) - assert not main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) +def test_set_loggers() -> None: + """ + Test set_loggers() update the logger level for third-party libraries + """ + previous_value1 = logging.getLogger('requests.packages.urllib3').level + previous_value2 = logging.getLogger('telegram').level + + set_loggers() + + value1 = logging.getLogger('requests.packages.urllib3').level + assert previous_value1 is not value1 + assert value1 is logging.INFO + + value2 = logging.getLogger('telegram').level + assert previous_value2 is not value2 + assert value2 is logging.INFO -def test_process_maybe_execute_sell(default_conf, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.handle_trade', return_value=True) - mocker.patch('freqtrade.exchange.get_order', return_value=1) - trade = MagicMock() - trade.open_order_id = '123' - assert not main.process_maybe_execute_sell(trade, int(default_conf['ticker_interval'])) - trade.is_open = True - trade.open_order_id = None - # Assert we call handle_trade() if trade is feasible for execution - assert main.process_maybe_execute_sell(trade, int(default_conf['ticker_interval'])) - - -def test_process_maybe_execute_buy_exception(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.create_trade', MagicMock(side_effect=DependencyException)) - main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) - assert log_has('Unable to create trade: ', caplog.record_tuples) - - -def test_process_trade_creation(default_conf, ticker, limit_buy_order, health, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_wallet_health=health, - buy=MagicMock(return_value='mocked_limit_buy'), - get_order=MagicMock(return_value=limit_buy_order)) - init(default_conf, create_engine('sqlite://')) - - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - assert not trades - - result = _process(interval=int(default_conf['ticker_interval'])) - assert result is True - - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - assert len(trades) == 1 - trade = trades[0] - assert trade is not None - assert trade.stake_amount == default_conf['stake_amount'] - assert trade.is_open - assert trade.open_date is not None - assert trade.exchange == Exchanges.BITTREX.name - assert trade.open_rate == 0.00001099 - assert trade.amount == 90.99181073703367 - - -def test_process_exchange_failures(default_conf, ticker, health, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_wallet_health=health, - buy=MagicMock(side_effect=requests.exceptions.RequestException)) - init(default_conf, create_engine('sqlite://')) - result = _process(interval=int(default_conf['ticker_interval'])) - assert result is False - assert sleep_mock.has_calls() - - -def test_process_operational_exception(default_conf, ticker, health, mocker): - msg_mock = MagicMock() - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=msg_mock) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_wallet_health=health, - buy=MagicMock(side_effect=OperationalException)) - init(default_conf, create_engine('sqlite://')) - assert get_state() == State.RUNNING - - result = _process(interval=int(default_conf['ticker_interval'])) - assert result is False - assert get_state() == State.STOPPED - assert 'OperationalException' in msg_mock.call_args_list[-1][0][0] - - -def test_process_trade_handling(default_conf, ticker, limit_buy_order, health, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_wallet_health=health, - buy=MagicMock(return_value='mocked_limit_buy'), - get_order=MagicMock(return_value=limit_buy_order)) - init(default_conf, create_engine('sqlite://')) - - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - assert not trades - result = _process(interval=int(default_conf['ticker_interval'])) - assert result is True - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - assert len(trades) == 1 - - result = _process(interval=int(default_conf['ticker_interval'])) - assert result is False - - -def test_create_trade(default_conf, ticker, limit_buy_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - # Save state of current whitelist - whitelist = copy.deepcopy(default_conf['exchange']['pair_whitelist']) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade is not None - assert trade.stake_amount == 0.001 - assert trade.is_open - assert trade.open_date is not None - assert trade.exchange == Exchanges.BITTREX.name - - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) - - assert trade.open_rate == 0.00001099 - assert trade.amount == 90.99181073 - - assert whitelist == default_conf['exchange']['pair_whitelist'] - - -def test_create_trade_minimal_amount(default_conf, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - buy_mock = mocker.patch( - 'freqtrade.main.exchange.buy', MagicMock(return_value='mocked_limit_buy') +def test_main(mocker, caplog) -> None: + """ + Test main() function + In this test we are skipping the while True loop by throwing an exception. + """ + mocker.patch.multiple( + 'freqtrade.freqtradebot.FreqtradeBot', + _init_modules=MagicMock(), + worker=MagicMock( + side_effect=KeyboardInterrupt + ), + clean=MagicMock(), ) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - init(default_conf, create_engine('sqlite://')) - min_stake_amount = 0.0005 - create_trade(min_stake_amount, int(default_conf['ticker_interval'])) - rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] - assert rate * amount >= min_stake_amount + args = ['-c', 'config.json.example'] + # Test Main + the KeyboardInterrupt exception + with pytest.raises(SystemExit) as pytest_wrapped_e: + main(args) + log_has('Starting freqtrade', caplog.record_tuples) + log_has('Got SIGINT, aborting ...', caplog.record_tuples) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 42 -def test_create_trade_no_stake_amount(default_conf, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy'), - get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5)) - with pytest.raises(DependencyException, match=r'.*stake amount.*'): - create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) - - -def test_create_trade_no_pairs(default_conf, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - - with pytest.raises(DependencyException, match=r'.*No currency pairs in whitelist.*'): - conf = copy.deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = [] - mocker.patch.dict('freqtrade.main._CONF', conf) - create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) - - -def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - - with pytest.raises(DependencyException, match=r'.*No currency pairs in whitelist.*'): - conf = copy.deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = ["BTC_ETH"] - conf['exchange']['pair_blacklist'] = ["BTC_ETH"] - mocker.patch.dict('freqtrade.main._CONF', conf) - create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) - - -def test_create_trade_no_signal(default_conf, mocker): - default_conf['dry_run'] = True - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', MagicMock(return_value=(False, False))) - mocker.patch.multiple('freqtrade.exchange', - get_ticker_history=MagicMock(return_value=20)) - mocker.patch.multiple('freqtrade.main.exchange', - get_balance=MagicMock(return_value=20)) - stake_amount = 10 - Trade.query = MagicMock() - Trade.query.filter = MagicMock() - assert not create_trade(stake_amount, int(default_conf['ticker_interval'])) - - -def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 - }), - buy=MagicMock(return_value='mocked_limit_buy'), - sell=MagicMock(return_value='mocked_limit_sell')) - mocker.patch.multiple('freqtrade.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0})) - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - trade.update(limit_buy_order) - assert trade.is_open is True - - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) is True - assert trade.open_order_id == 'mocked_limit_sell' - - # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) - - assert trade.close_rate == 0.00001173 - assert trade.close_profit == 0.06201057 - assert trade.calc_profit() == 0.00006217 - assert trade.close_date is not None - - -def test_handle_overlpapping_signals(default_conf, ticker, mocker): - default_conf.update({'experimental': {'use_sell_signal': True}}) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, True)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - # Buy and Sell triggering, so doing nothing ... - trades = Trade.query.all() - nb_trades = len(trades) - assert nb_trades == 0 - - # Buy is triggering, so buying ... - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - create_trade(0.001, int(default_conf['ticker_interval'])) - trades = Trade.query.all() - nb_trades = len(trades) - assert nb_trades == 1 - assert trades[0].is_open is True - - # Buy and Sell are not triggering, so doing nothing ... - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, False)) - assert handle_trade(trades[0], int(default_conf['ticker_interval'])) is False - trades = Trade.query.all() - nb_trades = len(trades) - assert nb_trades == 1 - assert trades[0].is_open is True - - # Buy and Sell are triggering, so doing nothing ... - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, True)) - assert handle_trade(trades[0], int(default_conf['ticker_interval'])) is False - trades = Trade.query.all() - nb_trades = len(trades) - assert nb_trades == 1 - assert trades[0].is_open is True - - # Sell is triggering, guess what : we are Selling! - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - trades = Trade.query.all() - assert handle_trade(trades[0], int(default_conf['ticker_interval'])) is True - - -def test_handle_trade_roi(default_conf, ticker, mocker, caplog): - caplog.set_level(logging.DEBUG) - default_conf.update({'experimental': {'use_sell_signal': True}}) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - mocker.patch('freqtrade.main.min_roi_reached', return_value=True) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.is_open = True - - # FIX: sniffing logs, suggest handle_trade should not execute_sell - # instead that responsibility should be moved out of handle_trade(), - # we might just want to check if we are in a sell condition without - # executing - # if ROI is reached we must sell - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, interval=int(default_conf['ticker_interval'])) - assert log_has('Required profit reached. Selling..', caplog.record_tuples) - # if ROI is reached we must sell even if sell-signal is not signalled - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, interval=int(default_conf['ticker_interval'])) - assert log_has('Required profit reached. Selling..', caplog.record_tuples) - - -def test_handle_trade_experimental(default_conf, ticker, mocker, caplog): - caplog.set_level(logging.DEBUG) - default_conf.update({'experimental': {'use_sell_signal': True}}) - mocker.patch.dict('freqtrade.main._CONF', default_conf) - - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.is_open = True - - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, False)) - value_returned = handle_trade(trade, int(default_conf['ticker_interval'])) - assert value_returned is False - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) - assert log_has('Sell signal received. Selling..', caplog.record_tuples) - - -def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - buy=MagicMock(return_value='mocked_limit_buy')) - - # Create trade and sell it - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - trade.update(limit_buy_order) - trade.update(limit_sell_order) - assert trade.is_open is False - - with pytest.raises(ValueError, match=r'.*closed trade.*'): - handle_trade(trade, int(default_conf['ticker_interval'])) - - -def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - cancel_order_mock = MagicMock() - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_order=MagicMock(return_value=limit_buy_order_old), - cancel_order=cancel_order_mock) - init(default_conf, create_engine('sqlite://')) - - trade_buy = Trade( - pair='BTC_ETH', - open_rate=0.00001099, - exchange='BITTREX', - open_order_id='123456789', - amount=90.99181073, - fee=0.0, - stake_amount=1, - open_date=arrow.utcnow().shift(minutes=-601).datetime, - is_open=True + # Test the BaseException case + mocker.patch( + 'freqtrade.freqtradebot.FreqtradeBot.worker', + MagicMock(side_effect=BaseException) ) - - Trade.session.add(trade_buy) - - # check it does cancel buy orders over the time limit - check_handle_timedout(600) - assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 - trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() - nb_trades = len(trades) - assert nb_trades == 0 - - -def test_handle_timedout_limit_buy(mocker): - cancel_order = MagicMock() - mocker.patch('freqtrade.exchange.cancel_order', cancel_order) - Trade.session = MagicMock() - trade = MagicMock() - order = {'remaining': 1, - 'amount': 1} - assert main.handle_timedout_limit_buy(trade, order) - assert cancel_order.call_count == 1 - order['amount'] = 2 - assert not main.handle_timedout_limit_buy(trade, order) - assert cancel_order.call_count == 2 - - -def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - cancel_order_mock = MagicMock() - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_order=MagicMock(return_value=limit_sell_order_old), - cancel_order=cancel_order_mock) - init(default_conf, create_engine('sqlite://')) - - trade_sell = Trade( - pair='BTC_ETH', - open_rate=0.00001099, - exchange='BITTREX', - open_order_id='123456789', - amount=90.99181073, - fee=0.0, - stake_amount=1, - open_date=arrow.utcnow().shift(hours=-5).datetime, - close_date=arrow.utcnow().shift(minutes=-601).datetime, - is_open=False - ) - - Trade.session.add(trade_sell) - - # check it does cancel sell orders over the time limit - check_handle_timedout(600) - assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 - assert trade_sell.is_open is True - - -def test_handle_timedout_limit_sell(mocker): - cancel_order = MagicMock() - mocker.patch('freqtrade.exchange.cancel_order', cancel_order) - trade = MagicMock() - order = {'remaining': 1, - 'amount': 1} - assert main.handle_timedout_limit_sell(trade, order) - assert cancel_order.call_count == 1 - order['amount'] = 2 - assert not main.handle_timedout_limit_sell(trade, order) - # Assert cancel_order was not called (callcount remains unchanged) - assert cancel_order.call_count == 1 - - -def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, - mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - cancel_order_mock = MagicMock() - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker, - get_order=MagicMock(return_value=limit_buy_order_old_partial), - cancel_order=cancel_order_mock) - init(default_conf, create_engine('sqlite://')) - - trade_buy = Trade( - pair='BTC_ETH', - open_rate=0.00001099, - exchange='BITTREX', - open_order_id='123456789', - amount=90.99181073, - fee=0.0, - stake_amount=1, - open_date=arrow.utcnow().shift(minutes=-601).datetime, - is_open=True - ) - - Trade.session.add(trade_buy) - - # check it does cancel buy orders over the time limit - # note this is for a partially-complete buy order - check_handle_timedout(600) - assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 - trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() - assert len(trades) == 1 - assert trades[0].amount == 23.0 - assert trades[0].stake_amount == trade_buy.open_rate * trades[0].amount - - -def test_balance_fully_ask_side(mocker): - mocker.patch.dict('freqtrade.main._CONF', {'bid_strategy': {'ask_last_balance': 0.0}}) - assert get_target_bid({'ask': 20, 'last': 10}) == 20 - - -def test_balance_fully_last_side(mocker): - mocker.patch.dict('freqtrade.main._CONF', {'bid_strategy': {'ask_last_balance': 1.0}}) - assert get_target_bid({'ask': 20, 'last': 10}) == 10 - - -def test_balance_bigger_last_ask(mocker): - mocker.patch.dict('freqtrade.main._CONF', {'bid_strategy': {'ask_last_balance': 1.0}}) - assert get_target_bid({'ask': 5, 'last': 10}) == 5 - - -def test_execute_sell_up(default_conf, ticker, ticker_sell_up, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - # Increase the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) - - execute_sell(trade=trade, limit=ticker_sell_up()['bid']) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert 'Profit' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] - assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] - - -def test_execute_sell_down(default_conf, ticker, ticker_sell_down, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.rpc.telegram', - _CONF=default_conf, - init=MagicMock(), - send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - # Decrease the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_down) - - execute_sell(trade=trade, limit=ticker_sell_down()['bid']) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] - assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] - - -def test_execute_sell_without_conf_sell_down(default_conf, ticker, ticker_sell_down, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - # Decrease the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_down) - mocker.patch('freqtrade.main._CONF', {}) - - execute_sell(trade=trade, limit=ticker_sell_down()['bid']) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] - - -def test_execute_sell_without_conf_sell_up(default_conf, ticker, ticker_sell_up, mocker): - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch('freqtrade.rpc.init', MagicMock()) - rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker) - init(default_conf, create_engine('sqlite://')) - - # Create some test data - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - assert trade - - # Increase the price and sell it - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=ticker_sell_up) - mocker.patch('freqtrade.main._CONF', {}) - - execute_sell(trade=trade, limit=ticker_sell_up()['bid']) - - assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[BTC_ETH]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert '(profit: 6.11%, 0.00006126)' in rpc_mock.call_args_list[-1][0][0] - assert 'USD' not in rpc_mock.call_args_list[-1][0][0] - - -def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': True, - } - - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00002172, - 'ask': 0.00002173, - 'last': 0.00002172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.update(limit_buy_order) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) is True - - -def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': False, - } - - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00002172, - 'ask': 0.00002173, - 'last': 0.00002172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.update(limit_buy_order) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) is True - - -def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': True, - } - - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.update(limit_buy_order) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) is False - - -def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': False, - } - - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) - - init(default_conf, create_engine('sqlite://')) - create_trade(0.001, int(default_conf['ticker_interval'])) - - trade = Trade.query.first() - trade.update(limit_buy_order) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, True)) - assert handle_trade(trade, int(default_conf['ticker_interval'])) is True + with pytest.raises(SystemExit): + main(args) + log_has('Got fatal exception!', caplog.record_tuples) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index c0d4db7a1..3560b2db1 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -1,188 +1,34 @@ # pragma pylint: disable=missing-docstring,C0103 -import argparse -import json -import time -from copy import deepcopy -from unittest.mock import MagicMock + +""" +Unit test file for misc.py +""" import datetime -import pytest -from jsonschema import ValidationError -from freqtrade.analyze import parse_ticker_dataframe -from freqtrade.misc import (common_args_parser, file_dump_json, load_config, - parse_args, parse_timerange, throttle, datesarray_to_datetimearray) +from unittest.mock import MagicMock + +from freqtrade.analyze import Analyze +from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, + common_datearray, file_dump_json) +from freqtrade.optimize.__init__ import load_tickerdata_file -def test_throttle(): - - def func(): - return 42 - - start = time.time() - result = throttle(func, min_secs=0.1) - end = time.time() - - assert result == 42 - assert end - start > 0.1 - - result = throttle(func, min_secs=-1) - assert result == 42 - - -def test_throttle_with_assets(): - - def func(nb_assets=-1): - return nb_assets - - result = throttle(func, min_secs=0.1, nb_assets=666) - assert result == 666 - - result = throttle(func, min_secs=0.1) - assert result == -1 - - -# Parse common command-line-arguments. Used for all tools - -def test_parse_args_none(): - args = common_args_parser('') - assert isinstance(args, argparse.ArgumentParser) - - -def test_parse_args_defaults(): - args = parse_args([], '') - assert args.config == 'config.json' - assert args.dynamic_whitelist is None - assert args.loglevel == 20 - - -def test_parse_args_config(): - args = parse_args(['-c', '/dev/null'], '') - assert args.config == '/dev/null' - - args = parse_args(['--config', '/dev/null'], '') - assert args.config == '/dev/null' - - -def test_parse_args_verbose(): - args = parse_args(['-v'], '') - assert args.loglevel == 10 - - args = parse_args(['--verbose'], '') - assert args.loglevel == 10 - - -def test_parse_args_version(): - with pytest.raises(SystemExit, match=r'0'): - parse_args(['--version'], '') - - -def test_parse_args_invalid(): - with pytest.raises(SystemExit, match=r'2'): - parse_args(['-c'], '') - - -# Parse command-line-arguments -# used for main, backtesting and hyperopt - - -def test_parse_args_dynamic_whitelist(): - args = parse_args(['--dynamic-whitelist'], '') - assert args.dynamic_whitelist == 20 - - -def test_parse_args_dynamic_whitelist_10(): - args = parse_args(['--dynamic-whitelist', '10'], '') - assert args.dynamic_whitelist == 10 - - -def test_parse_args_dynamic_whitelist_invalid_values(): - with pytest.raises(SystemExit, match=r'2'): - parse_args(['--dynamic-whitelist', 'abc'], '') - - -def test_parse_args_backtesting_invalid(): - with pytest.raises(SystemExit, match=r'2'): - parse_args(['backtesting --ticker-interval'], '') - - with pytest.raises(SystemExit, match=r'2'): - parse_args(['backtesting --ticker-interval', 'abc'], '') - - -def test_parse_args_backtesting_custom(): - args = [ - '-c', 'test_conf.json', - 'backtesting', - '--live', - '--ticker-interval', '1', - '--refresh-pairs-cached'] - call_args = parse_args(args, '') - assert call_args.config == 'test_conf.json' - assert call_args.live is True - assert call_args.loglevel == 20 - assert call_args.subparser == 'backtesting' - assert call_args.func is not None - assert call_args.ticker_interval == 1 - assert call_args.refresh_pairs is True - - -def test_parse_args_hyperopt_custom(): - args = ['-c', 'test_conf.json', 'hyperopt', '--epochs', '20'] - call_args = parse_args(args, '') - assert call_args.config == 'test_conf.json' - assert call_args.epochs == 20 - assert call_args.loglevel == 20 - assert call_args.subparser == 'hyperopt' - assert call_args.func is not None - - -def test_file_dump_json(mocker): - file_open = mocker.patch('freqtrade.misc.open', MagicMock()) - json_dump = mocker.patch('json.dump', MagicMock()) - file_dump_json('somefile', [1, 2, 3]) - assert file_open.call_count == 1 - assert json_dump.call_count == 1 - - -def test_parse_timerange_incorrect(): - assert ((None, 'line'), None, -200) == parse_timerange('-200') - assert (('line', None), 200, None) == parse_timerange('200-') - with pytest.raises(Exception, match=r'Incorrect syntax.*'): - parse_timerange('-') - - -def test_load_config(default_conf, mocker): - file_mock = mocker.patch('freqtrade.misc.open', mocker.mock_open( - read_data=json.dumps(default_conf) - )) - validated_conf = load_config('somefile') - assert file_mock.call_count == 1 - assert validated_conf.items() >= default_conf.items() - - -def test_load_config_invalid_pair(default_conf, mocker): - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'].append('BTC-ETH') - mocker.patch( - 'freqtrade.misc.open', - mocker.mock_open( - read_data=json.dumps(conf))) - with pytest.raises(ValidationError, match=r'.*does not match.*'): - load_config('somefile') - - -def test_load_config_missing_attributes(default_conf, mocker): - conf = deepcopy(default_conf) - conf.pop('exchange') - mocker.patch( - 'freqtrade.misc.open', - mocker.mock_open( - read_data=json.dumps(conf))) - with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): - load_config('somefile') +def test_shorten_date() -> None: + """ + Test shorten_date() function + :return: None + """ + str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago' + str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago' + assert shorten_date(str_data) == str_shorten_data def test_datesarray_to_datetimearray(ticker_history): - dataframes = parse_ticker_dataframe(ticker_history) + """ + Test datesarray_to_datetimearray() function + :return: None + """ + dataframes = Analyze.parse_ticker_dataframe(ticker_history) dates = datesarray_to_datetimearray(dataframes['date']) assert isinstance(dates[0], datetime.datetime) @@ -194,3 +40,34 @@ def test_datesarray_to_datetimearray(ticker_history): date_len = len(dates) assert date_len == 3 + + +def test_common_datearray(default_conf, mocker) -> None: + """ + Test common_datearray() + :return: None + """ + mocker.patch('freqtrade.strategy.strategy.Strategy', MagicMock()) + + analyze = Analyze(default_conf) + tick = load_tickerdata_file(None, 'BTC_UNITEST', 1) + tickerlist = {'BTC_UNITEST': tick} + dataframes = analyze.tickerdata_to_dataframe(tickerlist) + + dates = common_datearray(dataframes) + + assert dates.size == dataframes['BTC_UNITEST']['date'].size + assert dates[0] == dataframes['BTC_UNITEST']['date'][0] + assert dates[-1] == dataframes['BTC_UNITEST']['date'][-1] + + +def test_file_dump_json(mocker) -> None: + """ + Test file_dump_json() + :return: None + """ + file_open = mocker.patch('freqtrade.misc.open', MagicMock()) + json_dump = mocker.patch('json.dump', MagicMock()) + file_dump_json('somefile', [1, 2, 3]) + assert file_open.call_count == 1 + assert json_dump.call_count == 1 diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index f6c5318ce..70199b12a 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,7 +1,9 @@ # pragma pylint: disable=missing-docstring, C0103 import os + import pytest from sqlalchemy import create_engine + from freqtrade.exchange import Exchanges from freqtrade.persistence import Trade, init, clean_dry_run_db diff --git a/freqtrade/tests/test_state.py b/freqtrade/tests/test_state.py new file mode 100644 index 000000000..51fa06cc2 --- /dev/null +++ b/freqtrade/tests/test_state.py @@ -0,0 +1,14 @@ +""" +Unit test file for constants.py +""" + +from freqtrade.state import State + + +def test_state_object() -> None: + """ + Test the State object has the mandatory states + :return: None + """ + assert hasattr(State, 'RUNNING') + assert hasattr(State, 'STOPPED') diff --git a/freqtrade/tests/testdata/download_backtest_data.py b/freqtrade/tests/testdata/download_backtest_data.py index 0cb545b3a..ceb8388a1 100755 --- a/freqtrade/tests/testdata/download_backtest_data.py +++ b/freqtrade/tests/testdata/download_backtest_data.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """This script generate json data from bittrex""" -import sys import json +import sys from freqtrade import exchange -from freqtrade.exchange import Bittrex from freqtrade import misc +from freqtrade.exchange import Bittrex parser = misc.common_args_parser('download utility') parser.add_argument( diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index e62465bf7..285ba6d97 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -1,41 +1,55 @@ #!/usr/bin/env python3 +""" +Script to display when the bot will buy a specific pair + +Mandatory Cli parameters: +-p / --pair: pair to examine + +Optional Cli parameters +-s / --strategy: strategy to use +-d / --datadir: path to pair backtest data +--timerange: specify what timerange of data to use. +-l / --live: Live, to download the latest ticker for the pair +""" import sys -import logging +from argparse import Namespace + +from typing import List from plotly import tools from plotly.offline import plot import plotly.graph_objs as go -from freqtrade import exchange, analyze -from freqtrade.strategy.strategy import Strategy -import freqtrade.misc as misc +from freqtrade.arguments import Arguments +from freqtrade.analyze import Analyze +from freqtrade import exchange +from freqtrade.logger import Logger import freqtrade.optimize as optimize -logger = logging.getLogger(__name__) +logger = Logger(name="Graph dataframe").get_logger() -def plot_parse_args(args): - parser = misc.common_args_parser('Graph dataframe') - misc.backtesting_options(parser) - misc.scripts_options(parser) - return parser.parse_args(args) - - -def plot_analyzed_dataframe(args) -> None: +def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe - :param pair: pair as str :return: None """ pair = args.pair.replace('-', '_') - timerange = misc.parse_timerange(args.timerange) + timerange = Arguments.parse_timerange(args.timerange) # Init strategy - strategy = Strategy() - strategy.init({'strategy': args.strategy}) - tick_interval = strategy.ticker_interval + try: + analyze = Analyze({'strategy': args.strategy}) + except AttributeError: + logger.critical( + 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', + args.strategy + ) + exit() + + tick_interval = analyze.strategy.ticker_interval tickers = {} if args.live: @@ -44,27 +58,32 @@ def plot_analyzed_dataframe(args) -> None: exchange._API = exchange.Bittrex({'key': '', 'secret': ''}) tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: - tickers = optimize.load_data(args.datadir, pairs=[pair], - ticker_interval=tick_interval, - refresh_pairs=False, - timerange=timerange) - dataframes = optimize.tickerdata_to_dataframe(tickers) + tickers = optimize.load_data( + datadir=args.datadir, + pairs=[pair], + ticker_interval=tick_interval, + refresh_pairs=False, + timerange=timerange + ) + dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) - if (len(dataframe.index) > 750): - logger.warn('Ticker contained more than 750 candles, clipping.') - df = dataframe.tail(750) + if len(dataframe.index) > 750: + logger.warning('Ticker contained more than 750 candles, clipping.') + data = dataframe.tail(750) - candles = go.Candlestick(x=df.date, - open=df.open, - high=df.high, - low=df.low, - close=df.close, - name='Price') + candles = go.Candlestick( + x=data.date, + open=data.open, + high=data.high, + low=data.low, + close=data.close, + name='Price' + ) - df_buy = df[df['buy'] == 1] + df_buy = data[data['buy'] == 1] buys = go.Scattergl( x=df_buy.date, y=df_buy.close, @@ -73,13 +92,11 @@ def plot_analyzed_dataframe(args) -> None: marker=dict( symbol='triangle-up-dot', size=9, - line=dict( - width=1, - ), + line=dict(width=1), color='green', ) ) - df_sell = df[df['sell'] == 1] + df_sell = data[data['sell'] == 1] sells = go.Scattergl( x=df_sell.date, y=df_sell.close, @@ -88,30 +105,28 @@ def plot_analyzed_dataframe(args) -> None: marker=dict( symbol='triangle-down-dot', size=9, - line=dict( - width=1, - ), + line=dict(width=1), color='red', ) ) bb_lower = go.Scatter( - x=df.date, - y=df.bb_lowerband, + x=data.date, + y=data.bb_lowerband, name='BB lower', line={'color': "transparent"}, ) bb_upper = go.Scatter( - x=df.date, - y=df.bb_upperband, + x=data.date, + y=data.bb_upperband, name='BB upper', fill="tonexty", fillcolor="rgba(0,176,246,0.2)", line={'color': "transparent"}, ) - macd = go.Scattergl(x=df['date'], y=df['macd'], name='MACD') - macdsignal = go.Scattergl(x=df['date'], y=df['macdsignal'], name='MACD signal') - volume = go.Bar(x=df['date'], y=df['volume'], name='Volume') + macd = go.Scattergl(x=data['date'], y=data['macd'], name='MACD') + macdsignal = go.Scattergl(x=data['date'], y=data['macdsignal'], name='MACD signal') + volume = go.Bar(x=data['date'], y=data['volume'], name='Volume') fig = tools.make_subplots( rows=3, @@ -138,6 +153,31 @@ def plot_analyzed_dataframe(args) -> None: plot(fig, filename='freqtrade-plot.html') +def plot_parse_args(args: List[str]) -> Namespace: + """ + Parse args passed to the script + :param args: Cli arguments + :return: args: Array with all arguments + """ + arguments = Arguments(args, 'Graph dataframe') + arguments.scripts_options() + arguments.common_args_parser() + arguments.optimizer_shared_options(arguments.parser) + arguments.backtesting_options(arguments.parser) + + return arguments.parse_args() + + +def main(sysargv: List[str]) -> None: + """ + This function will initiate the bot and start the trading loop. + :return: None + """ + logger.info('Starting Plot Dataframe') + plot_analyzed_dataframe( + plot_parse_args(sysargv) + ) + + if __name__ == '__main__': - args = plot_parse_args(sys.argv[1:]) - plot_analyzed_dataframe(args) + main(sys.argv[1:]) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index c51b29309..022bbf33c 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -1,32 +1,45 @@ #!/usr/bin/env python3 +""" +Script to display profits + +Mandatory Cli parameters: +-p / --pair: pair to examine + +Optional Cli parameters +-c / --config: specify configuration file +-s / --strategy: strategy to use +--timerange: specify what timerange of data to use. +""" import sys import json +from argparse import Namespace +from typing import List, Optional import numpy as np from plotly import tools from plotly.offline import plot import plotly.graph_objs as go +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.analyze import Analyze +from freqtrade.logger import Logger + import freqtrade.optimize as optimize import freqtrade.misc as misc -from freqtrade.strategy.strategy import Strategy -def plot_parse_args(args): - parser = misc.common_args_parser('Graph profits') - # FIX: perhaps delete those backtesting options that are not feasible (shows up in -h) - misc.backtesting_options(parser) - misc.scripts_options(parser) - return parser.parse_args(args) +logger = Logger(name="Graph profits").get_logger() # data:: [ pair, profit-%, enter, exit, time, duration] -# data:: ['BTC_XMR', 0.00537847, '1511176800', '1511178000', 5057, 1] -# FIX: make use of the enter/exit dates to insert the -# profit more precisely into the pg array -def make_profit_array(data, px, filter_pairs=[]): +# data:: ["BTC_ETH", 0.0023975, "1515598200", "1515602100", "2018-01-10 07:30:00+00:00", 65] +def make_profit_array( + data: List, px: int, min_date: int, + interval: int, filter_pairs: Optional[List] = None) -> np.ndarray: pg = np.zeros(px) + filter_pairs = filter_pairs or [] # Go through the trades # and make an total profit # array @@ -35,10 +48,11 @@ def make_profit_array(data, px, filter_pairs=[]): if filter_pairs and pair not in filter_pairs: continue profit = trade[1] - tim = trade[4] - dur = trade[5] - ix = tim + dur - 1 + trade_sell_time = int(trade[3]) + + ix = define_index(min_date, trade_sell_time, interval) if ix < px: + logger.debug('[%s]: Add profit %s on %s', pair, profit, trade[4]) pg[ix] += profit # rewrite the pg array to go from @@ -53,7 +67,7 @@ def make_profit_array(data, px, filter_pairs=[]): return pg -def plot_profit(args) -> None: +def plot_profit(args: Namespace) -> None: """ Plots the total profit for all pairs. Note, the profit calculation isn't realistic. @@ -64,47 +78,62 @@ def plot_profit(args) -> None: # We need to use the same pairs, same tick_interval # and same timeperiod as used in backtesting # to match the tickerdata against the profits-results + timerange = Arguments.parse_timerange(args.timerange) - filter_pairs = args.pair - - config = misc.load_config(args.config) - config.update({'strategy': args.strategy}) + config = Configuration(args).get_config() # Init strategy - strategy = Strategy() - strategy.init(config) + try: + analyze = Analyze({'strategy': config.get('strategy')}) + except AttributeError: + logger.critical( + 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', + config.get('strategy') + ) + exit() + # Take pairs from the cli otherwise switch to the pair in the config file + if args.pair: + filter_pairs = args.pair + filter_pairs = filter_pairs.split(',') + else: + filter_pairs = config['exchange']['pair_whitelist'] + + tick_interval = analyze.strategy.ticker_interval pairs = config['exchange']['pair_whitelist'] if filter_pairs: - filter_pairs = filter_pairs.split(',') pairs = list(set(pairs) & set(filter_pairs)) - print('Filter, keep pairs %s' % pairs) + logger.info('Filter, keep pairs %s' % pairs) - timerange = misc.parse_timerange(args.timerange) - tickers = optimize.load_data(args.datadir, pairs=pairs, - ticker_interval=strategy.ticker_interval, - refresh_pairs=False, - timerange=timerange) - dataframes = optimize.preprocess(tickers) + tickers = optimize.load_data( + datadir=args.datadir, + pairs=pairs, + ticker_interval=tick_interval, + refresh_pairs=False, + timerange=timerange + ) + dataframes = analyze.tickerdata_to_dataframe(tickers) # NOTE: the dataframes are of unequal length, # 'dates' is an merged date array of them all. dates = misc.common_datearray(dataframes) - max_x = dates.size + min_date = int(min(dates).timestamp()) + max_date = int(max(dates).timestamp()) + num_iterations = define_index(min_date, max_date, tick_interval) + 1 # Make an average close price of all the pairs that was involved. # this could be useful to gauge the overall market trend # We are essentially saying: # array <- sum dataframes[*]['close'] / num_items dataframes # FIX: there should be some onliner numpy/panda for this - avgclose = np.zeros(max_x) + avgclose = np.zeros(num_iterations) num = 0 for pair, pair_data in dataframes.items(): close = pair_data['close'] maxprice = max(close) # Normalize price to [0,1] - print('Pair %s has length %s' % (pair, len(close))) + logger.info('Pair %s has length %s' % (pair, len(close))) for x in range(0, len(close)): avgclose[x] += close[x] / maxprice # avgclose += close @@ -114,10 +143,16 @@ def plot_profit(args) -> None: # Load the profits results # And make an profits-growth array - filename = 'backtest-result.json' - with open(filename) as file: - data = json.load(file) - pg = make_profit_array(data, max_x, filter_pairs) + try: + filename = 'backtest-result.json' + with open(filename) as file: + data = json.load(file) + except FileNotFoundError: + logger.critical('File "backtest-result.json" not found. This script require backtesting ' + 'results to run.\nPlease run a backtesting with the parameter --export.') + exit(0) + + pg = make_profit_array(data, num_iterations, min_date, tick_interval, filter_pairs) # # Plot the pairs average close prices, and total profit growth @@ -128,6 +163,7 @@ def plot_profit(args) -> None: y=avgclose, name='Avg close price', ) + profit = go.Scattergl( x=dates, y=pg, @@ -140,7 +176,7 @@ def plot_profit(args) -> None: fig.append_trace(profit, 2, 1) for pair in pairs: - pg = make_profit_array(data, max_x, pair) + pg = make_profit_array(data, num_iterations, min_date, tick_interval, pair) pair_profit = go.Scattergl( x=dates, y=pg, @@ -151,6 +187,38 @@ def plot_profit(args) -> None: plot(fig, filename='freqtrade-profit-plot.html') +def define_index(min_date: int, max_date: int, interval: int) -> int: + """ + Return the index of a specific date + """ + return int((max_date - min_date) / (interval * 60)) + + +def plot_parse_args(args: List[str]) -> Namespace: + """ + Parse args passed to the script + :param args: Cli arguments + :return: args: Array with all arguments + """ + arguments = Arguments(args, 'Graph profits') + arguments.scripts_options() + arguments.common_args_parser() + arguments.optimizer_shared_options(arguments.parser) + arguments.backtesting_options(arguments.parser) + + return arguments.parse_args() + + +def main(sysargv: List[str]) -> None: + """ + This function will initiate the bot and start the trading loop. + :return: None + """ + logger.info('Starting Plot Dataframe') + plot_profit( + plot_parse_args(sysargv) + ) + + if __name__ == '__main__': - args = plot_parse_args(sys.argv[1:]) - plot_profit(args) + main(sys.argv[1:])