diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 22041094c..741a306a1 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -1,30 +1,26 @@ -import argparse -import enum +""" +Various tool function for Freqtrade and scripts +""" + +import re import json import logging -import time -import os -import re from datetime import datetime -from typing import Any, Callable, Dict, List - import numpy as np -from jsonschema import Draft4Validator, validate -from jsonschema.exceptions import ValidationError, best_match -from wrapt import synchronized - -from freqtrade import __version__ logger = logging.getLogger(__name__) -class State(enum.Enum): - RUNNING = 0 - STOPPED = 1 - - -# Current application state -_STATE = State.STOPPED +def shorten_date(_date): + """ + Trim the date so it fits on small screens + """ + new_date = re.sub('seconds?', 'sec', _date) + new_date = re.sub('minutes?', 'min', new_date) + new_date = re.sub('hours?', 'h', new_date) + new_date = re.sub('days?', 'd', new_date) + new_date = re.sub('^an?', '1', new_date) + return new_date ############################################ @@ -32,7 +28,6 @@ _STATE = State.STOPPED # Matplotlib doesn't support ::datetime64, # # so we need to convert it into ::datetime # ############################################ - def datesarray_to_datetimearray(dates): """ Convert an pandas-array of timestamps into @@ -41,13 +36,18 @@ def datesarray_to_datetimearray(dates): """ times = [] dates = dates.astype(datetime) - for i in range(0, dates.size): - date = dates[i].to_pydatetime() + for index in range(0, dates.size): + date = dates[index].to_pydatetime() times.append(date) return np.array(times) def common_datearray(dfs): + """ + Return dates from Dataframe + :param dfs: Dataframe + :return: List of dates + """ alldates = {} for pair, pair_data in dfs.items(): dates = datesarray_to_datetimearray(pair_data['date']) @@ -61,375 +61,11 @@ def common_datearray(dfs): def file_dump_json(filename, data) -> None: - with open(filename, 'w') as fp: - json.dump(data, fp) - - -@synchronized -def update_state(state: State) -> None: """ - Updates the application state - :param state: new state - :return: None - """ - global _STATE - _STATE = state - - -@synchronized -def get_state() -> State: - """ - Gets the current application state + Dump JSON data into a file + :param filename: file to create + :param data: JSON Data to save :return: """ - return _STATE - - -def load_config(path: str) -> Dict: - """ - Loads a config file from the given path - :param path: path as str - :return: configuration as dictionary - """ - with open(path) as file: - conf = json.load(file) - if 'internals' not in conf: - conf['internals'] = {} - logger.info('Validating configuration ...') - try: - validate(conf, CONF_SCHEMA) - return conf - except ValidationError as exception: - logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception) - raise ValidationError( - best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message - ) - - -def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: - """ - Throttles the given callable that it - takes at least `min_secs` to finish execution. - :param func: Any callable - :param min_secs: minimum execution time in seconds - :return: Any - """ - start = time.time() - result = func(*args, **kwargs) - end = time.time() - duration = max(min_secs - (end - start), 0.0) - logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) - time.sleep(duration) - return result - - -def common_args_parser(description: str): - """ - Parses given common arguments and returns them as a parsed object. - """ - parser = argparse.ArgumentParser( - description=description - ) - parser.add_argument( - '-v', '--verbose', - help='be verbose', - action='store_const', - dest='loglevel', - const=logging.DEBUG, - default=logging.INFO, - ) - parser.add_argument( - '--version', - action='version', - version='%(prog)s {}'.format(__version__), - ) - parser.add_argument( - '-c', '--config', - help='specify configuration file (default: config.json)', - dest='config', - default='config.json', - type=str, - metavar='PATH', - ) - parser.add_argument( - '--datadir', - help='path to backtest data (default freqdata/tests/testdata)', - dest='datadir', - default=os.path.join('freqtrade', 'tests', 'testdata'), - type=str, - metavar='PATH', - ) - parser.add_argument( - '-s', '--strategy', - help='specify strategy file (default: freqtrade/strategy/default_strategy.py)', - dest='strategy', - default='default_strategy', - type=str, - metavar='PATH', - ) - return parser - - -def parse_args(args: List[str], description: str): - """ - Parses given arguments and returns an argparse Namespace instance. - Returns None if a sub command has been selected and executed. - """ - parser = common_args_parser(description) - parser.add_argument( - '--dry-run-db', - help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \ - instead of memory DB. Work only if dry_run is enabled.', - action='store_true', - dest='dry_run_db', - ) - parser.add_argument( - '--dynamic-whitelist', - help='dynamically generate and update whitelist \ - based on 24h BaseVolume (Default 20 currencies)', # noqa - dest='dynamic_whitelist', - const=20, - type=int, - metavar='INT', - nargs='?', - ) - - build_subcommands(parser) - return parser.parse_args(args) - - -def scripts_options(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - '-p', '--pair', - help='Show profits for only this pairs. Pairs are comma-separated.', - dest='pair', - default=None - ) - - -def backtesting_options(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - '-l', '--live', - action='store_true', - dest='live', - help='using live data', - ) - parser.add_argument( - '-i', '--ticker-interval', - help='specify ticker interval in minutes (1, 5, 30, 60, 1440)', - dest='ticker_interval', - type=int, - metavar='INT', - ) - parser.add_argument( - '--realistic-simulation', - help='uses max_open_trades from config to simulate real world limitations', - action='store_true', - dest='realistic_simulation', - ) - parser.add_argument( - '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \ - Use it if you want to run your backtesting with up-to-date data.', - action='store_true', - dest='refresh_pairs', - ) - parser.add_argument( - '--export', - help='Export backtest results, argument are: trades\ - Example --export=trades', - type=str, - default=None, - dest='export', - ) - parser.add_argument( - '--timerange', - help='Specify what timerange of data to use.', - default=None, - type=str, - dest='timerange', - ) - - -def hyperopt_options(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - '-e', '--epochs', - help='specify number of epochs (default: 100)', - dest='epochs', - default=100, - type=int, - metavar='INT', - ) - parser.add_argument( - '--use-mongodb', - help='parallelize evaluations with mongodb (requires mongod in PATH)', - dest='mongodb', - action='store_true', - ) - parser.add_argument( - '-i', '--ticker-interval', - help='specify ticker interval in minutes (1, 5, 30, 60, 1440)', - dest='ticker_interval', - type=int, - metavar='INT', - ) - parser.add_argument( - '--timerange', - help='Specify what timerange of data to use.', - default=None, - type=str, - dest='timerange', - ) - - -def parse_timerange(text): - if text is None: - return None - syntax = [(r'^-(\d{8})$', (None, 'date')), - (r'^(\d{8})-$', ('date', None)), - (r'^(\d{8})-(\d{8})$', ('date', 'date')), - (r'^(-\d+)$', (None, 'line')), - (r'^(\d+)-$', ('line', None)), - (r'^(\d+)-(\d+)$', ('index', 'index'))] - for rex, stype in syntax: - # Apply the regular expression to text - match = re.match(rex, text) - if match: # Regex has matched - rvals = match.groups() - index = 0 - start = None - stop = None - if stype[0]: - start = rvals[index] - if stype[0] != 'date': - start = int(start) - index += 1 - if stype[1]: - stop = rvals[index] - if stype[1] != 'date': - stop = int(stop) - return (stype, start, stop) - raise Exception('Incorrect syntax for timerange "%s"' % text) - - -def build_subcommands(parser: argparse.ArgumentParser) -> None: - """ Builds and attaches all subcommands """ - from freqtrade.optimize import backtesting, hyperopt - - subparsers = parser.add_subparsers(dest='subparser') - - # Add backtesting subcommand - backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module') - backtesting_cmd.set_defaults(func=backtesting.start) - backtesting_options(backtesting_cmd) - - # Add hyperopt subcommand - hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') - hyperopt_cmd.set_defaults(func=hyperopt.start) - hyperopt_options(hyperopt_cmd) - - -# Required json-schema for user specified config -CONF_SCHEMA = { - 'type': 'object', - 'properties': { - 'max_open_trades': {'type': 'integer', 'minimum': 1}, - 'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, - 'stake_amount': {'type': 'number', 'minimum': 0.0005}, - 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', - 'CLP', 'CNY', 'CZK', 'DKK', - 'EUR', 'GBP', 'HKD', 'HUF', - 'IDR', 'ILS', 'INR', 'JPY', - 'KRW', 'MXN', 'MYR', 'NOK', - 'NZD', 'PHP', 'PKR', 'PLN', - 'RUB', 'SEK', 'SGD', 'THB', - 'TRY', 'TWD', 'ZAR', 'USD']}, - 'dry_run': {'type': 'boolean'}, - 'minimal_roi': { - 'type': 'object', - 'patternProperties': { - '^[0-9.]+$': {'type': 'number'} - }, - 'minProperties': 1 - }, - 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, - 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, - 'bid_strategy': { - 'type': 'object', - 'properties': { - 'ask_last_balance': { - 'type': 'number', - 'minimum': 0, - 'maximum': 1, - 'exclusiveMaximum': False - }, - }, - 'required': ['ask_last_balance'] - }, - 'exchange': {'$ref': '#/definitions/exchange'}, - 'experimental': { - 'type': 'object', - 'properties': { - 'use_sell_signal': {'type': 'boolean'}, - 'sell_profit_only': {'type': 'boolean'} - } - }, - 'telegram': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'token': {'type': 'string'}, - 'chat_id': {'type': 'string'}, - }, - 'required': ['enabled', 'token', 'chat_id'] - }, - 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, - 'internals': { - 'type': 'object', - 'properties': { - 'process_throttle_secs': {'type': 'number'}, - 'interval': {'type': 'integer'} - } - } - }, - 'definitions': { - 'exchange': { - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'key': {'type': 'string'}, - 'secret': {'type': 'string'}, - 'pair_whitelist': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' - }, - 'uniqueItems': True - }, - 'pair_blacklist': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' - }, - 'uniqueItems': True - } - }, - 'required': ['name', 'key', 'secret', 'pair_whitelist'] - } - }, - 'anyOf': [ - {'required': ['exchange']} - ], - 'required': [ - 'max_open_trades', - 'stake_currency', - 'stake_amount', - 'fiat_display_currency', - 'dry_run', - 'bid_strategy', - 'telegram' - ] -} + with open(filename, 'w') as file: + json.dump(data, file) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index c0d4db7a1..95b9bf338 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -1,188 +1,31 @@ # pragma pylint: disable=missing-docstring,C0103 -import argparse -import json -import time -from copy import deepcopy -from unittest.mock import MagicMock + +""" +Unit test file for misc.py +""" import datetime -import pytest -from jsonschema import ValidationError -from freqtrade.analyze import parse_ticker_dataframe -from freqtrade.misc import (common_args_parser, file_dump_json, load_config, - parse_args, parse_timerange, throttle, datesarray_to_datetimearray) +from unittest.mock import MagicMock +from freqtrade.analyze import Analyze +from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, file_dump_json) -def test_throttle(): - - def func(): - return 42 - - start = time.time() - result = throttle(func, min_secs=0.1) - end = time.time() - - assert result == 42 - assert end - start > 0.1 - - result = throttle(func, min_secs=-1) - assert result == 42 - - -def test_throttle_with_assets(): - - def func(nb_assets=-1): - return nb_assets - - result = throttle(func, min_secs=0.1, nb_assets=666) - assert result == 666 - - result = throttle(func, min_secs=0.1) - assert result == -1 - - -# Parse common command-line-arguments. Used for all tools - -def test_parse_args_none(): - args = common_args_parser('') - assert isinstance(args, argparse.ArgumentParser) - - -def test_parse_args_defaults(): - args = parse_args([], '') - assert args.config == 'config.json' - assert args.dynamic_whitelist is None - assert args.loglevel == 20 - - -def test_parse_args_config(): - args = parse_args(['-c', '/dev/null'], '') - assert args.config == '/dev/null' - - args = parse_args(['--config', '/dev/null'], '') - assert args.config == '/dev/null' - - -def test_parse_args_verbose(): - args = parse_args(['-v'], '') - assert args.loglevel == 10 - - args = parse_args(['--verbose'], '') - assert args.loglevel == 10 - - -def test_parse_args_version(): - with pytest.raises(SystemExit, match=r'0'): - parse_args(['--version'], '') - - -def test_parse_args_invalid(): - with pytest.raises(SystemExit, match=r'2'): - parse_args(['-c'], '') - - -# Parse command-line-arguments -# used for main, backtesting and hyperopt - - -def test_parse_args_dynamic_whitelist(): - args = parse_args(['--dynamic-whitelist'], '') - assert args.dynamic_whitelist == 20 - - -def test_parse_args_dynamic_whitelist_10(): - args = parse_args(['--dynamic-whitelist', '10'], '') - assert args.dynamic_whitelist == 10 - - -def test_parse_args_dynamic_whitelist_invalid_values(): - with pytest.raises(SystemExit, match=r'2'): - parse_args(['--dynamic-whitelist', 'abc'], '') - - -def test_parse_args_backtesting_invalid(): - with pytest.raises(SystemExit, match=r'2'): - parse_args(['backtesting --ticker-interval'], '') - - with pytest.raises(SystemExit, match=r'2'): - parse_args(['backtesting --ticker-interval', 'abc'], '') - - -def test_parse_args_backtesting_custom(): - args = [ - '-c', 'test_conf.json', - 'backtesting', - '--live', - '--ticker-interval', '1', - '--refresh-pairs-cached'] - call_args = parse_args(args, '') - assert call_args.config == 'test_conf.json' - assert call_args.live is True - assert call_args.loglevel == 20 - assert call_args.subparser == 'backtesting' - assert call_args.func is not None - assert call_args.ticker_interval == 1 - assert call_args.refresh_pairs is True - - -def test_parse_args_hyperopt_custom(): - args = ['-c', 'test_conf.json', 'hyperopt', '--epochs', '20'] - call_args = parse_args(args, '') - assert call_args.config == 'test_conf.json' - assert call_args.epochs == 20 - assert call_args.loglevel == 20 - assert call_args.subparser == 'hyperopt' - assert call_args.func is not None - - -def test_file_dump_json(mocker): - file_open = mocker.patch('freqtrade.misc.open', MagicMock()) - json_dump = mocker.patch('json.dump', MagicMock()) - file_dump_json('somefile', [1, 2, 3]) - assert file_open.call_count == 1 - assert json_dump.call_count == 1 - - -def test_parse_timerange_incorrect(): - assert ((None, 'line'), None, -200) == parse_timerange('-200') - assert (('line', None), 200, None) == parse_timerange('200-') - with pytest.raises(Exception, match=r'Incorrect syntax.*'): - parse_timerange('-') - - -def test_load_config(default_conf, mocker): - file_mock = mocker.patch('freqtrade.misc.open', mocker.mock_open( - read_data=json.dumps(default_conf) - )) - validated_conf = load_config('somefile') - assert file_mock.call_count == 1 - assert validated_conf.items() >= default_conf.items() - - -def test_load_config_invalid_pair(default_conf, mocker): - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'].append('BTC-ETH') - mocker.patch( - 'freqtrade.misc.open', - mocker.mock_open( - read_data=json.dumps(conf))) - with pytest.raises(ValidationError, match=r'.*does not match.*'): - load_config('somefile') - - -def test_load_config_missing_attributes(default_conf, mocker): - conf = deepcopy(default_conf) - conf.pop('exchange') - mocker.patch( - 'freqtrade.misc.open', - mocker.mock_open( - read_data=json.dumps(conf))) - with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): - load_config('somefile') +def test_shorten_date() -> None: + """ + Test shorten_date() function + :return: None + """ + str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago' + str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago' + assert shorten_date(str_data) == str_shorten_data def test_datesarray_to_datetimearray(ticker_history): - dataframes = parse_ticker_dataframe(ticker_history) + """ + Test datesarray_to_datetimearray() function + :return: None + """ + dataframes = Analyze.parse_ticker_dataframe(ticker_history) dates = datesarray_to_datetimearray(dataframes['date']) assert isinstance(dates[0], datetime.datetime) @@ -194,3 +37,15 @@ def test_datesarray_to_datetimearray(ticker_history): date_len = len(dates) assert date_len == 3 + + +def test_file_dump_json(mocker): + """ + Test file_dump_json() + :return: None + """ + file_open = mocker.patch('freqtrade.misc.open', MagicMock()) + json_dump = mocker.patch('json.dump', MagicMock()) + file_dump_json('somefile', [1, 2, 3]) + assert file_open.call_count == 1 + assert json_dump.call_count == 1