Keep in misc file only tool functions

This commit is contained in:
Gerald Lonlas 2018-02-03 23:33:54 -08:00
parent 89e3729955
commit e025dc0dba
2 changed files with 59 additions and 568 deletions

View File

@ -1,30 +1,26 @@
import argparse
import enum
"""
Various tool function for Freqtrade and scripts
"""
import re
import json
import logging
import time
import os
import re
from datetime import datetime
from typing import Any, Callable, Dict, List
import numpy as np
from jsonschema import Draft4Validator, validate
from jsonschema.exceptions import ValidationError, best_match
from wrapt import synchronized
from freqtrade import __version__
logger = logging.getLogger(__name__)
class State(enum.Enum):
RUNNING = 0
STOPPED = 1
# Current application state
_STATE = State.STOPPED
def shorten_date(_date):
"""
Trim the date so it fits on small screens
"""
new_date = re.sub('seconds?', 'sec', _date)
new_date = re.sub('minutes?', 'min', new_date)
new_date = re.sub('hours?', 'h', new_date)
new_date = re.sub('days?', 'd', new_date)
new_date = re.sub('^an?', '1', new_date)
return new_date
############################################
@ -32,7 +28,6 @@ _STATE = State.STOPPED
# Matplotlib doesn't support ::datetime64, #
# so we need to convert it into ::datetime #
############################################
def datesarray_to_datetimearray(dates):
"""
Convert an pandas-array of timestamps into
@ -41,13 +36,18 @@ def datesarray_to_datetimearray(dates):
"""
times = []
dates = dates.astype(datetime)
for i in range(0, dates.size):
date = dates[i].to_pydatetime()
for index in range(0, dates.size):
date = dates[index].to_pydatetime()
times.append(date)
return np.array(times)
def common_datearray(dfs):
"""
Return dates from Dataframe
:param dfs: Dataframe
:return: List of dates
"""
alldates = {}
for pair, pair_data in dfs.items():
dates = datesarray_to_datetimearray(pair_data['date'])
@ -61,375 +61,11 @@ def common_datearray(dfs):
def file_dump_json(filename, data) -> None:
with open(filename, 'w') as fp:
json.dump(data, fp)
@synchronized
def update_state(state: State) -> None:
"""
Updates the application state
:param state: new state
:return: None
"""
global _STATE
_STATE = state
@synchronized
def get_state() -> State:
"""
Gets the current application state
Dump JSON data into a file
:param filename: file to create
:param data: JSON Data to save
:return:
"""
return _STATE
def load_config(path: str) -> Dict:
"""
Loads a config file from the given path
:param path: path as str
:return: configuration as dictionary
"""
with open(path) as file:
conf = json.load(file)
if 'internals' not in conf:
conf['internals'] = {}
logger.info('Validating configuration ...')
try:
validate(conf, CONF_SCHEMA)
return conf
except ValidationError as exception:
logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception)
raise ValidationError(
best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message
)
def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any:
"""
Throttles the given callable that it
takes at least `min_secs` to finish execution.
:param func: Any callable
:param min_secs: minimum execution time in seconds
:return: Any
"""
start = time.time()
result = func(*args, **kwargs)
end = time.time()
duration = max(min_secs - (end - start), 0.0)
logger.debug('Throttling %s for %.2f seconds', func.__name__, duration)
time.sleep(duration)
return result
def common_args_parser(description: str):
"""
Parses given common arguments and returns them as a parsed object.
"""
parser = argparse.ArgumentParser(
description=description
)
parser.add_argument(
'-v', '--verbose',
help='be verbose',
action='store_const',
dest='loglevel',
const=logging.DEBUG,
default=logging.INFO,
)
parser.add_argument(
'--version',
action='version',
version='%(prog)s {}'.format(__version__),
)
parser.add_argument(
'-c', '--config',
help='specify configuration file (default: config.json)',
dest='config',
default='config.json',
type=str,
metavar='PATH',
)
parser.add_argument(
'--datadir',
help='path to backtest data (default freqdata/tests/testdata)',
dest='datadir',
default=os.path.join('freqtrade', 'tests', 'testdata'),
type=str,
metavar='PATH',
)
parser.add_argument(
'-s', '--strategy',
help='specify strategy file (default: freqtrade/strategy/default_strategy.py)',
dest='strategy',
default='default_strategy',
type=str,
metavar='PATH',
)
return parser
def parse_args(args: List[str], description: str):
"""
Parses given arguments and returns an argparse Namespace instance.
Returns None if a sub command has been selected and executed.
"""
parser = common_args_parser(description)
parser.add_argument(
'--dry-run-db',
help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \
instead of memory DB. Work only if dry_run is enabled.',
action='store_true',
dest='dry_run_db',
)
parser.add_argument(
'--dynamic-whitelist',
help='dynamically generate and update whitelist \
based on 24h BaseVolume (Default 20 currencies)', # noqa
dest='dynamic_whitelist',
const=20,
type=int,
metavar='INT',
nargs='?',
)
build_subcommands(parser)
return parser.parse_args(args)
def scripts_options(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
'-p', '--pair',
help='Show profits for only this pairs. Pairs are comma-separated.',
dest='pair',
default=None
)
def backtesting_options(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
'-l', '--live',
action='store_true',
dest='live',
help='using live data',
)
parser.add_argument(
'-i', '--ticker-interval',
help='specify ticker interval in minutes (1, 5, 30, 60, 1440)',
dest='ticker_interval',
type=int,
metavar='INT',
)
parser.add_argument(
'--realistic-simulation',
help='uses max_open_trades from config to simulate real world limitations',
action='store_true',
dest='realistic_simulation',
)
parser.add_argument(
'-r', '--refresh-pairs-cached',
help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \
Use it if you want to run your backtesting with up-to-date data.',
action='store_true',
dest='refresh_pairs',
)
parser.add_argument(
'--export',
help='Export backtest results, argument are: trades\
Example --export=trades',
type=str,
default=None,
dest='export',
)
parser.add_argument(
'--timerange',
help='Specify what timerange of data to use.',
default=None,
type=str,
dest='timerange',
)
def hyperopt_options(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
'-e', '--epochs',
help='specify number of epochs (default: 100)',
dest='epochs',
default=100,
type=int,
metavar='INT',
)
parser.add_argument(
'--use-mongodb',
help='parallelize evaluations with mongodb (requires mongod in PATH)',
dest='mongodb',
action='store_true',
)
parser.add_argument(
'-i', '--ticker-interval',
help='specify ticker interval in minutes (1, 5, 30, 60, 1440)',
dest='ticker_interval',
type=int,
metavar='INT',
)
parser.add_argument(
'--timerange',
help='Specify what timerange of data to use.',
default=None,
type=str,
dest='timerange',
)
def parse_timerange(text):
if text is None:
return None
syntax = [(r'^-(\d{8})$', (None, 'date')),
(r'^(\d{8})-$', ('date', None)),
(r'^(\d{8})-(\d{8})$', ('date', 'date')),
(r'^(-\d+)$', (None, 'line')),
(r'^(\d+)-$', ('line', None)),
(r'^(\d+)-(\d+)$', ('index', 'index'))]
for rex, stype in syntax:
# Apply the regular expression to text
match = re.match(rex, text)
if match: # Regex has matched
rvals = match.groups()
index = 0
start = None
stop = None
if stype[0]:
start = rvals[index]
if stype[0] != 'date':
start = int(start)
index += 1
if stype[1]:
stop = rvals[index]
if stype[1] != 'date':
stop = int(stop)
return (stype, start, stop)
raise Exception('Incorrect syntax for timerange "%s"' % text)
def build_subcommands(parser: argparse.ArgumentParser) -> None:
""" Builds and attaches all subcommands """
from freqtrade.optimize import backtesting, hyperopt
subparsers = parser.add_subparsers(dest='subparser')
# Add backtesting subcommand
backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module')
backtesting_cmd.set_defaults(func=backtesting.start)
backtesting_options(backtesting_cmd)
# Add hyperopt subcommand
hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')
hyperopt_cmd.set_defaults(func=hyperopt.start)
hyperopt_options(hyperopt_cmd)
# Required json-schema for user specified config
CONF_SCHEMA = {
'type': 'object',
'properties': {
'max_open_trades': {'type': 'integer', 'minimum': 1},
'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]},
'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']},
'stake_amount': {'type': 'number', 'minimum': 0.0005},
'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF',
'CLP', 'CNY', 'CZK', 'DKK',
'EUR', 'GBP', 'HKD', 'HUF',
'IDR', 'ILS', 'INR', 'JPY',
'KRW', 'MXN', 'MYR', 'NOK',
'NZD', 'PHP', 'PKR', 'PLN',
'RUB', 'SEK', 'SGD', 'THB',
'TRY', 'TWD', 'ZAR', 'USD']},
'dry_run': {'type': 'boolean'},
'minimal_roi': {
'type': 'object',
'patternProperties': {
'^[0-9.]+$': {'type': 'number'}
},
'minProperties': 1
},
'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True},
'unfilledtimeout': {'type': 'integer', 'minimum': 0},
'bid_strategy': {
'type': 'object',
'properties': {
'ask_last_balance': {
'type': 'number',
'minimum': 0,
'maximum': 1,
'exclusiveMaximum': False
},
},
'required': ['ask_last_balance']
},
'exchange': {'$ref': '#/definitions/exchange'},
'experimental': {
'type': 'object',
'properties': {
'use_sell_signal': {'type': 'boolean'},
'sell_profit_only': {'type': 'boolean'}
}
},
'telegram': {
'type': 'object',
'properties': {
'enabled': {'type': 'boolean'},
'token': {'type': 'string'},
'chat_id': {'type': 'string'},
},
'required': ['enabled', 'token', 'chat_id']
},
'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
'internals': {
'type': 'object',
'properties': {
'process_throttle_secs': {'type': 'number'},
'interval': {'type': 'integer'}
}
}
},
'definitions': {
'exchange': {
'type': 'object',
'properties': {
'name': {'type': 'string'},
'key': {'type': 'string'},
'secret': {'type': 'string'},
'pair_whitelist': {
'type': 'array',
'items': {
'type': 'string',
'pattern': '^[0-9A-Z]+_[0-9A-Z]+$'
},
'uniqueItems': True
},
'pair_blacklist': {
'type': 'array',
'items': {
'type': 'string',
'pattern': '^[0-9A-Z]+_[0-9A-Z]+$'
},
'uniqueItems': True
}
},
'required': ['name', 'key', 'secret', 'pair_whitelist']
}
},
'anyOf': [
{'required': ['exchange']}
],
'required': [
'max_open_trades',
'stake_currency',
'stake_amount',
'fiat_display_currency',
'dry_run',
'bid_strategy',
'telegram'
]
}
with open(filename, 'w') as file:
json.dump(data, file)

View File

@ -1,188 +1,31 @@
# pragma pylint: disable=missing-docstring,C0103
import argparse
import json
import time
from copy import deepcopy
from unittest.mock import MagicMock
"""
Unit test file for misc.py
"""
import datetime
import pytest
from jsonschema import ValidationError
from freqtrade.analyze import parse_ticker_dataframe
from freqtrade.misc import (common_args_parser, file_dump_json, load_config,
parse_args, parse_timerange, throttle, datesarray_to_datetimearray)
from unittest.mock import MagicMock
from freqtrade.analyze import Analyze
from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, file_dump_json)
def test_throttle():
def func():
return 42
start = time.time()
result = throttle(func, min_secs=0.1)
end = time.time()
assert result == 42
assert end - start > 0.1
result = throttle(func, min_secs=-1)
assert result == 42
def test_throttle_with_assets():
def func(nb_assets=-1):
return nb_assets
result = throttle(func, min_secs=0.1, nb_assets=666)
assert result == 666
result = throttle(func, min_secs=0.1)
assert result == -1
# Parse common command-line-arguments. Used for all tools
def test_parse_args_none():
args = common_args_parser('')
assert isinstance(args, argparse.ArgumentParser)
def test_parse_args_defaults():
args = parse_args([], '')
assert args.config == 'config.json'
assert args.dynamic_whitelist is None
assert args.loglevel == 20
def test_parse_args_config():
args = parse_args(['-c', '/dev/null'], '')
assert args.config == '/dev/null'
args = parse_args(['--config', '/dev/null'], '')
assert args.config == '/dev/null'
def test_parse_args_verbose():
args = parse_args(['-v'], '')
assert args.loglevel == 10
args = parse_args(['--verbose'], '')
assert args.loglevel == 10
def test_parse_args_version():
with pytest.raises(SystemExit, match=r'0'):
parse_args(['--version'], '')
def test_parse_args_invalid():
with pytest.raises(SystemExit, match=r'2'):
parse_args(['-c'], '')
# Parse command-line-arguments
# used for main, backtesting and hyperopt
def test_parse_args_dynamic_whitelist():
args = parse_args(['--dynamic-whitelist'], '')
assert args.dynamic_whitelist == 20
def test_parse_args_dynamic_whitelist_10():
args = parse_args(['--dynamic-whitelist', '10'], '')
assert args.dynamic_whitelist == 10
def test_parse_args_dynamic_whitelist_invalid_values():
with pytest.raises(SystemExit, match=r'2'):
parse_args(['--dynamic-whitelist', 'abc'], '')
def test_parse_args_backtesting_invalid():
with pytest.raises(SystemExit, match=r'2'):
parse_args(['backtesting --ticker-interval'], '')
with pytest.raises(SystemExit, match=r'2'):
parse_args(['backtesting --ticker-interval', 'abc'], '')
def test_parse_args_backtesting_custom():
args = [
'-c', 'test_conf.json',
'backtesting',
'--live',
'--ticker-interval', '1',
'--refresh-pairs-cached']
call_args = parse_args(args, '')
assert call_args.config == 'test_conf.json'
assert call_args.live is True
assert call_args.loglevel == 20
assert call_args.subparser == 'backtesting'
assert call_args.func is not None
assert call_args.ticker_interval == 1
assert call_args.refresh_pairs is True
def test_parse_args_hyperopt_custom():
args = ['-c', 'test_conf.json', 'hyperopt', '--epochs', '20']
call_args = parse_args(args, '')
assert call_args.config == 'test_conf.json'
assert call_args.epochs == 20
assert call_args.loglevel == 20
assert call_args.subparser == 'hyperopt'
assert call_args.func is not None
def test_file_dump_json(mocker):
file_open = mocker.patch('freqtrade.misc.open', MagicMock())
json_dump = mocker.patch('json.dump', MagicMock())
file_dump_json('somefile', [1, 2, 3])
assert file_open.call_count == 1
assert json_dump.call_count == 1
def test_parse_timerange_incorrect():
assert ((None, 'line'), None, -200) == parse_timerange('-200')
assert (('line', None), 200, None) == parse_timerange('200-')
with pytest.raises(Exception, match=r'Incorrect syntax.*'):
parse_timerange('-')
def test_load_config(default_conf, mocker):
file_mock = mocker.patch('freqtrade.misc.open', mocker.mock_open(
read_data=json.dumps(default_conf)
))
validated_conf = load_config('somefile')
assert file_mock.call_count == 1
assert validated_conf.items() >= default_conf.items()
def test_load_config_invalid_pair(default_conf, mocker):
conf = deepcopy(default_conf)
conf['exchange']['pair_whitelist'].append('BTC-ETH')
mocker.patch(
'freqtrade.misc.open',
mocker.mock_open(
read_data=json.dumps(conf)))
with pytest.raises(ValidationError, match=r'.*does not match.*'):
load_config('somefile')
def test_load_config_missing_attributes(default_conf, mocker):
conf = deepcopy(default_conf)
conf.pop('exchange')
mocker.patch(
'freqtrade.misc.open',
mocker.mock_open(
read_data=json.dumps(conf)))
with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'):
load_config('somefile')
def test_shorten_date() -> None:
"""
Test shorten_date() function
:return: None
"""
str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago'
str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago'
assert shorten_date(str_data) == str_shorten_data
def test_datesarray_to_datetimearray(ticker_history):
dataframes = parse_ticker_dataframe(ticker_history)
"""
Test datesarray_to_datetimearray() function
:return: None
"""
dataframes = Analyze.parse_ticker_dataframe(ticker_history)
dates = datesarray_to_datetimearray(dataframes['date'])
assert isinstance(dates[0], datetime.datetime)
@ -194,3 +37,15 @@ def test_datesarray_to_datetimearray(ticker_history):
date_len = len(dates)
assert date_len == 3
def test_file_dump_json(mocker):
"""
Test file_dump_json()
:return: None
"""
file_open = mocker.patch('freqtrade.misc.open', MagicMock())
json_dump = mocker.patch('json.dump', MagicMock())
file_dump_json('somefile', [1, 2, 3])
assert file_open.call_count == 1
assert json_dump.call_count == 1