stable/freqtrade/misc.py

436 lines
13 KiB
Python
Raw Normal View History

import argparse
import enum
2017-11-17 16:54:31 +00:00
import json
import logging
import time
import os
2018-01-15 21:25:02 +00:00
import re
from datetime import datetime
2018-01-10 07:51:36 +00:00
from typing import Any, Callable, Dict, List
import numpy as np
2018-01-10 07:51:36 +00:00
from jsonschema import Draft4Validator, validate
from jsonschema.exceptions import ValidationError, best_match
from wrapt import synchronized
from freqtrade import __version__
2017-11-11 15:47:19 +00:00
logger = logging.getLogger(__name__)
class State(enum.Enum):
RUNNING = 0
STOPPED = 1
# Current application state
_STATE = State.STOPPED
############################################
# Used by scripts #
# Matplotlib doesn't support ::datetime64, #
# so we need to convert it into ::datetime #
############################################
def datesarray_to_datetimearray(dates):
"""
Convert an pandas-array of timestamps into
An numpy-array of datetimes
:return: numpy-array of datetime
"""
times = []
dates = dates.astype(datetime)
for i in range(0, dates.size):
date = dates[i].to_pydatetime()
times.append(date)
return np.array(times)
def common_datearray(dfs):
alldates = {}
for pair, pair_data in dfs.items():
dates = datesarray_to_datetimearray(pair_data['date'])
for date in dates:
alldates[date] = 1
lst = []
for date, _ in alldates.items():
lst.append(date)
arr = np.array(lst)
return np.sort(arr, axis=0)
2018-01-28 22:28:28 +00:00
def file_dump_json(filename, data) -> None:
with open(filename, 'w') as fp:
json.dump(data, fp)
@synchronized
def update_state(state: State) -> None:
"""
Updates the application state
:param state: new state
:return: None
"""
global _STATE
_STATE = state
@synchronized
def get_state() -> State:
"""
Gets the current application state
:return:
"""
return _STATE
2017-05-12 17:11:56 +00:00
2017-11-17 16:54:31 +00:00
def load_config(path: str) -> Dict:
"""
Loads a config file from the given path
:param path: path as str
:return: configuration as dictionary
"""
with open(path) as file:
conf = json.load(file)
if 'internals' not in conf:
conf['internals'] = {}
logger.info('Validating configuration ...')
try:
validate(conf, CONF_SCHEMA)
return conf
2018-01-09 11:06:28 +00:00
except ValidationError as exception:
logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception)
raise ValidationError(
best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message
)
2017-11-17 16:54:31 +00:00
2017-11-11 15:47:19 +00:00
def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any:
"""
Throttles the given callable that it
takes at least `min_secs` to finish execution.
:param func: Any callable
:param min_secs: minimum execution time in seconds
:return: Any
"""
start = time.time()
result = func(*args, **kwargs)
end = time.time()
duration = max(min_secs - (end - start), 0.0)
logger.debug('Throttling %s for %.2f seconds', func.__name__, duration)
time.sleep(duration)
return result
2018-01-09 11:06:28 +00:00
def common_args_parser(description: str):
2017-11-17 16:18:31 +00:00
"""
Parses given common arguments and returns them as a parsed object.
2017-11-17 16:18:31 +00:00
"""
parser = argparse.ArgumentParser(
description=description
)
parser.add_argument(
'-v', '--verbose',
help='be verbose',
action='store_const',
dest='loglevel',
const=logging.DEBUG,
default=logging.INFO,
)
parser.add_argument(
'--version',
action='version',
version='%(prog)s {}'.format(__version__),
)
parser.add_argument(
'-c', '--config',
2018-02-04 13:48:41 +00:00
help='specify configuration file (default: %(default)s)',
dest='config',
default='config.json',
type=str,
metavar='PATH',
)
2018-01-12 09:52:44 +00:00
parser.add_argument(
2018-02-04 13:48:41 +00:00
'-d', '--datadir',
help='path to backtest data (default: %(default)s',
2018-01-12 09:52:44 +00:00
dest='datadir',
default=os.path.join('freqtrade', 'tests', 'testdata'),
type=str,
metavar='PATH',
)
2018-01-15 08:35:11 +00:00
parser.add_argument(
'-s', '--strategy',
2018-02-04 13:48:41 +00:00
help='specify strategy file (default: %(default)s)',
2018-01-15 08:35:11 +00:00
dest='strategy',
default='default_strategy',
2018-01-15 08:35:11 +00:00
type=str,
metavar='PATH',
)
return parser
def parse_args(args: List[str], description: str):
"""
Parses given arguments and returns an argparse Namespace instance.
Returns None if a sub command has been selected and executed.
"""
2018-01-09 11:06:28 +00:00
parser = common_args_parser(description)
parser.add_argument(
'--dry-run-db',
2018-01-09 11:06:28 +00:00
help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \
instead of memory DB. Work only if dry_run is enabled.',
action='store_true',
dest='dry_run_db',
)
parser.add_argument(
'--dynamic-whitelist',
2018-01-09 11:06:28 +00:00
help='dynamically generate and update whitelist \
based on 24h BaseVolume (Default 20 currencies)', # noqa
dest='dynamic_whitelist',
const=20,
type=int,
metavar='INT',
nargs='?',
)
2017-11-14 21:15:24 +00:00
build_subcommands(parser)
return parser.parse_args(args)
def scripts_options(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
'-p', '--pair',
help='Show profits for only this pairs. Pairs are comma-separated.',
dest='pair',
default=None
)
2018-02-15 13:23:49 +00:00
def optimizer_shared_options(parser: argparse.ArgumentParser) -> None:
2018-01-12 09:52:44 +00:00
parser.add_argument(
2017-11-14 21:37:30 +00:00
'-i', '--ticker-interval',
help='specify ticker interval in minutes (1, 5, 30, 60, 1440)',
2017-11-14 21:37:30 +00:00
dest='ticker_interval',
type=int,
metavar='INT',
)
2018-01-12 09:52:44 +00:00
parser.add_argument(
'--realistic-simulation',
2017-11-22 23:25:06 +00:00
help='uses max_open_trades from config to simulate real world limitations',
action='store_true',
dest='realistic_simulation',
2017-11-22 23:25:06 +00:00
)
2018-02-15 13:23:49 +00:00
parser.add_argument(
'--timerange',
help='Specify what timerange of data to use.',
default=None,
type=str,
dest='timerange',
)
def backtesting_options(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
'-l', '--live',
action='store_true',
dest='live',
help='using live data',
)
2018-01-12 09:52:44 +00:00
parser.add_argument(
'-r', '--refresh-pairs-cached',
help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \
Use it if you want to run your backtesting with up-to-date data.',
action='store_true',
dest='refresh_pairs',
)
parser.add_argument(
2018-01-11 14:45:39 +00:00
'--export',
help='Export backtest results, argument are: trades\
Example --export=trades',
2018-01-11 14:45:39 +00:00
type=str,
default=None,
dest='export',
)
2017-11-14 21:15:24 +00:00
2018-01-12 09:52:44 +00:00
def hyperopt_options(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
2017-11-25 00:12:44 +00:00
'-e', '--epochs',
2018-02-04 13:48:41 +00:00
help='specify number of epochs (default: %(default)d)',
2017-11-25 00:12:44 +00:00
dest='epochs',
default=100,
type=int,
metavar='INT',
)
2018-01-12 09:52:44 +00:00
parser.add_argument(
'--use-mongodb',
help='parallelize evaluations with mongodb (requires mongod in PATH)',
dest='mongodb',
action='store_true',
)
2018-02-09 18:59:06 +00:00
parser.add_argument(
'-s', '--spaces',
help='Specify which parameters to hyperopt. Space separate list. \
Default: %(default)s',
choices=['all', 'buy', 'roi', 'stoploss'],
2018-02-09 18:59:06 +00:00
default='all',
nargs='+',
2018-02-09 18:59:06 +00:00
dest='spaces',
)
2017-11-14 21:15:24 +00:00
2018-01-15 21:25:02 +00:00
def parse_timerange(text):
if text is None:
return None
2018-01-28 22:28:28 +00:00
syntax = [(r'^-(\d{8})$', (None, 'date')),
(r'^(\d{8})-$', ('date', None)),
(r'^(\d{8})-(\d{8})$', ('date', 'date')),
(r'^(-\d+)$', (None, 'line')),
(r'^(\d+)-$', ('line', None)),
(r'^(\d+)-(\d+)$', ('index', 'index'))]
2018-01-15 21:25:02 +00:00
for rex, stype in syntax:
# Apply the regular expression to text
2018-01-28 22:28:28 +00:00
match = re.match(rex, text)
if match: # Regex has matched
rvals = match.groups()
index = 0
2018-01-15 21:25:02 +00:00
start = None
stop = None
if stype[0]:
2018-01-28 22:28:28 +00:00
start = rvals[index]
2018-01-15 21:25:02 +00:00
if stype[0] != 'date':
start = int(start)
2018-01-28 22:28:28 +00:00
index += 1
2018-01-15 21:25:02 +00:00
if stype[1]:
2018-01-28 22:28:28 +00:00
stop = rvals[index]
2018-01-15 21:25:02 +00:00
if stype[1] != 'date':
stop = int(stop)
return (stype, start, stop)
raise Exception('Incorrect syntax for timerange "%s"' % text)
2017-11-14 21:15:24 +00:00
2018-01-12 09:52:44 +00:00
def build_subcommands(parser: argparse.ArgumentParser) -> None:
""" Builds and attaches all subcommands """
from freqtrade.optimize import backtesting, hyperopt
subparsers = parser.add_subparsers(dest='subparser')
# Add backtesting subcommand
backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module')
backtesting_cmd.set_defaults(func=backtesting.start)
2018-02-15 13:23:49 +00:00
optimizer_shared_options(backtesting_cmd)
2018-01-12 09:52:44 +00:00
backtesting_options(backtesting_cmd)
# Add hyperopt subcommand
hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')
hyperopt_cmd.set_defaults(func=hyperopt.start)
2018-02-15 13:23:49 +00:00
optimizer_shared_options(hyperopt_cmd)
2018-01-12 09:52:44 +00:00
hyperopt_options(hyperopt_cmd)
# Required json-schema for user specified config
2017-09-08 21:10:22 +00:00
CONF_SCHEMA = {
'type': 'object',
'properties': {
2017-09-08 05:49:50 +00:00
'max_open_trades': {'type': 'integer', 'minimum': 1},
'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]},
'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']},
'stake_amount': {'type': 'number', 'minimum': 0.0005},
'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF',
'CLP', 'CNY', 'CZK', 'DKK',
'EUR', 'GBP', 'HKD', 'HUF',
'IDR', 'ILS', 'INR', 'JPY',
'KRW', 'MXN', 'MYR', 'NOK',
'NZD', 'PHP', 'PKR', 'PLN',
'RUB', 'SEK', 'SGD', 'THB',
'TRY', 'TWD', 'ZAR', 'USD']},
'dry_run': {'type': 'boolean'},
'minimal_roi': {
'type': 'object',
'patternProperties': {
'^[0-9.]+$': {'type': 'number'}
},
'minProperties': 1
},
'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True},
'unfilledtimeout': {'type': 'integer', 'minimum': 0},
'bid_strategy': {
'type': 'object',
'properties': {
'ask_last_balance': {
'type': 'number',
'minimum': 0,
'maximum': 1,
'exclusiveMaximum': False
},
},
'required': ['ask_last_balance']
},
2017-10-06 10:22:04 +00:00
'exchange': {'$ref': '#/definitions/exchange'},
'experimental': {
'type': 'object',
'properties': {
'use_sell_signal': {'type': 'boolean'},
'sell_profit_only': {'type': 'boolean'}
}
},
'telegram': {
'type': 'object',
'properties': {
'enabled': {'type': 'boolean'},
'token': {'type': 'string'},
'chat_id': {'type': 'string'},
},
'required': ['enabled', 'token', 'chat_id']
},
'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
2017-11-11 15:47:19 +00:00
'internals': {
'type': 'object',
'properties': {
'process_throttle_secs': {'type': 'number'},
'interval': {'type': 'integer'}
2017-11-11 15:47:19 +00:00
}
}
},
'definitions': {
'exchange': {
'type': 'object',
'properties': {
2017-10-06 10:22:04 +00:00
'name': {'type': 'string'},
'key': {'type': 'string'},
'secret': {'type': 'string'},
'pair_whitelist': {
'type': 'array',
'items': {
'type': 'string',
'pattern': '^[0-9A-Z]+_[0-9A-Z]+$'
},
'uniqueItems': True
},
'pair_blacklist': {
'type': 'array',
'items': {
'type': 'string',
'pattern': '^[0-9A-Z]+_[0-9A-Z]+$'
},
'uniqueItems': True
}
},
'required': ['name', 'key', 'secret', 'pair_whitelist']
}
},
'anyOf': [
2017-10-06 10:22:04 +00:00
{'required': ['exchange']}
],
'required': [
'max_open_trades',
'stake_currency',
'stake_amount',
'fiat_display_currency',
'dry_run',
'bid_strategy',
'telegram'
]
}