stable/freqtrade/misc.py

import argparse
import enum
import json
import logging
import time
import os
import re
from datetime import datetime
from typing import Any, Callable, Dict, List

import numpy as np
from jsonschema import Draft4Validator, validate
from jsonschema.exceptions import ValidationError, best_match
from wrapt import synchronized

from freqtrade import __version__

logger = logging.getLogger(__name__)


class State(enum.Enum):
    RUNNING = 0
    STOPPED = 1


# Current application state
_STATE = State.STOPPED


############################################
# Used by scripts                          #
# Matplotlib doesn't support ::datetime64, #
# so we need to convert it into ::datetime #
############################################

def datesarray_to_datetimearray(dates):
    """
    Convert an pandas-array of timestamps into
    An numpy-array of datetimes
    :return: numpy-array of datetime
    """
    times = []
    dates = dates.astype(datetime)
    for i in range(0, dates.size):
        date = dates[i].to_pydatetime()
        times.append(date)
    return np.array(times)


def common_datearray(dfs):
    alldates = {}
    for pair, pair_data in dfs.items():
        dates = datesarray_to_datetimearray(pair_data['date'])
        for date in dates:
            alldates[date] = 1
    lst = []
    for date, _ in alldates.items():
        lst.append(date)
    arr = np.array(lst)
    return np.sort(arr, axis=0)


def file_dump_json(filename, data) -> None:
    with open(filename, 'w') as fp:
        json.dump(data, fp)


@synchronized
def update_state(state: State) -> None:
    """
    Updates the application state
    :param state: new state
    :return: None
    """
    global _STATE
    _STATE = state


@synchronized
def get_state() -> State:
    """
    Gets the current application state
    :return:
    """
    return _STATE


def load_config(path: str) -> Dict:
    """
    Loads a config file from the given path
    :param path: path as str
    :return: configuration as dictionary
    """
    with open(path) as file:
        conf = json.load(file)
    if 'internals' not in conf:
        conf['internals'] = {}
    logger.info('Validating configuration ...')
    try:
        validate(conf, CONF_SCHEMA)
        return conf
    except ValidationError as exception:
        logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception)
        raise ValidationError(
            best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message
        )


def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any:
    """
    Throttles the given callable that it
    takes at least `min_secs` to finish execution.
    :param func: Any callable
    :param min_secs: minimum execution time in seconds
    :return: Any
    """
    start = time.time()
    result = func(*args, **kwargs)
    end = time.time()
    duration = max(min_secs - (end - start), 0.0)
    logger.debug('Throttling %s for %.2f seconds', func.__name__, duration)
    time.sleep(duration)
    return result


def common_args_parser(description: str):
    """
    Parses given common arguments and returns them as a parsed object.
    """
    parser = argparse.ArgumentParser(
        description=description
    )
    parser.add_argument(
        '-v', '--verbose',
        help='be verbose',
        action='store_const',
        dest='loglevel',
        const=logging.DEBUG,
        default=logging.INFO,
    )
    parser.add_argument(
        '--version',
        action='version',
        version='%(prog)s {}'.format(__version__),
    )
    parser.add_argument(
        '-c', '--config',
        help='specify configuration file (default: %(default)s)',
        dest='config',
        default='config.json',
        type=str,
        metavar='PATH',
    )
    parser.add_argument(
        '-d', '--datadir',
        help='path to backtest data (default: %(default)s',
        dest='datadir',
        default=os.path.join('freqtrade', 'tests', 'testdata'),
        type=str,
        metavar='PATH',
    )
    parser.add_argument(
        '-s', '--strategy',
        help='specify strategy file (default: %(default)s)',
        dest='strategy',
        default='default_strategy',
        type=str,
        metavar='PATH',
    )
    return parser


def parse_args(args: List[str], description: str):
    """
    Parses given arguments and returns an argparse Namespace instance.
    Returns None if a sub command has been selected and executed.
    """
    parser = common_args_parser(description)
    parser.add_argument(
        '--dry-run-db',
        help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \
             instead of memory DB. Work only if dry_run is enabled.',
        action='store_true',
        dest='dry_run_db',
    )
    parser.add_argument(
        '--dynamic-whitelist',
        help='dynamically generate and update whitelist \
             based on 24h BaseVolume (Default 20 currencies)',  # noqa
        dest='dynamic_whitelist',
        const=20,
        type=int,
        metavar='INT',
        nargs='?',
    )

    build_subcommands(parser)
    return parser.parse_args(args)


def scripts_options(parser: argparse.ArgumentParser) -> None:
    parser.add_argument(
        '-p', '--pair',
        help='Show profits for only this pairs. Pairs are comma-separated.',
        dest='pair',
        default=None
    )


def optimizer_shared_options(parser: argparse.ArgumentParser) -> None:
    parser.add_argument(
        '-i', '--ticker-interval',
        help='specify ticker interval in minutes (1, 5, 30, 60, 1440)',
        dest='ticker_interval',
        type=int,
        metavar='INT',
    )
    parser.add_argument(
        '--realistic-simulation',
        help='uses max_open_trades from config to simulate real world limitations',
        action='store_true',
        dest='realistic_simulation',
    )
    parser.add_argument(
        '--timerange',
        help='Specify what timerange of data to use.',
        default=None,
        type=str,
        dest='timerange',
    )


def backtesting_options(parser: argparse.ArgumentParser) -> None:
    parser.add_argument(
        '-l', '--live',
        action='store_true',
        dest='live',
        help='using live data',
    )
    parser.add_argument(
        '-r', '--refresh-pairs-cached',
        help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \
              Use it if you want to run your backtesting with up-to-date data.',
        action='store_true',
        dest='refresh_pairs',
    )
    parser.add_argument(
        '--export',
        help='Export backtest results, argument are: trades\
              Example --export=trades',
        type=str,
        default=None,
        dest='export',
    )


def hyperopt_options(parser: argparse.ArgumentParser) -> None:
    parser.add_argument(
        '-e', '--epochs',
        help='specify number of epochs (default: %(default)d)',
        dest='epochs',
        default=100,
        type=int,
        metavar='INT',
    )
    parser.add_argument(
        '--use-mongodb',
        help='parallelize evaluations with mongodb (requires mongod in PATH)',
        dest='mongodb',
        action='store_true',
    )
    parser.add_argument(
        '-s', '--spaces',
        help='Specify which parameters to hyperopt. Space separate list. \
              Default: %(default)s',
        choices=['all', 'buy', 'roi', 'stoploss'],
        default='all',
        nargs='+',
        dest='spaces',
    )


def parse_timerange(text):
    if text is None:
        return None
    syntax = [(r'^-(\d{8})$', (None, 'date')),
              (r'^(\d{8})-$', ('date', None)),
              (r'^(\d{8})-(\d{8})$', ('date', 'date')),
              (r'^(-\d+)$', (None, 'line')),
              (r'^(\d+)-$', ('line', None)),
              (r'^(\d+)-(\d+)$', ('index', 'index'))]
    for rex, stype in syntax:
        # Apply the regular expression to text
        match = re.match(rex, text)
        if match:  # Regex has matched
            rvals = match.groups()
            index = 0
            start = None
            stop = None
            if stype[0]:
                start = rvals[index]
                if stype[0] != 'date':
                    start = int(start)
                index += 1
            if stype[1]:
                stop = rvals[index]
                if stype[1] != 'date':
                    stop = int(stop)
            return (stype, start, stop)
    raise Exception('Incorrect syntax for timerange "%s"' % text)


def build_subcommands(parser: argparse.ArgumentParser) -> None:
    """ Builds and attaches all subcommands """
    from freqtrade.optimize import backtesting, hyperopt

    subparsers = parser.add_subparsers(dest='subparser')

    # Add backtesting subcommand
    backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module')
    backtesting_cmd.set_defaults(func=backtesting.start)
    optimizer_shared_options(backtesting_cmd)
    backtesting_options(backtesting_cmd)

    # Add hyperopt subcommand
    hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')
    hyperopt_cmd.set_defaults(func=hyperopt.start)
    optimizer_shared_options(hyperopt_cmd)
    hyperopt_options(hyperopt_cmd)


# Required json-schema for user specified config
CONF_SCHEMA = {
    'type': 'object',
    'properties': {
        'max_open_trades': {'type': 'integer', 'minimum': 1},
        'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]},
        'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']},
        'stake_amount': {'type': 'number', 'minimum': 0.0005},
        'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF',
                                                             'CLP', 'CNY', 'CZK', 'DKK',
                                                             'EUR', 'GBP', 'HKD', 'HUF',
                                                             'IDR', 'ILS', 'INR', 'JPY',
                                                             'KRW', 'MXN', 'MYR', 'NOK',
                                                             'NZD', 'PHP', 'PKR', 'PLN',
                                                             'RUB', 'SEK', 'SGD', 'THB',
                                                             'TRY', 'TWD', 'ZAR', 'USD']},
        'dry_run': {'type': 'boolean'},
        'minimal_roi': {
            'type': 'object',
            'patternProperties': {
                '^[0-9.]+$': {'type': 'number'}
            },
            'minProperties': 1
        },
        'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True},
        'unfilledtimeout': {'type': 'integer', 'minimum': 0},
        'bid_strategy': {
            'type': 'object',
            'properties': {
                'ask_last_balance': {
                    'type': 'number',
                    'minimum': 0,
                    'maximum': 1,
                    'exclusiveMaximum': False
                },
            },
            'required': ['ask_last_balance']
        },
        'exchange': {'$ref': '#/definitions/exchange'},
        'experimental': {
            'type': 'object',
            'properties': {
                'use_sell_signal': {'type': 'boolean'},
                'sell_profit_only': {'type': 'boolean'}
            }
        },
        'telegram': {
            'type': 'object',
            'properties': {
                'enabled': {'type': 'boolean'},
                'token': {'type': 'string'},
                'chat_id': {'type': 'string'},
            },
            'required': ['enabled', 'token', 'chat_id']
        },
        'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
        'internals': {
            'type': 'object',
            'properties': {
                'process_throttle_secs': {'type': 'number'},
                'interval': {'type': 'integer'}
            }
        }
    },
    'definitions': {
        'exchange': {
            'type': 'object',
            'properties': {
                'name': {'type': 'string'},
                'key': {'type': 'string'},
                'secret': {'type': 'string'},
                'pair_whitelist': {
                    'type': 'array',
                    'items': {
                        'type': 'string',
                        'pattern': '^[0-9A-Z]+_[0-9A-Z]+$'
                    },
                    'uniqueItems': True
                },
                'pair_blacklist': {
                    'type': 'array',
                    'items': {
                        'type': 'string',
                        'pattern': '^[0-9A-Z]+_[0-9A-Z]+$'
                    },
                    'uniqueItems': True
                }
            },
            'required': ['name', 'key', 'secret', 'pair_whitelist']
        }
    },
    'anyOf': [
        {'required': ['exchange']}
    ],
    'required': [
        'max_open_trades',
        'stake_currency',
        'stake_amount',
        'fiat_display_currency',
        'dry_run',
        'bid_strategy',
        'telegram'
    ]
}