Merge branch 'develop' into pr/hroff-1902/1804

This commit is contained in:
Matthias 2019-05-01 12:54:36 +02:00
commit b9d7bb2d8e
13 changed files with 121 additions and 163 deletions

View File

@ -22,7 +22,7 @@ requirements:
- requirements.txt
- requirements-dev.txt
- requirements-plot.txt
- requirements-pi.txt
- requirements-common.txt
# configure the branch prefix the bot is using

View File

@ -16,7 +16,7 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
ENV LD_LIBRARY_PATH /usr/local/lib
# Install dependencies
COPY requirements.txt /freqtrade/
COPY requirements.txt requirements-common.txt /freqtrade/
RUN pip install numpy --no-cache-dir \
&& pip install -r requirements.txt --no-cache-dir

View File

@ -27,9 +27,9 @@ RUN wget https://github.com/jjhelmus/berryconda/releases/download/v2.0.0/Berryco
&& rm Berryconda3-2.0.0-Linux-armv7l.sh
# Install dependencies
COPY requirements-pi.txt /freqtrade/
COPY requirements-common.txt /freqtrade/
RUN ~/berryconda3/bin/conda install -y numpy pandas scipy \
&& ~/berryconda3/bin/pip install -r requirements-pi.txt --no-cache-dir
&& ~/berryconda3/bin/pip install -r requirements-common.txt --no-cache-dir
# Install and execute
COPY . /freqtrade/

View File

@ -326,7 +326,7 @@ conda activate freqtrade
conda install scipy pandas numpy
sudo apt install libffi-dev
python3 -m pip install -r requirements-pi.txt
python3 -m pip install -r requirements-common.txt
python3 -m pip install -e .
```

View File

@ -7,7 +7,7 @@ import os
import sys
from argparse import Namespace
from logging.handlers import RotatingFileHandler
from typing import Any, Dict, List, Optional
from typing import Any, Callable, Dict, List, Optional
from jsonschema import Draft4Validator, validators
from jsonschema.exceptions import ValidationError, best_match
@ -17,7 +17,6 @@ from freqtrade.exchange import is_exchange_supported, supported_exchanges
from freqtrade.misc import deep_merge_dicts
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
@ -95,14 +94,8 @@ class Configuration(object):
# Load Common configuration
config = self._load_common_config(config)
# Load Backtesting
config = self._load_backtesting_config(config)
# Load Edge
config = self._load_edge_config(config)
# Load Hyperopt
config = self._load_hyperopt_config(config)
# Load Optimize configurations
config = self._load_optimize_config(config)
# Set runmode
if not self.runmode:
@ -216,25 +209,41 @@ class Configuration(object):
logger.info(f'Created data directory: {datadir}')
return datadir
def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: # noqa: C901
def _args_to_config(self, config: Dict[str, Any], argname: str,
logstring: str, logfun: Optional[Callable] = None) -> None:
"""
Extract information for sys.argv and load Backtesting configuration
:param config: Configuration dictionary
:param argname: Argumentname in self.args - will be copied to config dict.
:param logstring: Logging String
:param logfun: logfun is applied to the configuration entry before passing
that entry to the log string using .format().
sample: logfun=len (prints the length of the found
configuration instead of the content)
"""
if argname in self.args and getattr(self.args, argname):
config.update({argname: getattr(self.args, argname)})
if logfun:
logger.info(logstring.format(logfun(config[argname])))
else:
logger.info(logstring.format(config[argname]))
def _load_optimize_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""
Extract information for sys.argv and load Optimize configuration
:return: configuration as dictionary
"""
# This will override the strategy configuration
if 'ticker_interval' in self.args and self.args.ticker_interval:
config.update({'ticker_interval': self.args.ticker_interval})
logger.info('Parameter -i/--ticker-interval detected ...')
logger.info('Using ticker_interval: %s ...', config.get('ticker_interval'))
self._args_to_config(config, argname='ticker_interval',
logstring='Parameter -i/--ticker-interval detected ... '
'Using ticker_interval: {} ...')
if 'live' in self.args and self.args.live:
config.update({'live': True})
logger.info('Parameter -l/--live detected ...')
self._args_to_config(config, argname='live',
logstring='Parameter -l/--live detected ...')
if 'position_stacking' in self.args and self.args.position_stacking:
config.update({'position_stacking': True})
logger.info('Parameter --enable-position-stacking detected ...')
self._args_to_config(config, argname='position_stacking',
logstring='Parameter --enable-position-stacking detected ...')
if 'use_max_market_positions' in self.args and not self.args.use_max_market_positions:
config.update({'use_max_market_positions': False})
@ -247,14 +256,12 @@ class Configuration(object):
else:
logger.info('Using max_open_trades: %s ...', config.get('max_open_trades'))
if 'stake_amount' in self.args and self.args.stake_amount:
config.update({'stake_amount': self.args.stake_amount})
logger.info('Parameter --stake_amount detected, overriding stake_amount to: %s ...',
config.get('stake_amount'))
self._args_to_config(config, argname='stake_amount',
logstring='Parameter --stake_amount detected, '
'overriding stake_amount to: {} ...')
if 'timerange' in self.args and self.args.timerange:
config.update({'timerange': self.args.timerange})
logger.info('Parameter --timerange detected: %s ...', self.args.timerange)
self._args_to_config(config, argname='timerange',
logstring='Parameter --timerange detected: {} ...')
if 'datadir' in self.args and self.args.datadir:
config.update({'datadir': self._create_datadir(config, self.args.datadir)})
@ -262,38 +269,22 @@ class Configuration(object):
config.update({'datadir': self._create_datadir(config, None)})
logger.info('Using data folder: %s ...', config.get('datadir'))
if 'refresh_pairs' in self.args and self.args.refresh_pairs:
config.update({'refresh_pairs': True})
logger.info('Parameter -r/--refresh-pairs-cached detected ...')
self._args_to_config(config, argname='refresh_pairs',
logstring='Parameter -r/--refresh-pairs-cached detected ...')
if 'strategy_list' in self.args and self.args.strategy_list:
config.update({'strategy_list': self.args.strategy_list})
logger.info('Using strategy list of %s Strategies', len(self.args.strategy_list))
self._args_to_config(config, argname='strategy_list',
logstring='Using strategy list of {} Strategies', logfun=len)
if 'ticker_interval' in self.args and self.args.ticker_interval:
config.update({'ticker_interval': self.args.ticker_interval})
logger.info('Overriding ticker interval with Command line argument')
self._args_to_config(config, argname='ticker_interval',
logstring='Overriding ticker interval with Command line argument')
if 'export' in self.args and self.args.export:
config.update({'export': self.args.export})
logger.info('Parameter --export detected: %s ...', self.args.export)
self._args_to_config(config, argname='export',
logstring='Parameter --export detected: {} ...')
if 'export' in config and 'exportfilename' in self.args and self.args.exportfilename:
config.update({'exportfilename': self.args.exportfilename})
logger.info('Storing backtest results to %s ...', self.args.exportfilename)
return config
def _load_edge_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""
Extract information for sys.argv and load Edge configuration
:return: configuration as dictionary
"""
if 'timerange' in self.args and self.args.timerange:
config.update({'timerange': self.args.timerange})
logger.info('Parameter --timerange detected: %s ...', self.args.timerange)
self._args_to_config(config, argname='exportfilename',
logstring='Storing backtest results to {} ...')
# Edge section:
if 'stoploss_range' in self.args and self.args.stoploss_range:
txt_range = eval(self.args.stoploss_range)
config['edge'].update({'stoploss_range_min': txt_range[0]})
@ -301,48 +292,26 @@ class Configuration(object):
config['edge'].update({'stoploss_range_step': txt_range[2]})
logger.info('Parameter --stoplosses detected: %s ...', self.args.stoploss_range)
if 'refresh_pairs' in self.args and self.args.refresh_pairs:
config.update({'refresh_pairs': True})
logger.info('Parameter -r/--refresh-pairs-cached detected ...')
# Hyperopt section
self._args_to_config(config, argname='hyperopt',
logstring='Using Hyperopt file {}')
return config
self._args_to_config(config, argname='epochs',
logstring='Parameter --epochs detected ... '
'Will run Hyperopt with for {} epochs ...'
)
def _load_hyperopt_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""
Extract information for sys.argv and load Hyperopt configuration
:return: configuration as dictionary
"""
self._args_to_config(config, argname='spaces',
logstring='Parameter -s/--spaces detected: {}')
if "hyperopt" in self.args:
# Add the hyperopt file to use
config.update({'hyperopt': self.args.hyperopt})
self._args_to_config(config, argname='print_all',
logstring='Parameter --print-all detected ...')
if 'epochs' in self.args and self.args.epochs:
config.update({'epochs': self.args.epochs})
logger.info('Parameter --epochs detected ...')
logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs'))
if 'spaces' in self.args and self.args.spaces:
config.update({'spaces': self.args.spaces})
logger.info('Parameter -s/--spaces detected: %s', config.get('spaces'))
if 'print_all' in self.args and self.args.print_all:
config.update({'print_all': self.args.print_all})
logger.info('Parameter --print-all detected: %s', config.get('print_all'))
if 'hyperopt_jobs' in self.args and self.args.hyperopt_jobs:
config.update({'hyperopt_jobs': self.args.hyperopt_jobs})
logger.info('Parameter -j/--job-workers detected: %s', config.get('hyperopt_jobs'))
if 'refresh_pairs' in self.args and self.args.refresh_pairs:
config.update({'refresh_pairs': True})
logger.info('Parameter -r/--refresh-pairs-cached detected ...')
if 'hyperopt_random_state' in self.args and self.args.hyperopt_random_state is not None:
config.update({'hyperopt_random_state': self.args.hyperopt_random_state})
logger.info("Parameter --random-state detected: %s",
config.get('hyperopt_random_state'))
self._args_to_config(config, argname='hyperopt_jobs',
logstring='Parameter -j/--job-workers detected: {}')
self._args_to_config(config, argname='hyperopt_random_state',
logstring='Parameter --random-state detected: {}')
return config
def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]:

View File

@ -23,7 +23,7 @@ from freqtrade.optimize.backtesting import (Backtesting, setup_configuration,
from freqtrade.state import RunMode
from freqtrade.strategy.default_strategy import DefaultStrategy
from freqtrade.strategy.interface import SellType
from freqtrade.tests.conftest import log_has, patch_exchange
from freqtrade.tests.conftest import log_has, log_has_re, patch_exchange
def get_args(args) -> List[str]:
@ -190,7 +190,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
caplog.record_tuples
)
assert 'ticker_interval' in config
assert not log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog.record_tuples)
assert 'live' not in config
assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples)
@ -242,11 +242,8 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
caplog.record_tuples
)
assert 'ticker_interval' in config
assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
assert log_has(
'Using ticker_interval: 1m ...',
caplog.record_tuples
)
assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
caplog.record_tuples)
assert 'live' in config
assert log_has('Parameter -l/--live detected ...', caplog.record_tuples)
@ -853,8 +850,7 @@ def test_backtest_start_live(default_conf, mocker, caplog):
start(args)
# check the logs, that will contain the backtest result
exists = [
'Parameter -i/--ticker-interval detected ...',
'Using ticker_interval: 1m ...',
'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
'Parameter -l/--live detected ...',
'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
'Parameter --timerange detected: -100 ...',
@ -912,8 +908,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog):
# check the logs, that will contain the backtest result
exists = [
'Parameter -i/--ticker-interval detected ...',
'Using ticker_interval: 1m ...',
'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
'Parameter -l/--live detected ...',
'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
'Parameter --timerange detected: -100 ...',

View File

@ -1,14 +1,15 @@
# pragma pylint: disable=missing-docstring, C0103, C0330
# pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments
from unittest.mock import MagicMock
import json
from typing import List
from freqtrade.edge import PairInfo
from unittest.mock import MagicMock
from freqtrade.arguments import Arguments
from freqtrade.optimize.edge_cli import (EdgeCli, setup_configuration, start)
from freqtrade.edge import PairInfo
from freqtrade.optimize.edge_cli import EdgeCli, setup_configuration, start
from freqtrade.state import RunMode
from freqtrade.tests.conftest import log_has, patch_exchange
from freqtrade.tests.conftest import log_has, log_has_re, patch_exchange
def get_args(args) -> List[str]:
@ -40,7 +41,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
caplog.record_tuples
)
assert 'ticker_interval' in config
assert not log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog.record_tuples)
assert 'refresh_pairs' not in config
assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
@ -79,11 +80,8 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N
caplog.record_tuples
)
assert 'ticker_interval' in config
assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
assert log_has(
'Using ticker_interval: 1m ...',
caplog.record_tuples
)
assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
caplog.record_tuples)
assert 'refresh_pairs' in config
assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)

View File

@ -1,7 +1,7 @@
# pragma pylint: disable=missing-docstring,W0212,C0103
from datetime import datetime
import json
import os
from datetime import datetime
from unittest.mock import MagicMock
import pandas as pd
@ -10,11 +10,11 @@ import pytest
from freqtrade import DependencyException
from freqtrade.data.converter import parse_ticker_dataframe
from freqtrade.data.history import load_tickerdata_file
from freqtrade.optimize.hyperopt import Hyperopt, start, setup_configuration
from freqtrade.optimize.default_hyperopt import DefaultHyperOpts
from freqtrade.resolvers import StrategyResolver, HyperOptResolver
from freqtrade.optimize.hyperopt import Hyperopt, setup_configuration, start
from freqtrade.resolvers import HyperOptResolver
from freqtrade.state import RunMode
from freqtrade.tests.conftest import log_has, patch_exchange
from freqtrade.tests.conftest import log_has, log_has_re, patch_exchange
from freqtrade.tests.optimize.test_backtesting import get_args
@ -64,7 +64,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca
caplog.record_tuples
)
assert 'ticker_interval' in config
assert not log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog.record_tuples)
assert 'live' not in config
assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples)
@ -114,11 +114,8 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo
caplog.record_tuples
)
assert 'ticker_interval' in config
assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
assert log_has(
'Using ticker_interval: 1m ...',
caplog.record_tuples
)
assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
caplog.record_tuples)
assert 'position_stacking' in config
assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples)
@ -137,7 +134,8 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo
)
assert 'epochs' in config
assert log_has('Parameter --epochs detected ...', caplog.record_tuples)
assert log_has('Parameter --epochs detected ... Will run Hyperopt with for 1000 epochs ...',
caplog.record_tuples)
assert 'spaces' in config
assert log_has(
@ -145,7 +143,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo
caplog.record_tuples
)
assert 'print_all' in config
assert log_has('Parameter --print-all detected: True', caplog.record_tuples)
assert log_has('Parameter --print-all detected ...', caplog.record_tuples)
def test_hyperoptresolver(mocker, default_conf, caplog) -> None:
@ -185,7 +183,6 @@ def test_start(mocker, default_conf, caplog) -> None:
'--epochs', '5'
]
args = get_args(args)
StrategyResolver({'strategy': 'DefaultStrategy'})
start(args)
import pprint
@ -214,7 +211,6 @@ def test_start_failure(mocker, default_conf, caplog) -> None:
'--epochs', '5'
]
args = get_args(args)
StrategyResolver({'strategy': 'DefaultStrategy'})
with pytest.raises(DependencyException):
start(args)
assert log_has(
@ -224,7 +220,6 @@ def test_start_failure(mocker, default_conf, caplog) -> None:
def test_loss_calculation_prefer_correct_trade_count(hyperopt) -> None:
StrategyResolver({'strategy': 'DefaultStrategy'})
correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20)
over = hyperopt.calculate_loss(1, hyperopt.target_trades + 100, 20)

View File

@ -360,11 +360,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
caplog.record_tuples
)
assert 'ticker_interval' in config
assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
assert log_has(
'Using ticker_interval: 1m ...',
caplog.record_tuples
)
assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
caplog.record_tuples)
assert 'live' in config
assert log_has('Parameter -l/--live detected ...', caplog.record_tuples)
@ -425,11 +422,8 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non
caplog.record_tuples
)
assert 'ticker_interval' in config
assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples)
assert log_has(
'Using ticker_interval: 1m ...',
caplog.record_tuples
)
assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
caplog.record_tuples)
assert 'strategy_list' in config
assert log_has('Using strategy list of 2 Strategies', caplog.record_tuples)
@ -463,8 +457,8 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None:
assert 'epochs' in config
assert int(config['epochs']) == 10
assert log_has('Parameter --epochs detected ...', caplog.record_tuples)
assert log_has('Will run Hyperopt with for 10 epochs ...', caplog.record_tuples)
assert log_has('Parameter --epochs detected ... Will run Hyperopt with for 10 epochs ...',
caplog.record_tuples)
assert 'spaces' in config
assert config['spaces'] == ['all']

View File

@ -7,10 +7,11 @@ import pytest
from freqtrade import OperationalException
from freqtrade.arguments import Arguments
from freqtrade.worker import Worker
from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.main import main
from freqtrade.state import State
from freqtrade.tests.conftest import log_has, patch_exchange
from freqtrade.worker import Worker
def test_parse_args_backtesting(mocker) -> None:
@ -107,24 +108,30 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None:
def test_main_reload_conf(mocker, default_conf, caplog) -> None:
patch_exchange(mocker)
mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cleanup', MagicMock())
mocker.patch('freqtrade.worker.Worker._worker', MagicMock(return_value=State.RELOAD_CONF))
# Simulate Running, reload, running workflow
worker_mock = MagicMock(side_effect=[State.RUNNING,
State.RELOAD_CONF,
State.RUNNING,
OperationalException("Oh snap!")])
mocker.patch('freqtrade.worker.Worker._worker', worker_mock)
mocker.patch(
'freqtrade.configuration.Configuration._load_config_file',
lambda *args, **kwargs: default_conf
)
reconfigure_mock = mocker.patch('freqtrade.main.Worker._reconfigure', MagicMock())
mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
# Raise exception as side effect to avoid endless loop
reconfigure_mock = mocker.patch(
'freqtrade.main.Worker._reconfigure', MagicMock(side_effect=Exception)
)
args = Arguments(['-c', 'config.json.example'], '').get_parsed_arg()
worker = Worker(args=args, config=default_conf)
with pytest.raises(SystemExit):
main(['-c', 'config.json.example'])
assert reconfigure_mock.call_count == 1
assert log_has('Using config: config.json.example ...', caplog.record_tuples)
assert worker_mock.call_count == 4
assert reconfigure_mock.call_count == 1
assert isinstance(worker.freqtrade, FreqtradeBot)
def test_reconfigure(mocker, default_conf) -> None:

View File

@ -39,7 +39,7 @@ class Worker(object):
logger.debug("sd_notify: READY=1")
self._sd_notify.notify("READY=1")
def _init(self, reconfig: bool):
def _init(self, reconfig: bool) -> None:
"""
Also called from the _reconfigure() method (with reconfig=True).
"""
@ -63,17 +63,17 @@ class Worker(object):
return self.freqtrade.state
@state.setter
def state(self, value: State):
def state(self, value: State) -> None:
self.freqtrade.state = value
def run(self):
def run(self) -> None:
state = None
while True:
state = self._worker(old_state=state)
if state == State.RELOAD_CONF:
self.freqtrade = self._reconfigure()
self._reconfigure()
def _worker(self, old_state: State, throttle_secs: Optional[float] = None) -> State:
def _worker(self, old_state: Optional[State], throttle_secs: Optional[float] = None) -> State:
"""
Trading routine that must be run at each loop
:param old_state: the previous service state from the previous call
@ -148,7 +148,7 @@ class Worker(object):
# state_changed = True
return state_changed
def _reconfigure(self):
def _reconfigure(self) -> None:
"""
Cleans up current freqtradebot instance, reloads the configuration and
replaces it with the new instance
@ -174,7 +174,7 @@ class Worker(object):
logger.debug("sd_notify: READY=1")
self._sd_notify.notify("READY=1")
def exit(self):
def exit(self) -> None:
# Tell systemd that we are exiting now
if self._sd_notify:
logger.debug("sd_notify: STOPPING=1")

View File

@ -1,6 +1,6 @@
# requirements without requirements installable via conda
# mainly used for Raspberry pi installs
ccxt==1.18.489
ccxt==1.18.496
SQLAlchemy==1.3.3
python-telegram-bot==11.1.0
arrow==0.13.1

View File

@ -1,5 +1,5 @@
# Load common requirements
-r requirements-pi.txt
-r requirements-common.txt
numpy==1.16.3
pandas==0.24.2